From 8266155d879079929220b94eafcfc18ac29a1980 Mon Sep 17 00:00:00 2001 From: zhanghanduo Date: Tue, 1 Sep 2026 10:13:48 +0800 Subject: [PATCH 1/2] refactor: extract shared agent loop engine --- README.md | 15 +- agent_core/runtime/loop/__init__.py | 29 + agent_core/runtime/loop/agent_loop.py | 1235 +++++++++++++++++++ agent_core/runtime/loop/model_profile.py | 635 ++++++++++ agent_core/runtime/loop/tool_call_parser.py | 726 +++++++++++ agent_core/runtime/loop/tool_exec.py | 285 +++++ docs/agent-loop-boundary.md | 28 + tests/test_agent_loop_engine.py | 155 +++ tests/test_model_profile_native.py | 215 ++++ tests/test_multi_format_tool_call_parser.py | 299 +++++ tests/test_thinking_history_policy.py | 208 ++++ tests/test_tool_call_parser_generic.py | 228 ++++ tests/test_tool_call_parser_mixed_native.py | 93 ++ tests/test_tool_exec.py | 120 ++ 14 files changed, 4266 insertions(+), 5 deletions(-) create mode 100644 agent_core/runtime/loop/agent_loop.py create mode 100644 agent_core/runtime/loop/model_profile.py create mode 100644 agent_core/runtime/loop/tool_call_parser.py create mode 100644 agent_core/runtime/loop/tool_exec.py create mode 100644 docs/agent-loop-boundary.md create mode 100644 tests/test_agent_loop_engine.py create mode 100644 tests/test_model_profile_native.py create mode 100644 tests/test_multi_format_tool_call_parser.py create mode 100644 tests/test_thinking_history_policy.py create mode 100644 tests/test_tool_call_parser_generic.py create mode 100644 tests/test_tool_call_parser_mixed_native.py create mode 100644 tests/test_tool_exec.py diff --git a/README.md b/README.md index ea8e609..aeca97f 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ Version `0.1.x` contains the converged foundation layer: - streamed tool-call recovery checks for missing required arguments. - LLM binding, response normalization, streaming assembly/watchdogs, retry classification, runaway recovery, and physical-call orchestration. +- model profiles, thinking/history normalization, and multi-format tool-call + parsing; +- parallel tool execution and the complete agent-loop orchestration engine, + with product behavior isolated behind typed hooks. The initial extraction is based on the already-merged integration branches: @@ -32,10 +36,10 @@ The initial extraction is based on the already-merged integration branches: Those revisions are provenance, not runtime dependencies. AgentCore tests and builds without either product checkout. -The agent loop remains in the products until tool execution, model profiles, -tool-call parsing, and execution-context storage have converged boundaries. -The LLM runtime accepts the three remaining product decisions through explicit -hooks: wall-deadline lookup, provider-chain state, and sticky-session policy. +The shared loop accepts product decisions through explicit hooks. Products own +execution-context storage, endpoint registries, metering, deadline policy, +result spilling and aggregate budgets; they no longer need private copies of +the orchestration or parsing engines. ## Repository boundary @@ -115,7 +119,8 @@ edit both products' core copies, that is evidence it belongs here. classification, and explicit product hooks are shared. 3. **LLM runtime** (complete): binding, calls, streaming, response normalization, runaway recovery, and the public `llm_client` facade. -4. **Agent loop:** model/tool parsing, tool execution, and `agent_loop`. +4. **Agent loop** (complete in AgentCore): model/tool parsing, hook-driven tool + execution, and `agent_loop` orchestration. 5. Remove product compatibility facades after downstream imports have moved to `agent_core`. diff --git a/agent_core/runtime/loop/__init__.py b/agent_core/runtime/loop/__init__.py index f7ddd1e..42e0c1d 100644 --- a/agent_core/runtime/loop/__init__.py +++ b/agent_core/runtime/loop/__init__.py @@ -1,5 +1,6 @@ """Shared loop foundation primitives.""" +from agent_core.runtime.loop.agent_loop import AgentLoopHooks, run_agent_loop from agent_core.runtime.loop.compact import ( DefaultCompactionPolicy, DefaultMessageCompactor, @@ -28,28 +29,56 @@ NullTrimmer, TaskBoundaryTrimmer, ) +from agent_core.runtime.loop.model_profile import ( + DefaultThinkingParser, + HistoryPolicy, + ModelProfile, + NativeMessageNormalizer, + configure_model_registry, +) +from agent_core.runtime.loop.tool_call_parser import ( + DefaultToolCallParser, + MultiFormatToolCallParser, +) +from agent_core.runtime.loop.tool_exec import ( + DefaultToolResultPostProcessor, + ToolExecutionHooks, + execute_tools, +) __all__ = [ "RUNAWAY_STATE_KEY", "TRUNCATION_CONTINUATION_GUIDANCE", + "AgentLoopHooks", "DefaultCompactionPolicy", "DefaultMessageCompactor", + "DefaultThinkingParser", + "DefaultToolCallParser", + "DefaultToolResultPostProcessor", + "HistoryPolicy", "LLMCallExhausted", "LLMDeadlineExceeded", "LLMReasoningRunaway", "LLMStreamStalled", "MessageTrimmer", + "ModelProfile", + "MultiFormatToolCallParser", + "NativeMessageNormalizer", "NullTrimmer", "TaskBoundaryTrimmer", "ThinkTagSplitter", + "ToolExecutionHooks", "bind_max_tokens", "bind_session_id", "bind_temperature", "bind_tools", "call_llm", + "configure_model_registry", + "execute_tools", "extract_final_content", "extract_leaked_reasoning", "extract_model_name", "extract_usage", "is_truncated_with_text", + "run_agent_loop", ] diff --git a/agent_core/runtime/loop/agent_loop.py b/agent_core/runtime/loop/agent_loop.py new file mode 100644 index 0000000..06b4675 --- /dev/null +++ b/agent_core/runtime/loop/agent_loop.py @@ -0,0 +1,1235 @@ +# pyright: reportMissingTypeArgument=false, reportUnknownArgumentType=false, reportUnknownMemberType=false, reportUnknownParameterType=false, reportUnknownVariableType=false, reportUnnecessaryIsInstance=false +"""Domain-neutral, config-driven ReAct loop. + +Workflow phases, terminal tools, and recovery policy are injected through +configuration and observers rather than implemented in this kernel. +""" +from __future__ import annotations + +import asyncio +import contextlib +import inspect +import logging +import time +import uuid +from collections.abc import Awaitable, Callable, Coroutine, Sequence +from dataclasses import dataclass, field +from typing import Any + +from agent_core.llm import LLMClient +from agent_core.loop_types import ( + AgentLoopResult, + CompactionEvent, + LLMAttemptContext, + LLMDeltaContext, + LoopConfig, + LoopPolicy, + ToolResult, + TurnContext, + UsageMetadata, + merge_interventions, + notify_observers, + notify_tool_call, + notify_tool_result, +) +from agent_core.messages import ( + Message, + is_assistant_msg, + is_tool_msg, + system_msg, + tool_msg, + user_msg, +) +from agent_core.runtime.loop.compact import ( + COMPACTION_SEQ_KEY, + FORCE_COMPACTION_KEY, + INPUT_ESTIMATE_KEY, + DefaultCompactionPolicy, + DefaultMessageCompactor, + estimate_tokens, +) +from agent_core.runtime.loop.llm_client import ( + RUNAWAY_STATE_KEY, + TRUNCATION_CONTINUATION_GUIDANCE, + LLMCallExhausted, + bind_session_id, + bind_temperature, + bind_tools, + call_llm, + estimate_message_tokens, + estimate_text_tokens, + extract_final_content, + extract_leaked_reasoning, + extract_usage, + is_truncated_with_text, +) +from agent_core.runtime.loop.model_profile import ( + DefaultThinkingParser, + HistoryPolicy, + ModelProfile, + NativeMessageNormalizer, +) +from agent_core.runtime.loop.tool_call_parser import ( + MultiFormatToolCallParser, + ToolCallParser, +) +from agent_core.runtime.loop.tool_exec import ( + DefaultToolResultPostProcessor, + ToolExecutionHooks, + ToolLike, + execute_tools, +) + +logger = logging.getLogger(__name__) + + +# Rollback-attempts budget above ``cfg.max_turns``: when a rollback +# observer fires ``continue_to_next_turn=True`` we DON'T consume a turn +# from the ``max_turns`` budget, but we still cap total iterations at +# ``max_turns + EXTRA_ATTEMPTS_BUFFER`` to prevent runaway rollback +# loops (e.g. a flaky LLM that keeps emitting refusals/duplicates). +EXTRA_ATTEMPTS_BUFFER = 200 + + +# Signature: (turn_index, messages_snapshot, metadata) -> awaitable None. +# Fires once per completed turn, after observer `on_turn_end` and any +# message compaction. Exceptions are caught by the loop — a failing +# checkpoint writer never kills a run. +TurnCompleteHook = Callable[ + [int, list[Message], dict[str, Any]], Awaitable[None], +] + +# Signature: () -> awaitable bool. Returning ``True`` signals a graceful +# pause — the loop stops AFTER the current turn's checkpoint has been +# persisted. Exceptions are caught and treated as "no pause" so a +# broken pause-status reader can't brick a run. +PauseCheckHook = Callable[[], Awaitable[bool]] + + +def _false() -> bool: + return False + + +def _body_has_no_spill(_body: str) -> bool: + return False + + +def _no_deadline() -> float | None: + return None + + +def _enter_no_scope( + _cfg: LoopConfig, _phase_id: str, _metadata: dict[str, Any] +) -> tuple[Any, Any]: + return None, None + + +def _exit_no_scope(_token: Any) -> None: + return None + + +async def _no_cancel_cleanup() -> None: + return None + + +@dataclass(frozen=True) +class AgentLoopHooks: + """Product-owned runtime state injected around the shared loop engine.""" + + sticky_session_enabled: Callable[[], bool] | None = None + bind_session: Callable[[Any, str], Any] | None = None + wall_deadline_remaining: Callable[[], float | None] = _no_deadline + chain_fallback_active: Callable[[], bool] = _false + enter_scope: Callable[ + [LoopConfig, str, dict[str, Any]], tuple[Any, Any] + ] = _enter_no_scope + exit_scope: Callable[[Any], None] = _exit_no_scope + cancellation_cleanup: Callable[[], Awaitable[None]] = _no_cancel_cleanup + body_has_spill_reference: Callable[[str], bool] = _body_has_no_spill + tool_execution: ToolExecutionHooks = field(default_factory=ToolExecutionHooks) + + +async def _wait_for_tool_interrupt( + observers: list[Any], ctx: TurnContext, tool_call: dict, +) -> bool: + """Wait until any observer asks to interrupt a parked fan-in tool.""" + # ``observers`` is list[Any], so the hook has to be duck-typed off each one. + # Annotating the getattr result states the expected shape: narrowing a bare + # Any through callable() leaves a callable returning ``object``, which + # create_task rejects. + waiters: list[asyncio.Task[bool]] = [] + for observer in observers: + fn: Callable[..., Coroutine[Any, Any, bool]] | None = getattr( + observer, "wait_for_tool_interrupt", None, + ) + if callable(fn): + waiters.append(asyncio.create_task(fn(ctx, tool_call))) + if not waiters: + return False + pending = set(waiters) + try: + while pending: + done, pending = await asyncio.wait( + pending, return_when=asyncio.FIRST_COMPLETED, + ) + for task in done: + try: + if bool(task.result()): + return True + except asyncio.CancelledError: + raise + except Exception: + logger.warning( + "Observer wait_for_tool_interrupt failed", + exc_info=True, + ) + return False + finally: + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + + +async def run_agent_loop( + *, + system_prompt: str, + user_message: str, + llm: LLMClient, + tools: Sequence[ToolLike], + config: LoopConfig | None = None, + observers: list[Any] | None = None, + parser: ToolCallParser | None = None, + model_profile: ModelProfile | None = None, + history_policy: HistoryPolicy | None = None, + initial_messages: list[Message] | None = None, + on_turn_complete: TurnCompleteHook | None = None, + pause_check: PauseCheckHook | None = None, + scope_metadata: dict[str, Any] | None = None, + runtime_hooks: AgentLoopHooks | None = None, +) -> AgentLoopResult: + """Run a generic ReAct loop with observer and persistence hooks. + + Persistence precedes the pause probe, ensuring a paused run is resumable. + Hook failures are isolated from the loop. + """ + cfg = config or LoopConfig() + runtime = runtime_hooks or AgentLoopHooks() + obs = observers or [] + tc_parser = parser or MultiFormatToolCallParser() + profile = model_profile or ModelProfile(model_id="default", provider="openai") + policy = history_policy or HistoryPolicy() + thinking_parser = DefaultThinkingParser() + normalizer = NativeMessageNormalizer() + + tool_map: dict[str, ToolLike] = {t.name: t for t in tools} + tool_names: set[str] = set(tool_map.keys()) + + # Pin one conversation to one upstream worker when affinity is enabled. + llm_session_id = cfg.llm_session_id or cfg.task_id + if runtime.bind_session is not None: + llm_with_session = runtime.bind_session(llm, llm_session_id) + else: + llm_with_session = bind_session_id( + llm, + llm_session_id, + sticky_session_enabled=runtime.sticky_session_enabled, + ) + llm_with_tools = bind_tools(llm_with_session, list(tools)) + + # Empty user input resumes the supplied history without adding a turn. + if initial_messages is not None: + messages: list[Message] = list(initial_messages) + if user_message: + messages.append(user_msg(user_message)) + else: + messages = [ + system_msg(system_prompt), + user_msg(user_message), + ] + + metadata: dict[str, Any] = {"role_id": cfg.role_id} + + scope_meta: dict[str, Any] = {"agent_id": cfg.role_id} + if scope_metadata: + scope_meta.update(scope_metadata) + scope_meta.setdefault("llm_session_id", llm_session_id) + scope, scope_token = runtime.enter_scope(cfg, _resolve_phase_id(cfg), scope_meta) + + try: + return await _run_loop_inner( + cfg, obs, tc_parser, profile, policy, thinking_parser, + normalizer, tool_map, tool_names, llm_with_session, llm_with_tools, + messages, metadata, on_turn_complete, pause_check, + scope=scope, runtime_hooks=runtime, + ) + except asyncio.CancelledError: + # The loop task was cancelled mid-flight (wall deadline, fan-out + # timeout, caller gather teardown). ``on_loop_end`` never fires + # on this path — it is the last statement of ``_run_loop_inner`` + # — so observers holding live resources (e.g. an observer's own + # background snapshot-build task) would leak and later surface as + # asyncio's "Task was destroyed but it is pending!". Give them + # one bounded best-effort teardown pass, then let the + # cancellation propagate unchanged. + await _notify_loop_cancelled(obs) + try: + await runtime.cancellation_cleanup() + except asyncio.CancelledError: + raise + except Exception: + logger.warning("Product cancellation cleanup failed", exc_info=True) + raise + finally: + runtime.exit_scope(scope_token) + + +# Per-observer wall for the cancellation teardown pass. Deliberately short: +# the canceller is awaiting us, so this is cleanup-only — no finalize / LLM +# work belongs here (that's ``on_loop_end``'s job on the normal path). +_CANCEL_TEARDOWN_TIMEOUT_S = 10.0 + + +async def _notify_loop_cancelled(observers: list[Any]) -> None: + """Bounded best-effort ``on_loop_cancelled`` fan-out. + + Unlike ``notify_observers`` this is always awaited — fire-and-forget + would recreate the very leak it exists to fix — and each observer + gets its own short timeout so one stuck teardown can't pin the + canceller. A repeat cancellation aborts the pass immediately (the + caller re-raises CancelledError either way). + """ + for observer in observers: + fn = getattr(observer, "on_loop_cancelled", None) + if fn is None: + continue + try: + await asyncio.wait_for(fn(), timeout=_CANCEL_TEARDOWN_TIMEOUT_S) + except asyncio.CancelledError: + raise # re-cancelled while tearing down — stop immediately + except Exception as exc: + logger.warning( + "on_loop_cancelled failed for %s: %s", + type(observer).__name__, exc, + ) + + +async def _run_loop_inner( + cfg: LoopConfig, + obs: list, + tc_parser: Any, + profile: Any, + policy: Any, + thinking_parser: Any, + normalizer: Any, + tool_map: dict[str, ToolLike], + tool_names: set[str], + llm_with_session: Any, + llm_with_tools: Any, + messages: list[Message], + metadata: dict[str, Any], + on_turn_complete: TurnCompleteHook | None = None, + pause_check: PauseCheckHook | None = None, + scope: Any = None, + runtime_hooks: AgentLoopHooks | None = None, +) -> AgentLoopResult: + """Inner loop extracted so run_agent_loop can wrap with ExecutionScope.""" + runtime = runtime_hooks or AgentLoopHooks() + + await notify_observers(obs, "on_loop_start", cfg) + + stop_reason = "" + total_tool_calls = 0 + no_tool_retries = 0 + truncation_continuations = 0 + truncated_text_parts: list[str] = [] + + last_input_tokens = 0 + last_output_tokens = 0 + + stream_llm_tokens = bool( + cfg.reasoning_only_timeout_s or cfg.reasoning_only_max_tokens + ) or any( + bool(getattr(observer, "wants_llm_delta", False)) + for observer in obs + ) + if getattr(profile, "protocol", "chat_completions") in ( + "anthropic", "responses", "bedrock", + ): + stream_llm_tokens = False + + max_attempts = cfg.max_turns + EXTRA_ATTEMPTS_BUFFER + turn = 0 + attempts = 0 + + while turn < cfg.max_turns and attempts < max_attempts: + turn += 1 + attempts += 1 + if scope is not None: + scope.metadata["current_turn"] = turn + + ( + llm_for_turn, messages_for_call, strip_tools, stop_reason, + ) = await _prepare_llm_request( + cfg, obs, llm_with_session, llm_with_tools, messages, metadata, turn + ) + if stop_reason: + break + + ( + response, stop_reason, first_delta_at, llm_call_started, + llm_call_finished, + call_id, current_attempt_id, current_attempt_index + ) = await _call_llm_with_callbacks( + cfg, obs, profile, llm_for_turn, messages_for_call, metadata, turn, + stream_llm_tokens, runtime + ) + if stop_reason: + break + + ( + parsed_calls, ctx, stop_reason, + continue_to_next_turn, skip_tool_execution, + last_input_tokens, last_output_tokens, + ) = await _process_llm_response( + cfg, obs, tc_parser, profile, policy, thinking_parser, normalizer, + tool_names, messages, metadata, turn, response, strip_tools, + last_input_tokens, + last_output_tokens, first_delta_at, llm_call_started, llm_call_finished, + call_id, current_attempt_id, current_attempt_index + ) + if stop_reason: + break + + if continue_to_next_turn: + turn -= 1 + continue + + # A reply the output cap cut off is the one case where we KNOW the model + # was not finished, and it must never reach the ``no_tool`` branch below: + # truncation and "the model chose to stop talking" are opposite signals + # that happen to arrive with the same shape (no tool call), and under + # ``no_tool_behavior="stop"`` sharing that exit ends the run on a + # sentence cut mid-token. Checked before the branch, and on its own + # budget, so a truncation never spends the nudge allowance. + if not parsed_calls and is_truncated_with_text(response): + truncation_continuations += 1 + truncated_text_parts.append(ctx.ai_text) + if truncation_continuations <= cfg.truncation_max_continuations: + logger.warning( + "turn=%d response truncated at the output cap with visible " + "text — continuing (%d/%d)", + turn, truncation_continuations, cfg.truncation_max_continuations, + ) + # The partial text is already in history (``_process_llm_response`` + # appended it), so the work survives and the model is asked to + # resume from it rather than restart. + messages.append(user_msg(TRUNCATION_CONTINUATION_GUIDANCE)) + # Continuations have their own bounded retry allowance. Do not + # consume the workflow's logical turn budget, especially on the + # landing turn where no later turn would otherwise exist. + turn -= 1 + continue + # A model that truncates every continuation gets its own stop reason + # rather than ``no_tool``. The diagnosis was the invisible part of + # this failure: the trajectory showed a short sentence and a + # clean-looking ``no_tool``, which reads as a finished run. + stop_reason = "response_truncated" + metadata["_truncation_final_content"] = "".join( + truncated_text_parts, + ) + logger.warning( + "turn=%d response truncated after %d continuation(s) — stopping", + turn, cfg.truncation_max_continuations, + ) + break + + if not parsed_calls: + no_tool_retries += 1 + stops_for_no_tool = ( + cfg.loop_policy.no_tool_behavior != "nudge" + or no_tool_retries >= cfg.no_tool_max_retries + ) + if truncated_text_parts: + # ``extract_final_content`` normally returns only the latest + # assistant message. A successful continuation is the tail of + # the earlier capped response, so stitch the pieces back into + # the answer presented to the caller when this text actually + # terminates the loop. A nudge policy may reject the plain-text + # reply and continue toward a tool call, in which case it is not + # the run's final answer. + if stops_for_no_tool: + metadata["_truncation_final_content"] = "".join( + [*truncated_text_parts, ctx.ai_text], + ) + truncated_text_parts.clear() + if stops_for_no_tool: + stop_reason = "no_tool" + break + messages.append(user_msg(_build_no_tool_nudge(cfg.loop_policy))) + continue + + no_tool_retries = 0 + truncation_continuations = 0 + truncated_text_parts.clear() + + if not skip_tool_execution: + stop_reason, tool_calls_executed = await _execute_tool_calls( + cfg, obs, tool_map, messages, metadata, turn, total_tool_calls, + ctx, parsed_calls, runtime.tool_execution, + runtime.body_has_spill_reference, + ) + total_tool_calls += tool_calls_executed + if stop_reason: + break + + stop_reason = _handle_context_overflow( + cfg, messages, turn, last_input_tokens, last_output_tokens + ) + if stop_reason: + break + + stop_reason, continue_to_next_turn = await _handle_turn_end( + cfg, obs, messages, metadata, turn, ctx, on_turn_complete, pause_check + ) + if stop_reason: + break + if continue_to_next_turn: + turn -= 1 + continue + + else: + if attempts >= max_attempts and turn < cfg.max_turns: + stop_reason = "max_attempts" + logger.warning( + "loop exhausted rollback budget at turn=%d (attempts=%d/%d)", + turn, attempts, max_attempts, + ) + else: + stop_reason = "max_turns" + + return await _finalize_loop( + obs, messages, metadata, turn, total_tool_calls, stop_reason + ) + + +async def _prepare_llm_request( + cfg: LoopConfig, obs: list, llm_with_session: Any, llm_with_tools: Any, + messages: list[Message], metadata: dict[str, Any], turn: int +) -> tuple[Any, list[Message], bool, str]: + temp_override = metadata.pop("_llm_temp_override", None) + strip_tools = metadata.pop("_llm_strip_tools", False) + llm_base = llm_with_session if strip_tools else llm_with_tools + llm_for_turn = ( + bind_temperature(llm_base, temp_override) + if temp_override is not None + else llm_base + ) + + before_llm_ctx = TurnContext( + turn=turn, max_turns=cfg.max_turns, task_id=cfg.task_id, role_id=cfg.role_id, + ai_text="", thinking="", tool_calls=[], messages=messages, usage=None, metadata=metadata, + ) + before_llm_interventions = await notify_observers(obs, "on_before_llm", before_llm_ctx) + merged_before_llm = merge_interventions(before_llm_interventions) + + if merged_before_llm.inject_messages: + for msg_text in merged_before_llm.inject_messages: + messages.append(user_msg(msg_text)) + + messages_for_call = messages + if cfg.system_addendum_per_call and turn > cfg.system_addendum_min_turn: + messages_for_call = [*messages, system_msg(cfg.system_addendum_per_call)] + + # Publish the estimate of THIS request, after observer injections and the + # addendum. An observer comparing its own estimate against the provider's + # reported ``prompt_tokens`` needs both sides measured on the same list; + # sampling at turn end instead understates the ratio by whatever the + # completion and tool results added. + metadata[INPUT_ESTIMATE_KEY] = estimate_tokens(messages_for_call) + + return ( + llm_for_turn, + messages_for_call, + bool(strip_tools), + merged_before_llm.stop_reason or "", + ) + + +async def _call_llm_with_callbacks( + cfg: LoopConfig, obs: list, profile: Any, llm_for_turn: Any, messages_for_call: list[Message], + metadata: dict[str, Any], turn: int, stream_llm_tokens: bool, + runtime_hooks: AgentLoopHooks, +) -> tuple[Any, str, float | None, float, float, str, str, int]: + llm_call_started = time.perf_counter() + first_delta_at: float | None = None + call_id = f"llm_{uuid.uuid4().hex}" + current_attempt_index = 1 + current_attempt_id = f"{call_id}_attempt_01" + + metadata["_llm_call_id"] = call_id + metadata["_llm_attempt_id"] = current_attempt_id + metadata["_llm_attempt_index"] = current_attempt_index + metadata["_llm_attempt_outcome"] = "" + metadata["_llm_attempt_count"] = 0 + + async def _on_attempt(event: dict[str, Any]) -> None: + nonlocal current_attempt_id, current_attempt_index + current_attempt_index = int(event.get("attempt_index", 1) or 1) + current_attempt_id = f"{call_id}_attempt_{current_attempt_index:02d}" + phase = str(event.get("phase", "") or "") + outcome = str(event.get("outcome", "") or "") + if phase == "finished": + metadata["_llm_call_id"] = call_id + metadata["_llm_attempt_id"] = current_attempt_id + metadata["_llm_attempt_index"] = current_attempt_index + metadata["_llm_attempt_outcome"] = outcome + metadata["_llm_attempt_count"] = max( + int(metadata.get("_llm_attempt_count", 0) or 0), current_attempt_index + ) + attempt_usage = event.get("usage") + if isinstance(attempt_usage, dict): + attempt_usage = dict(attempt_usage) + if not attempt_usage.get("provider"): + attempt_usage["provider"] = str(getattr(profile, "provider", "") or "") + if not attempt_usage.get("model"): + attempt_usage["model"] = str(getattr(profile, "model_id", "") or "") + attempt_ctx = LLMAttemptContext( + turn=turn, max_turns=cfg.max_turns, task_id=cfg.task_id, role_id=cfg.role_id, + call_id=call_id, attempt_id=current_attempt_id, attempt_index=current_attempt_index, + phase=phase, outcome=outcome, reason=str(event.get("reason", "") or ""), + recovery_action=str(event.get("recovery_action", "") or ""), + duration_ms=int(event.get("duration_ms", 0) or 0), ttft_ms=event.get("ttft_ms"), + usage=attempt_usage, finish_reason=str(event.get("finish_reason", "") or ""), + visible_chars=int(event.get("visible_chars", 0) or 0), + reasoning_chars=int(event.get("reasoning_chars", 0) or 0), + tool_calls_count=int(event.get("tool_calls_count", 0) or 0), + max_tokens=event.get("max_tokens"), error_type=str(event.get("error_type", "") or ""), + metadata=metadata, + ) + await notify_observers(obs, "on_llm_attempt", attempt_ctx) + + async def _on_delta( + delta: str, accumulated: str, delta_index: int, thinking_delta: str = "", + *, tool_call_args_chunks: list[dict] | None = None, + ) -> None: + nonlocal first_delta_at + if first_delta_at is None and (delta or thinking_delta or tool_call_args_chunks): + first_delta_at = time.perf_counter() + ctx = LLMDeltaContext( + turn=turn, max_turns=cfg.max_turns, task_id=cfg.task_id, role_id=cfg.role_id, + delta=delta, accumulated_text=accumulated, delta_index=delta_index, + metadata=metadata, thinking_delta=thinking_delta, + tool_call_args_chunks=tool_call_args_chunks or [], + attempt_id=current_attempt_id, attempt_index=current_attempt_index, call_id=call_id, + ) + await notify_observers(obs, "on_llm_delta", ctx) + + try: + response = await call_llm( + llm_for_turn, messages_for_call, cfg.llm_timeout, cfg.max_llm_retries, turn, + on_delta=_on_delta if stream_llm_tokens else None, + retry_wait_fixed=cfg.retry_wait_fixed, + runaway_state=metadata.setdefault(RUNAWAY_STATE_KEY, {}), + first_chunk_s=cfg.first_chunk_timeout, + on_attempt=_on_attempt, + reasoning_only_timeout_s=cfg.reasoning_only_timeout_s, + reasoning_only_max_tokens=cfg.reasoning_only_max_tokens, + logical_call_timeout_s=cfg.logical_call_timeout_s, + max_completion_tokens_hint=( + cfg.max_completion_tokens if (cfg.reasoning_only_timeout_s or cfg.reasoning_only_max_tokens) else None + ), + context_token_limit_hint=cfg.context_token_limit, + wall_deadline_remaining=runtime_hooks.wall_deadline_remaining, + chain_fallback_active=runtime_hooks.chain_fallback_active, + ) + except LLMCallExhausted as exhausted: + if exhausted.reason == "wall_deadline": + logger.warning( + "agent_loop: wall deadline reached mid-turn %d; ending with wall_deadline for salvage: %s", + turn, exhausted.last_exc, + ) + return ( + None, "wall_deadline", first_delta_at, llm_call_started, + time.perf_counter(), call_id, current_attempt_id, + current_attempt_index, + ) + if turn == 1 and runtime_hooks.chain_fallback_active(): + logger.error( + "agent_loop: surfacing call_llm failure on turn 1 (reason=%s) so chain wrapper can advance: %s", + exhausted.reason, exhausted.last_exc, + ) + raise exhausted.last_exc from exhausted + logger.error( + "agent_loop: call_llm exhausted after turn=%d (reason=%s); ending with llm_error to preserve partial content: %s", + turn, exhausted.reason, exhausted.last_exc, + ) + metadata["llm_error"] = str(exhausted.last_exc) + metadata["llm_error_reason"] = exhausted.reason + return ( + None, "llm_error", first_delta_at, llm_call_started, + time.perf_counter(), call_id, current_attempt_id, + current_attempt_index, + ) + + llm_call_finished = time.perf_counter() + if response is None: + metadata.setdefault("llm_error", "LLM returned no response") + return ( + None, "llm_error", first_delta_at, llm_call_started, + llm_call_finished, call_id, current_attempt_id, + current_attempt_index, + ) + + return ( + response, "", first_delta_at, llm_call_started, llm_call_finished, + call_id, current_attempt_id, current_attempt_index, + ) + + +def _answer_dropped_tool_calls( + messages: list[Message], history_msg: Message, + parsed_calls: list[dict], tool_names: set[str], +) -> None: + """Give every ``tool_call_id`` in the assistant turn a tool response. + + ``history_msg`` is built from the raw response, so it carries every native + call the model emitted — including ones parsing then drops (an unknown + companion name alongside a real action, or an over-cap call). The provider + requires one ``tool`` message per id: an orphan is a hard HTTP 400 on Azure + and others, which would turn a recoverable mistake into a dead run. + + The message doubles as the correction the model needs, so a dropped call is + reported rather than silently vanishing and being reissued every turn. + """ + recorded = history_msg.get("tool_calls") or [] + if not recorded: + return + answered = { + message.get("tool_call_id") + for message in messages + if is_tool_msg(message) + } + answered.update(call.get("id") for call in parsed_calls) + for call in recorded: + call_id = call.get("id") + if not call_id or call_id in answered: + continue + name = str((call.get("function") or {}).get("name") or call.get("name") or "") + if name and name not in tool_names: + detail = ( + f"unknown tool '{name}' is not available. It was not run; the " + "other tool calls in this turn were. Use only the listed tools." + ) + else: + detail = "this tool call was not dispatched; re-issue it if still needed." + messages.append(tool_msg(f"[tool call not executed] {detail}", call_id)) + answered.add(call_id) + + +def _pop_last_assistant_turn(messages: list[Message]) -> None: + """Remove the assistant message a rollback observer rejected, in full. + + ``pop_last_message`` fires while the turn's tool calls are still + unexecuted, so the tail is normally just the assistant message. It is + NOT always: :func:`_answer_dropped_tool_calls` and the + ``max_tool_calls_per_turn`` cap append ``tool`` messages *after* it to + answer calls that will never run. Popping one message there would + strip an answer and leave the assistant message holding an unanswered + ``tool_call_id`` — which providers reject with a 400 on the next + request. Drop the trailing tool answers first, then the assistant + message itself. + + Only this turn's tail is in scope: real tool results are appended + later, in ``_execute_tool_calls``. + + If no assistant message sits beneath the trailing tool messages, the + tail is not this turn's shape at all — leave history untouched rather + than popping an unrelated prefix. + """ + idx = len(messages) - 1 + while idx >= 0 and is_tool_msg(messages[idx]): + idx -= 1 + if idx < 0 or not is_assistant_msg(messages[idx]): + logger.warning( + "_pop_last_assistant_turn: no assistant message beneath the " + "trailing tool messages; leaving history unchanged" + ) + return + del messages[idx:] + + +async def _process_llm_response( + cfg: LoopConfig, obs: list, tc_parser: Any, profile: Any, policy: Any, thinking_parser: Any, normalizer: Any, + tool_names: set[str], messages: list[Message], metadata: dict[str, Any], turn: int, + response: Any, strip_tools: bool, last_input_tokens: int, last_output_tokens: int, first_delta_at: float | None, + llm_call_started: float, llm_call_finished: float, call_id: str, current_attempt_id: str, current_attempt_index: int +) -> tuple[list[dict], TurnContext, str, bool, bool, int, int]: + metadata["llm_duration_ms"] = int((llm_call_finished - llm_call_started) * 1000) + metadata["llm_ttft_ms"] = int(((first_delta_at or llm_call_finished) - llm_call_started) * 1000) + + tr = thinking_parser.extract(response, profile) + history_msg = normalizer.to_history(response, tr, policy, profile.thinking_format) + messages.append(history_msg) + + if tr.thinking and profile.thinking_format == "tag": + with contextlib.suppress(Exception): + response.content = tr.visible_content + + parsed_calls = tc_parser.parse(response, tool_names) + if not parsed_calls and tr.thinking and hasattr(tc_parser, "parse_text"): + parsed_calls = tc_parser.parse_text(tr.thinking, tool_names) + if parsed_calls: + logger.warning("turn=%d recovered %d tool_call(s) leaked into ", turn, len(parsed_calls)) + + # Tool schemas are stripped on the landing turn, but text-mode models can + # still emit parseable tool markup. Permit only the workflow's bounded + # landing allowlist and balance rejected native calls with synthetic tool + # results so strict providers can safely reuse the history. + blocked_landing_calls: list[dict] = [] + if strip_tools and parsed_calls: + configured_landing_names = cfg.loop_policy.landing_tool_names + if configured_landing_names is None: + configured_landing_names = cfg.loop_policy.terminal_tool_names + landing_names = set(configured_landing_names) & tool_names + allowed_calls = [ + tc for tc in parsed_calls if tc.get("name") in landing_names + ] + blocked_landing_calls = [ + tc for tc in parsed_calls if tc.get("name") not in landing_names + ] + if blocked_landing_calls: + blocked_names = [ + str(tc.get("name") or "unknown") + for tc in blocked_landing_calls + ] + recorded = metadata.setdefault("blocked_final_turn_tool_calls", []) + if isinstance(recorded, list): + recorded.extend(blocked_names) + logger.warning( + "turn=%d blocked %d non-terminal tool call(s) on final turn: %s", + turn, len(blocked_landing_calls), blocked_names, + ) + native_ids = { + str(tc.get("id") or "") + for tc in (getattr(response, "tool_calls", None) or []) + if isinstance(tc, dict) and tc.get("id") + } + for blocked in blocked_landing_calls: + call_id = str(blocked.get("id") or "") + if call_id and call_id in native_ids: + messages.append(tool_msg( + "[tool call blocked] final turn accepts only " + "workflow-approved landing tools; report current progress.", + call_id, + )) + parsed_calls = allowed_calls + + cap = cfg.max_tool_calls_per_turn + if cap and cap > 0 and len(parsed_calls) > cap: + dropped_calls = parsed_calls[cap:] + parsed_calls = parsed_calls[:cap] + for dtc in dropped_calls: + dtc_id = dtc.get("id") + if not dtc_id: + continue + messages.append(tool_msg( + f"[tool call skipped] exceeded the per-turn tool-call cap of {cap}; re-issue it in a later turn if still needed.", + dtc_id, + )) + + _answer_dropped_tool_calls(messages, history_msg, parsed_calls, tool_names) + + usage: UsageMetadata | None = extract_usage(response) + if usage is None: + model_id = str(getattr(profile, "model_id", "") or "") + if model_id and model_id != "default": + usage = { + "provider": str(getattr(profile, "provider", "") or ""), "model": model_id, + "prompt_tokens": 0, "completion_tokens": 0, + "cache_read_tokens": 0, "cache_write_tokens": 0, + "cached_tokens": 0, "cache_creation_tokens": 0, + "reasoning_tokens": 0, "estimated": True, + } + if usage: + last_input_tokens = int(usage.get("prompt_tokens", 0) or 0) + last_output_tokens = int(usage.get("completion_tokens", 0) or 0) + + leaked_reasoning = extract_leaked_reasoning(response) + rmd = getattr(response, "response_metadata", None) or {} + metadata.pop("llm_fallback_used", None) + metadata.pop("llm_model_actually_used", None) + if "fallback_used" in rmd: + metadata["llm_fallback_used"] = rmd["fallback_used"] + if "model_actually_used" in rmd: + metadata["llm_model_actually_used"] = rmd["model_actually_used"] + metadata["finish_reason"] = getattr(response, "finish_reason", "") or "" + + post_content = getattr(response, "content", None) + ai_text = post_content if isinstance(post_content, str) else tr.visible_content + ctx = TurnContext( + turn=turn, max_turns=cfg.max_turns, task_id=cfg.task_id, role_id=cfg.role_id, + ai_text=ai_text, thinking=tr.thinking, tool_calls=parsed_calls, messages=messages, + usage=usage, metadata=metadata, leaked_reasoning=leaked_reasoning, + thinking_blocks=tr.raw_content_blocks or [], + blocked_tool_calls=blocked_landing_calls, + tool_schemas_stripped=bool(strip_tools), + ) + + llm_interventions = await notify_observers(obs, "on_llm_response", ctx) + merged_llm = merge_interventions(llm_interventions) + + stop_reason = merged_llm.stop_reason or "" + + if merged_llm.pop_last_message and messages: + _pop_last_assistant_turn(messages) + if merged_llm.continue_to_next_turn: + if merged_llm.inject_messages: + for msg_text in merged_llm.inject_messages: + messages.append(user_msg(msg_text)) + return ( + parsed_calls, ctx, stop_reason, True, False, + last_input_tokens, last_output_tokens, + ) + + if merged_llm.inject_messages: + for msg_text in merged_llm.inject_messages: + messages.append(user_msg(msg_text)) + + if not stop_reason and blocked_landing_calls and not parsed_calls: + return ( + parsed_calls, ctx, "max_turns", False, False, + last_input_tokens, last_output_tokens, + ) + + return ( + parsed_calls, ctx, stop_reason, False, + merged_llm.skip_tool_execution, last_input_tokens, last_output_tokens, + ) + + +async def _execute_tool_calls( + cfg: LoopConfig, obs: list, tool_map: dict[str, ToolLike], messages: list[Message], metadata: dict[str, Any], + turn: int, total_tool_calls: int, ctx: TurnContext, parsed_calls: list[dict], + execution_hooks: ToolExecutionHooks, + body_has_spill_reference: Callable[[str], bool], +) -> tuple[str, int]: + executable: list[tuple[int, dict]] = [] + synthetic: list[tuple[int, ToolResult]] = [] + for idx, tc in enumerate(parsed_calls): + tcv = await notify_tool_call(obs, ctx, tc) + if tcv.metadata_updates: + metadata.update(tcv.metadata_updates) + if tcv.rewrite_args is not None: + tc = {**tc, "args": tcv.rewrite_args} + if tcv.skip_with_result is not None: + synthetic.append((idx, ToolResult( + name=str(tc.get("name", "") or ""), + args=tc.get("args", {}) if isinstance(tc.get("args"), dict) else {}, + result=tcv.skip_with_result, duration_ms=0, + tool_call_id=str(tc.get("id") or f"call_{turn}_{idx}"), is_error=False, + ))) + else: + executable.append((idx, tc)) + + executed_results: list[ToolResult] = [] + if executable: + has_tool_interrupt_waiter = any( + callable(getattr(observer, "wait_for_tool_interrupt", None)) for observer in obs + ) + executed_results = await execute_tools( + [tc for _, tc in executable], tool_map, + timeout=cfg.tool_timeout, + turn=turn, + count_offset=total_tool_calls, + interrupt_waiter=( + (lambda tool_call: _wait_for_tool_interrupt(obs, ctx, tool_call)) + if has_tool_interrupt_waiter else None + ), + hooks=execution_hooks, + ) + + # Slots are pre-allocated so results reappear in the model's original call + # order regardless of completion order. + ordered: list[ToolResult | None] = [None] * len(parsed_calls) + for (idx, _), tr in zip(executable, executed_results, strict=False): + ordered[idx] = tr + for idx, tr in synthetic: + ordered[idx] = tr + + # ``executable`` and ``synthetic`` partition ``parsed_calls``, so every slot + # is normally filled. The zip above still truncates if ``execute_tools`` + # returns fewer results than calls it was handed, and the placeholder is a + # real None — previously typed away with a blanket ignore, which left the + # attribute access below to raise. Drop unfilled slots and say so instead. + results: list[ToolResult] = [tr for tr in ordered if tr is not None] + if len(results) != len(ordered): + logger.warning( + "Tool execution returned %d result(s) for %d call(s); " + "dropping the unfilled slot(s)", len(results), len(ordered), + ) + + processor = cfg.tool_result_post_processor or DefaultToolResultPostProcessor(cfg.tool_result_max_chars) + can_recover = "recover_result" in tool_map + for tr_result in results: + tr_result = await notify_tool_result(obs, ctx, tr_result) + # ``notify_tool_result`` ran FIRST, so the trajectory already holds + # ``tr_result.result`` in full. The post-processor cuts only the string + # that becomes the message — at a far smaller cap than the 150K upstream + # (15_000 for sub-agents) — and persists nothing, so without a pointer + # here the difference is simply lost to the model. Minted at this site + # only: the two earlier cuts happen before the ``ToolResult`` exists, so + # for those the trajectory holds the same preview the model already has. + body = processor.process(tr_result) + messages.append(tool_msg( + _with_recovery_handle( + body, + tr_result, + ctx.turn, + enabled=can_recover, + body_has_spill_reference=body_has_spill_reference, + ), + tr_result.tool_call_id, + )) + + if any(result.interrupted for result in results): + wait_interventions = await notify_observers(obs, "on_tool_wait_interrupted", ctx) + merged_wait = merge_interventions(wait_interventions) + if merged_wait.inject_messages: + for msg_text in merged_wait.inject_messages: + messages.append(user_msg(msg_text)) + if merged_wait.stop_reason: + return merged_wait.stop_reason, len(parsed_calls) + + return "", len(parsed_calls) + + +def _handle_context_overflow( + cfg: LoopConfig, messages: list[Message], turn: int, last_input_tokens: int, last_output_tokens: int +) -> str: + if not cfg.context_overflow_guard or not messages: + return "" + + trailing_tool_idx = len(messages) + while trailing_tool_idx > 0 and is_tool_msg(messages[trailing_tool_idx - 1]): + trailing_tool_idx -= 1 + buffer_factor = 1.5 + trailing_tool_tokens = 0 + for m in messages[trailing_tool_idx:]: + trailing_tool_tokens += int(estimate_message_tokens(m) * buffer_factor) + summary_tokens = int(estimate_text_tokens(cfg.summary_prompt) * buffer_factor) + estimated_total = ( + last_input_tokens + last_output_tokens + trailing_tool_tokens + + summary_tokens + cfg.max_completion_tokens + 1000 + ) + if estimated_total >= cfg.max_context_length: + logger.warning( + "Context overflow guard tripped at turn=%d " + "(estimated=%d / limit=%d, last_input=%d, last_output=%d, " + "trailing_tool=%d, summary=%d). " + "Popping trailing ToolMessage(s) + last AIMessage and " + "exiting loop with stopped_by='context_limit_reached'.", + turn, estimated_total, cfg.max_context_length, + last_input_tokens, last_output_tokens, trailing_tool_tokens, summary_tokens, + ) + while messages and is_tool_msg(messages[-1]): + messages.pop() + if messages and is_assistant_msg(messages[-1]): + messages.pop() + return "context_limit_reached" + return "" + + +def _with_recovery_handle( + body: str, + result: ToolResult, + turn: int, + *, + enabled: bool, + body_has_spill_reference: Callable[[str], bool] = _body_has_no_spill, +) -> str: + """Name the handle that fetches back what the post-processor cut. + + ``len(body) < len(result.result)`` is the exact condition under which + recovery helps — it says the trajectory holds content the model cannot see — + rather than an approximation of it. A processor that shortens a result some + other way (URL stubbing, for instance) still satisfies it, and the handle is + still correct there. + + The char count can drift by one case: ``notify_tool_result`` is last-mutation- + wins, so an observer sitting AFTER the trajectory one that rewrites the result + leaves the footer counting against a body the trajectory does not hold. The + handle still resolves, and ``recover_result`` reports the real totals in its + own header, so the agent sees the truth at the point it matters. + + Silent when the body already names a spill file. That is not a nicety: on a + live agent-team run EVERY result site 3 shortened was a ``bash`` result + carrying a gate-① spill pointer, the spill file held the full pre-cut output + (42,770 chars behind an 8,000-char body), and the agent recovered by running + ``cat`` on that path — 43 footers, zero tool calls. The footer only earns its + place where nothing else covers the cut. + + Gated on ``recover_result`` being bound for THIS agent, not on a config flag: + profiles carry their own tool lists (the stateful_react benchmark profile + binds no reader at all), and a footer naming a tool the agent cannot call is + worse than no footer — ``_spill_footer`` already carries a comment about that + exact failure. + + Worded as prose naming a tool and its arguments, never as + ``recover_result(turn=..., call_id="...")``. The callable form read as source + to the model, which reproduced it inside a ```bash block instead of emitting a + tool call; ``LeakedToolCallRetryObserver`` fired twice on that run. + + The note costs ~120 chars beyond the cap. The processors' own + ``[... truncated N chars past M-char cap]`` marker is already appended after + the cut, so a bounded overshoot is the existing behaviour rather than a + regression this introduces; on a 15_000-char cap this roughly triples a + 45-char overshoot and stays under 1%. + """ + if not enabled or not isinstance(body, str): + return body + original = result.result if isinstance(result.result, str) else "" + if len(body) >= len(original): + return body + call_id = result.tool_call_id or "" + if not call_id: + # The handle is (turn, call_id); without an id it resolves to nothing, + # and recover_result refuses empty ids rather than guessing. + return body + if body_has_spill_reference(body): + # Redundant, and measurably harmful as an alternative. Gate ① already + # spilled the FULL pre-cut output and left a path in this body, so the + # spill file is a strict superset of anything site 3 removed — verified on + # a live run: 42,770 chars behind an 8,000-char body, with the elided + # middle present in the file. Offering a second route to a subset of the + # same bytes cost real turns: the agent quoted this footer, wrote "Let me + # call recover_result", and then ran ``cat`` on the spill path anyway. + return body + return body + ( + f"\n\n[{len(original) - len(body):,} more chars were cut here. Use the " + f"recover_result tool (a tool call, not a shell command) with " + f"turn {turn} and call id {call_id}.]" + ) + + +async def _handle_turn_end( + cfg: LoopConfig, obs: list, messages: list[Message], metadata: dict[str, Any], turn: int, + ctx: TurnContext, on_turn_complete: TurnCompleteHook | None, pause_check: PauseCheckHook | None +) -> tuple[str, bool]: + turn_interventions = await notify_observers(obs, "on_turn_end", ctx) + merged_turn = merge_interventions(turn_interventions) + + if merged_turn.stop_reason: + if merged_turn.inject_messages: + for msg_text in merged_turn.inject_messages: + messages.append(user_msg(msg_text)) + return merged_turn.stop_reason, False + + # End-of-turn rollback runs after tool replies entered history. Remove the + # complete assistant turn, skip compaction/persistence for the discarded + # attempt, and let the caller retry without consuming a logical turn. + if merged_turn.pop_last_message: + _pop_last_assistant_turn(messages) + if merged_turn.inject_messages: + for msg_text in merged_turn.inject_messages: + messages.append(user_msg(msg_text)) + if merged_turn.continue_to_next_turn: + return "", True + + est_tokens = estimate_tokens(messages) + compaction_policy = cfg.compaction_policy or DefaultCompactionPolicy( + cfg.compact_after_turns, cfg.context_token_limit, + ) + forced_compaction = bool(metadata.pop(FORCE_COMPACTION_KEY, False)) + if forced_compaction or compaction_policy.should_compact(turn, messages, est_tokens): + compactor = cfg.compactor or DefaultMessageCompactor() + result = compactor.compact(messages, cfg.keep_recent) + messages[:] = await result if inspect.isawaitable(result) else result + metadata[COMPACTION_SEQ_KEY] = int( + metadata.get(COMPACTION_SEQ_KEY, 0) or 0, + ) + 1 + # Compaction is the one history rewrite observers never saw: it edits + # ``messages`` in place, so a trajectory built from the post-compaction + # history shows the rollup with the replaced turns already gone. Report + # it where the turn and sequence are known — the compactor knows neither. + # Compactors that expose no event (the default one) simply aren't + # reported, and ``notify_observers`` skips observers without the hook. + compaction_event = getattr(compactor, "last_event", None) + if isinstance(compaction_event, CompactionEvent): + compaction_event.turn = turn + compaction_event.seq = metadata[COMPACTION_SEQ_KEY] + await notify_observers(obs, "on_compaction", compaction_event) + + if on_turn_complete is not None: + try: + await on_turn_complete(turn, messages, metadata) + except Exception as exc: + logger.warning("on_turn_complete hook failed at turn %d: %s", turn, exc) + + if pause_check is not None: + try: + if await pause_check(): + return "paused", False + except Exception as exc: + logger.warning("pause_check failed at turn %d: %s", turn, exc) + + return "", False + + +async def _finalize_loop( + obs: list, messages: list[Message], metadata: dict[str, Any], + turn: int, total_tool_calls: int, stop_reason: str +) -> AgentLoopResult: + latched_answer = metadata.get("final_answer") if isinstance(metadata, dict) else None + continued_answer = metadata.pop("_truncation_final_content", "") + if isinstance(latched_answer, str) and latched_answer.strip(): + final_content = latched_answer + elif isinstance(continued_answer, str) and continued_answer.strip(): + final_content = continued_answer + else: + final_content = extract_final_content(messages) + result = AgentLoopResult( + messages=messages, + final_content=final_content, + turns_used=turn, + tool_calls_count=total_tool_calls, + stopped_by=stop_reason, + metadata=metadata, + ) + + await notify_observers(obs, "on_loop_end", result) + return result + +def _resolve_phase_id(cfg: LoopConfig) -> str: + """Resolve the execution-scope phase ID for this loop run. + + Priority: + 1. Explicit LoopPolicy phase_id + 2. role_id (generic fallback) + 3. empty string + """ + return cfg.loop_policy.phase_id or cfg.role_id or "" + + +def _build_no_tool_nudge(policy: LoopPolicy) -> str: + """Render the no-tool recovery message from injected loop policy.""" + if policy.no_tool_nudge_message.strip(): + return policy.no_tool_nudge_message.strip() + + if policy.terminal_tool_names: + terminals = ", ".join(f"`{name}`" for name in policy.terminal_tool_names) + if len(policy.terminal_tool_names) == 1: + finish_hint = f"If you are done, call {terminals}. " + else: + finish_hint = ( + "If you are done, call one of the terminal tools: " + f"{terminals}. " + ) + else: + finish_hint = "If your workflow defines a terminal action, use it now. " + + return ( + "This loop requires a structured tool call to continue. " + + finish_hint + + "Otherwise, call an appropriate tool instead of replying in plain text." + ) diff --git a/agent_core/runtime/loop/model_profile.py b/agent_core/runtime/loop/model_profile.py new file mode 100644 index 0000000..4665318 --- /dev/null +++ b/agent_core/runtime/loop/model_profile.py @@ -0,0 +1,635 @@ +# pyright: reportMissingModuleSource=false, reportUnknownArgumentType=false, reportUnknownMemberType=false, reportUnknownVariableType=false +"""Model-agnostic abstractions for thinking extraction and history management.""" + +from __future__ import annotations + +import json +import logging +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path +from typing import Any, Literal, Protocol, TypeGuard, get_args + +from agent_core.llm import LLMResponse +from agent_core.messages import ( + Message, + ToolCall, + assistant_msg, + assistant_msg_with_reasoning, +) + +logger = logging.getLogger(__name__) + +# ── Regex for tag extraction ───────────────────────────────────────── + +_THINK_RE = re.compile(r"([\s\S]*?)\s*", re.DOTALL) + + +# ── Thinking-format inference ──────────────────────────────────────────────── +# +# The pattern table is product data configured with +# :func:`configure_model_registry`. It changes every time a host onboards a new +# endpoint, so keeping it out of core Python lets ops update one YAML file +# without changing the shared engine. The loader is cached for the process +# lifetime; restart or explicitly reset the cache to pick up edits. + +ThinkingFormat = Literal["tag", "content_block", "reasoning_content", "none"] + +_VALID_FORMATS: frozenset[str] = frozenset(get_args(ThinkingFormat)) + +# Wire protocol the client speaks. Named alias so config readers can declare it +# instead of returning a bare str that every ModelProfile call site then rejects. +WireProtocol = Literal["chat_completions", "anthropic", "responses", "bedrock"] +_VALID_PROTOCOLS: frozenset[str] = frozenset(get_args(WireProtocol)) + + +def is_thinking_format(value: object) -> TypeGuard[ThinkingFormat]: + """Narrow an unvalidated value (YAML, model registry) to a ThinkingFormat. + + The isinstance check comes first because these frozensets are consulted with + whatever the YAML parser produced: a bare ``value in _VALID_FORMATS`` raises + ``TypeError: unhashable type`` for a list or dict, which would abort profile + loading instead of warning and falling back. + """ + return isinstance(value, str) and value in _VALID_FORMATS + + +def is_wire_protocol(value: object) -> TypeGuard[WireProtocol]: + """Narrow an unvalidated value (YAML ``llm.protocol``) to a WireProtocol. + + See :func:`is_thinking_format` for why the isinstance check is required. + """ + return isinstance(value, str) and value in _VALID_PROTOCOLS + +_registry_path: Path | None = None + + +def configure_model_registry(path: str | Path | None) -> None: + """Set the host-owned model registry used for thinking-format inference. + + AgentCore deliberately ships no product endpoint registry. Hosts configure + their packaged YAML path once from a compatibility facade; passing None + disables registry inference and falls back to the caller's default. + """ + + global _registry_path + _registry_path = Path(path) if path is not None else None + reset_thinking_format_cache() + + +@lru_cache(maxsize=1) +def _load_thinking_format_patterns() -> ( + tuple[tuple[re.Pattern[str], ThinkingFormat], ...] +): + """Read ``thinking_formats`` from ``model_registry.yaml``. + + Returns an empty tuple when the file is missing or malformed — + callers then get only the user-supplied default. Bad individual + entries are skipped with a warning rather than failing the whole + load (so a typo in one row never breaks production inference). + """ + if _registry_path is None or not _registry_path.is_file(): + logger.debug( + "model_registry.yaml not found at %s — inference returns default", + _registry_path, + ) + return () + try: + import yaml + except ImportError: # pragma: no cover — PyYAML is a project dep + return () + try: + raw = yaml.safe_load(_registry_path.read_text(encoding="utf-8")) or {} + except Exception as exc: + logger.warning("Failed to parse model_registry.yaml: %s", exc) + return () + + table = raw.get("thinking_formats") or [] + if not isinstance(table, list): + logger.warning( + "model_registry.yaml: 'thinking_formats' must be a list (got %s)", + type(table).__name__, + ) + return () + + out: list[tuple[re.Pattern[str], ThinkingFormat]] = [] + for entry in table: + if not isinstance(entry, dict): + continue + pattern_str = entry.get("pattern") + fmt = entry.get("format") + if not pattern_str or fmt not in _VALID_FORMATS: + logger.warning( + "model_registry.yaml: skipped invalid entry %r", entry, + ) + continue + try: + compiled = re.compile(pattern_str, re.IGNORECASE) + except re.error as exc: + logger.warning( + "model_registry.yaml: bad regex %r — %s", + pattern_str, exc, + ) + continue + out.append((compiled, fmt)) + return tuple(out) + + +def reset_thinking_format_cache() -> None: + """Clear the cached pattern table — used by tests that rewrite the YAML.""" + _load_thinking_format_patterns.cache_clear() + + +def infer_thinking_format( + model_id: str | None, + *, + default: ThinkingFormat = "tag", +) -> ThinkingFormat: + """Pick a thinking format for a model id when none is set explicitly. + + The pattern table is loaded from ``model_registry.yaml`` (see that + file's header for editing rules + endpoint-dependency caveat). + Matching is case-insensitive on the bare model id (no provider + prefix needed). Returns ``default`` for empty or unknown ids. + """ + if not model_id: + return default + needle = model_id.lower() + for pattern, fmt in _load_thinking_format_patterns(): + if pattern.search(needle): + return fmt + return default + + +# ── Data structures ─────────────────────────────────────────────────────────── + + +@dataclass +class ModelProfile: + """Static facts about a model's capabilities. + + These are properties of the *model*, not the agent using it. + """ + + model_id: str + provider: str + context_window: int = 128_000 + supports_native_fc: bool = True + supports_streaming: bool = True + supports_images: bool = False + thinking_format: ThinkingFormat = "none" + tool_call_format: Literal["native_fc", "text", "both"] = "native_fc" + # Wire protocol the client speaks: ``chat_completions`` (default), + # ``anthropic`` (native Messages API + extended thinking), or ``responses`` + # (OpenAI Responses API + encrypted reasoning). Native Anthropic + Responses + # both return content as a typed block list → ``thinking_format`` is + # ``content_block`` so the parser keeps the verbatim blocks (signatures / + # encrypted_content) for faithful multi-turn replay + trajectory. + protocol: WireProtocol = "chat_completions" + + +@dataclass +class HistoryPolicy: + """Agent design choices about how thinking output is used. + + These are properties of the *agent role*, not the model. + """ + + thinking_in_history: bool = False + thinking_in_memory: bool = True + thinking_in_sse: bool = True + tool_result_max_chars: int = 15_000 + compress_thinking_after_turns: int = 0 + max_images_in_history: int = 5 + # PER-ASSISTANT-TURN cap, not a budget over the whole history: ``None`` + # means one turn's reasoning is kept in full, a positive value keeps only + # the most recent reasoning of that turn that fits this many tokens. Total + # reasoning in history still grows with turn count — bounding *that* is + # compaction's job (see ``compress_thinking_after_turns`` / tiered_compact). + # Disabled history always wins over this cap. Kept last for positional + # compatibility with the original HistoryPolicy constructor. + thinking_history_max_tokens: int | None = None + + +@dataclass +class ThinkingResult: + """Parsed output from a single LLM response.""" + + thinking: str + visible_content: str + tool_calls: list[dict[str, Any]] = field(default_factory=list) + # Native Anthropic thinking / OpenAI Responses reasoning: the VERBATIM + # content-block list exactly as returned — [{type:"thinking", thinking, + # signature}, {type:"redacted_thinking", data}, {type:"reasoning", + # encrypted_content, summary}, {type:"text", ...}]. Preserved so multi-turn + # replay re-sends the blocks (incl. encrypted signatures / encrypted_content) + # UNMODIFIED — the provider validates them server-side — and the trajectory + # can persist them. ``None`` for every other thinking_format → all downstream + # signature/encrypted-reasoning logic is a no-op there. + raw_content_blocks: list[Any] | None = None + + +_CAP_ERROR = ( + "thinking_history_max_tokens must be a non-negative integer or null" +) +_TRUE_STRINGS = frozenset({"true", "yes", "on", "1"}) +_FALSE_STRINGS = frozenset({"false", "no", "off", "0"}) + + +def _coerce_optional_bool(value: Any, key: str) -> bool | None: + """Read a tri-state flag: absent (``None``), on, or off. + + A quoted YAML ``"false"`` must mean off. Plain ``bool(value)`` would read + that non-empty string as ON — the exact inversion this policy cannot + afford, since the flag decides whether reasoning is replayed at all. + """ + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + text = value.strip().lower() + if text in _TRUE_STRINGS: + return True + if text in _FALSE_STRINGS: + return False + raise ValueError(f"{key} must be a boolean or null, got {value!r}") + + +def _coerce_cap(value: Any) -> int: + """Read the cap as an exact non-negative integer; ``0``/absent = no cap. + + Rejects rather than coerces the two values ``int()`` would quietly + mangle: ``True`` (→ a 1-token cap, i.e. reasoning effectively off while + the config reads as enabled) and a fractional float (silently floored). + """ + if value is None: + return 0 + if isinstance(value, bool): + raise ValueError(f"{_CAP_ERROR}, not a boolean") + if isinstance(value, int): + cap = value + elif isinstance(value, float): + if not value.is_integer(): + raise ValueError(f"{_CAP_ERROR}; got the fractional {value!r}") + cap = int(value) + elif isinstance(value, str): + try: + cap = int(value.strip()) + except ValueError as exc: + raise ValueError(_CAP_ERROR) from exc + else: + raise ValueError(_CAP_ERROR) + if cap < 0: + raise ValueError(_CAP_ERROR) + return cap + + +def resolve_history_policy(config: Mapping[str, Any]) -> HistoryPolicy: + """Resolve the two history settings without losing explicit ``false``. + + ``thinking_in_history`` is the master switch. An explicit ``false`` always + disables reasoning history. Otherwise a positive + ``thinking_history_max_tokens`` implicitly enables capped history, while + ``true`` with a missing/zero cap keeps the full reasoning. With neither + setting present the legacy disabled default is preserved. + + Both values are validated here — at profile-load time, where a bad value + is a loud startup failure — rather than silently coerced per turn. + """ + enabled = _coerce_optional_bool( + config.get("thinking_in_history"), "thinking_in_history", + ) + cap = _coerce_cap(config.get("thinking_history_max_tokens")) + + if enabled is False: + return HistoryPolicy(thinking_in_history=False) + if cap > 0: + return HistoryPolicy( + thinking_in_history=True, + thinking_history_max_tokens=cap, + ) + return HistoryPolicy(thinking_in_history=bool(enabled)) + + +# ── Protocols ───────────────────────────────────────────────────────────────── + + +class ThinkingParser(Protocol): + """Extract thinking and visible content from an LLM response.""" + + def extract(self, response: Any, profile: ModelProfile) -> ThinkingResult: + """Parse *response* according to the model's ``thinking_format``.""" + ... + + +class MessageNormalizer(Protocol): + """Convert an LLM response to the message stored in conversation history.""" + + def to_history( + self, + response: Any, + thinking_result: ThinkingResult, + policy: HistoryPolicy, + ) -> Message: + """Return the message form appropriate for the conversation history.""" + ... + + +# ── Default implementations ─────────────────────────────────────────────────── + + +class DefaultThinkingParser: + """Handles four thinking formats. + + - ``"none"`` (classic OpenAI / non-reasoning models): no thinking + extraction; full text is visible. + - ``"tag"`` (Qwen / DeepSeek-via-SGLang): extract ``…`` + via regex from ``content``. + - ``"content_block"`` (Anthropic): ``content`` is a list of typed blocks; + separate ``{type: "thinking"}`` from ``{type: "text"}``. + - ``"reasoning_content"`` (DeepSeek-V4 API, gpt-5/o-series via standard + OpenAI-compatible proxies): thinking is a separate top-level + ``reasoning_content`` field on the response message. The native + :class:`~agent_core.llm.LLMResponse` surfaces it directly via + its ``reasoning_content`` attribute; this parser reads it from there. + """ + + def extract(self, response: Any, profile: ModelProfile) -> ThinkingResult: + tool_calls: list[dict[str, Any]] = list(response.tool_calls or []) + fmt = profile.thinking_format + + if fmt == "none": + content = response.content or "" + return ThinkingResult( + thinking="", + visible_content=content if isinstance(content, str) else "", + tool_calls=tool_calls, + ) + + if fmt == "tag": + raw = response.content or "" + if not isinstance(raw, str): + raw = "" + # Prefer the typed reasoning channel when the endpoint actually + # separates it (SGLang ``--reasoning-parser qwen3`` populates + # ``additional_kwargs.reasoning_content`` via our ChatOpenAI + # subclass). Falls back to the ``…`` regex when + # the channel is empty — that is the stock-SGLang / inline-tag + # case this format originally handled. + typed_rc = _extract_reasoning(response) + if typed_rc: + return ThinkingResult( + thinking=typed_rc, + visible_content=raw, + tool_calls=tool_calls, + ) + # Every block, not just the first: a turn can reopen + # between tool calls, and ``to_history`` rebuilds the message from + # this result, so a dropped block is reasoning lost from history. + # Substituting a newline (rather than deleting) keeps the text that + # surrounded a block from being concatenated — + # "ax\nb" must not become "ab". + matches = _THINK_RE.findall(raw) + if matches: + thinking = "\n".join(matches) + visible = _THINK_RE.sub("\n", raw).strip() + else: + thinking = "" + visible = raw + return ThinkingResult(thinking=thinking, visible_content=visible, tool_calls=tool_calls) + + if fmt == "content_block": + blocks = response.content or [] + # A content_block turn can still come back as a PLAIN STRING: the + # native Anthropic client returns a bare string whenever a turn + # carries no thinking block (adaptive thinking omitted, or the + # post-tool-call final answer turn), and any provider may degrade to + # a string. Iterating a string here would walk it CHARACTER by + # character, match no dict blocks, and silently drop the whole + # visible answer. Treat the string as visible text (reasoning read + # off the separate channel, if any). + if isinstance(blocks, str): + return ThinkingResult( + thinking=_extract_reasoning(response), + visible_content=blocks, + tool_calls=tool_calls, + ) + thinking_parts: list[str] = [] + text_parts: list[str] = [] + for block in blocks: + if not isinstance(block, dict): + continue + # ``thinking`` text may be "" when display=omitted, but the block + # still carries a ``signature`` preserved via raw_content_blocks. + if block.get("type") == "thinking": + thinking_parts.append(block.get("thinking", "")) + elif block.get("type") == "reasoning": + # OpenAI Responses reasoning item: the opaque + # ``encrypted_content`` is preserved in raw_content_blocks + # below; its human-readable ``summary`` (often empty unless + # opted-in) becomes the thinking text. Anthropic emits no + # "reasoning" blocks, so this never fires there. + for s in block.get("summary") or []: + if isinstance(s, dict): + thinking_parts.append(s.get("text", "")) + elif isinstance(s, str): + thinking_parts.append(s) + elif block.get("type") == "text": + text_parts.append(block.get("text", "")) + return ThinkingResult( + thinking="\n".join(thinking_parts), + visible_content="\n".join(text_parts), + tool_calls=tool_calls, + # Keep the verbatim block list (thinking/redacted_thinking incl. + # signatures / reasoning incl. encrypted_content) for faithful + # replay + trajectory. Only when content really is a non-empty list. + raw_content_blocks=( + list(blocks) if isinstance(blocks, list) and blocks else None + ), + ) + + if fmt == "reasoning_content": + # Reasoning lives on a separate channel — the native + # ``LLMResponse.reasoning_content`` field. ``_extract_reasoning`` + # reads it. Content stays untouched and becomes the visible reply. + content = response.content or "" + return ThinkingResult( + thinking=_extract_reasoning(response), + visible_content=content if isinstance(content, str) else "", + tool_calls=tool_calls, + ) + + # Unknown format — treat as no thinking + content = response.content or "" + return ThinkingResult( + thinking="", + visible_content=content if isinstance(content, str) else "", + tool_calls=tool_calls, + ) + + +# ── Native (langchain-free) layer ───────────────────────────────────────── +# +# ``_extract_reasoning`` reads the model's reasoning channel from the native +# ``LLMResponse.reasoning_content`` field so ``DefaultThinkingParser`` can +# surface thinking regardless of the model's ``thinking_format``. +# +# ``NativeMessageNormalizer`` returns an OpenAI-wire ``Message`` dict and, +# critically, does NOT carry ``reasoning_content`` onto the wire except for the +# one format that requires it. For +# ``tag`` models the reasoning is inlined into ``content`` as +# ``{rc}\n{visible}`` instead. + + +def _extract_reasoning(response: Any) -> str: + """Reasoning text from a native ``LLMResponse``. + + Reads the native ``reasoning_content`` attribute. As a defensive + fallback (no langchain dependency) it also accepts a response object + exposing ``additional_kwargs['reasoning_content']`` — harmless on a + native ``LLMResponse``, which has no such attribute. + """ + rc = getattr(response, "reasoning_content", "") or "" + if isinstance(rc, str) and rc: + return rc + kwargs = getattr(response, "additional_kwargs", None) or {} + if isinstance(kwargs, dict): + v = kwargs.get("reasoning_content") + if isinstance(v, str): + return v + return "" + + +def _to_openai_tool_calls(parsed: list[dict[str, Any]]) -> list[ToolCall]: + """Echo parsed tool calls into OpenAI wire format. + + Key order ``{type, id, function: {name, arguments}}`` — served checkpoints + are sensitive to this byte shape (migration gotcha #2). + + Native streaming responses can end with an empty or truncated + ``function.arguments`` string. The executor already treats that as + ``{}``, but echoing the malformed string into history makes the *next* LLM + request invalid on stricter OpenAI-compatible gateways. Re-serialise + malformed arguments as ``{}`` so an unknown/bad tool call remains a normal + recoverable tool error instead of poisoning every later request, including + the force-final/report call. + """ + def _arguments_json(raw: Any) -> str: + if isinstance(raw, dict): + return json.dumps(raw, ensure_ascii=False) + if isinstance(raw, str) and raw.strip(): + try: + decoded = json.loads(raw) + except (TypeError, ValueError): + return "{}" + return raw if isinstance(decoded, dict) else "{}" + return "{}" + + out: list[ToolCall] = [] + for tc in parsed: + args = tc.get("args") if "args" in tc else None + if args is None and "function" in tc: + raw_args = tc["function"].get("arguments", "{}") + args_str = _arguments_json(raw_args) + name = tc["function"].get("name", "") + else: + args_str = _arguments_json(args) + name = tc.get("name", "") + out.append({ + "type": "function", + "id": tc.get("id") or "", + "function": {"name": name, "arguments": args_str}, + }) + return out + + +class NativeMessageNormalizer: + """Convert an LLM response to the OpenAI-wire ``Message`` stored in history. + + Returns a plain OpenAI-wire ``Message`` dict. ``reasoning_content`` is + deliberately omitted from the wire message except for the + ``reasoning_content`` format — see the leak-guard note above. + """ + + def to_history( + self, + response: LLMResponse, + thinking_result: ThinkingResult, + policy: HistoryPolicy, + thinking_format: str = "none", + ) -> Message: + # Native Anthropic thinking / OpenAI Responses reasoning: when the + # response carried a verbatim content-block list (thinking+signature / + # redacted_thinking / reasoning+encrypted_content / text), keep it INTACT + # as the history message content. It is re-sent UNMODIFIED next turn — the + # provider validates signatures / encrypted_content server-side — so it + # must survive independently of the agent's thinking_in_history flag + # (verbatim replay is transport correctness, not an agent choice). No + # effect on other formats, where raw_content_blocks is None. + if thinking_result.raw_content_blocks is not None: + tool_calls = _to_openai_tool_calls(thinking_result.tool_calls) + return assistant_msg( + thinking_result.raw_content_blocks, tool_calls=tool_calls, + ) + + # The parser is the source of truth for visible text. Reusing raw + # ``response.content`` when it contains inline tags makes the + # builder below prepend the same reasoning a second time. + visible = thinking_result.visible_content + tool_calls = _to_openai_tool_calls(thinking_result.tool_calls) + reasoning = "" + if policy.thinking_in_history: + reasoning = ( + thinking_result.thinking + or getattr(response, "reasoning_content", "") + or "" + ) + if reasoning and policy.thinking_history_max_tokens: + reasoning = _cap_reasoning_tail( + reasoning, + policy.thinking_history_max_tokens, + ) + + # Wire round-trip of reasoning, by format — this is where the kernel + # owns the outbound reasoning shape (previously scattered across + # per-workflow ChatOpenAI reasoning subclasses). ``reasoning_content`` + # is NEVER emitted as a bare wire field except in the one format that + # requires it. The per-format rules live in + # ``assistant_msg_with_reasoning`` (core.messages) so self-contained + # workflow loops that bypass this normalizer share one implementation. + return assistant_msg_with_reasoning( + visible, reasoning, + tool_calls=tool_calls, thinking_format=thinking_format, + ) + + +def _cap_reasoning_tail(reasoning: str, max_tokens: int) -> str: + """Keep the most recent reasoning of ONE turn within ``max_tokens``. + + Conclusions and tool-use intent tend to occur at the end of a reasoning + trace, so the cap discards the oldest prefix. Token estimation uses the + loop's non-blocking tokenizer with its CJK-aware fallback; ``estimator`` is + passed explicitly rather than relying on the default so tests can swap the + module-level estimator. + """ + from .context_budget import estimate_tokens, truncate_text_to_tokens + + if max_tokens <= 0: + return reasoning + capped = truncate_text_to_tokens( + reasoning, + max_tokens, + marker="[... earlier reasoning truncated by the per-turn history token cap ...]\n", + estimator=estimate_tokens, + keep="tail", + ) + if capped != reasoning: + logger.debug( + "thinking history: capped this turn's reasoning to %d tokens " + "(%d chars kept of %d)", + max_tokens, len(capped), len(reasoning), + ) + return capped diff --git a/agent_core/runtime/loop/tool_call_parser.py b/agent_core/runtime/loop/tool_call_parser.py new file mode 100644 index 0000000..d7e0fb9 --- /dev/null +++ b/agent_core/runtime/loop/tool_call_parser.py @@ -0,0 +1,726 @@ +# pyright: reportMissingTypeArgument=false, reportUnknownArgumentType=false, reportUnknownMemberType=false, reportUnknownParameterType=false, reportUnknownVariableType=false, reportUnusedFunction=false +"""Generic tool call parser: native function-calling + JSON text fallback.""" + +from __future__ import annotations + +import contextlib +import json +import logging +import re +from typing import Any, Protocol, runtime_checkable + +logger = logging.getLogger(__name__) + +# Compiled once at import time for performance. +_TOOL_CALL_RE = re.compile(r"\s*(.*?)\s*", re.DOTALL) +_MCP_USE_TOOL_RE = re.compile( + r"\s*(.*?)\s*", + re.DOTALL | re.IGNORECASE, +) +# Dangling ``{...`` — closing tag (or trailing brace) lost to +# a ``max_tokens`` truncation. Captures the JSON body from the first +# unmatched opening; brace-balancing happens in +# ``_parse_dangling_json_tool_call``. +_DANGLING_TOOL_CALL_RE = re.compile( + r"\s*(\{[\s\S]*)\Z", re.DOTALL, +) + +# XML fallback patterns for models that leak tool calls as text. +_QWEN_TOOL_CALL_RE = re.compile( + r"\s*(.*?)\s*", re.DOTALL +) +_QWEN_PARAM_RE = re.compile(r"(.*?)", re.DOTALL) +_SEED_FUNCTION_RE = re.compile(r'(.*?)', re.DOTALL) +_SEED_PARAM_RE = re.compile( + r']*>(.*?)', re.DOTALL +) +_SEED_TAG_HINT_RE = re.compile(r"\s*(\[.*?\])\s*<\|FunctionCallEnd\|>", re.DOTALL, +) + +# Leaked inner-monologue / reasoning tags that some models emit into +# ``content`` even though they're supposed to be private. These fragments +# confuse downstream parsers and pollute the visible conversation +# history — we strip them defensively before any parser sees the text. +_LEAKED_TAG_RE = re.compile( + r"<(?:think|thinking|reasoning|seed:think|seed:reasoning)>" + r"(.*?)" + r"", + re.DOTALL | re.IGNORECASE, +) +# Open tag with no matching close → trim from the open tag onward. +_DANGLING_LEAKED_TAG_RE = re.compile( + r"<(?:think|thinking|reasoning|seed:think|seed:reasoning)>(.*)\Z", + re.DOTALL | re.IGNORECASE, +) +# Thinking-tag variants Seed/GPT-OSS emit despite the token never making it +# into the tokenizer vocabulary. Covered separately so the inner content can +# be captured as reasoning rather than silently discarded. +_NEVER_USED_TAG_RE = re.compile( + r"]*>(.*?)]*>", + re.DOTALL | re.IGNORECASE, +) +_NEVER_USED_DANGLING_RE = re.compile( + r"]*>(.*)\Z", + re.DOTALL | re.IGNORECASE, +) +_MODEL_THINKING_RE = re.compile( + r"(.*?)", + re.DOTALL | re.IGNORECASE, +) + +# Key used to stash salvaged reasoning on AIMessage.additional_kwargs so +# downstream consumers (trace logger, SSE observer, UI) can render it. +LEAKED_REASONING_KEY = "leaked_reasoning" + + +@runtime_checkable +class ToolCallParser(Protocol): + """Protocol for tool call parsers.""" + + def parse(self, response: Any, tool_names: set[str]) -> list[dict]: + """Extract tool calls from an LLM response. + + Args: + response: An AIMessage-like object with optional ``.tool_calls`` + and ``.content`` attributes. + tool_names: Set of known/allowed tool names. Calls whose name is + not in this set are filtered out. + + Returns: + List of dicts with at least ``"name"`` and ``"args"`` keys, + matching the LangChain tool-call dict structure. + """ + ... + + +def _normalize_native_tool_call(tc: Any) -> dict | None: + """Normalise one native tool_call into the executor's ``{name, args, id}``. + + Accepts the OpenAI wire shape ``{id, type, function:{name, arguments}}`` + (a native ``LLMResponse`` carries this — ``arguments`` is a JSON string) + and the already-parsed langchain shape ``{name, args(dict), id}``. Returns + ``None`` for anything unrecognised. + """ + if not isinstance(tc, dict): + return None + fn = tc.get("function") + if isinstance(fn, dict): + raw_args = fn.get("arguments", "") + if isinstance(raw_args, str): + try: + args = json.loads(raw_args) if raw_args.strip() else {} + except (ValueError, TypeError): + args = {} + elif isinstance(raw_args, dict): + args = raw_args + else: + args = {} + return { + "name": fn.get("name", "") or "", + "args": args if isinstance(args, dict) else {}, + "id": tc.get("id", "") or "", + } + # Already-parsed shape (legacy langchain AIMessage.tool_calls). + return { + "name": tc.get("name", "") or "", + "args": tc.get("args", {}) or {}, + "id": tc.get("id", "") or "", + } + + +class DefaultToolCallParser: + """Two-strategy tool call parser (native FC first, JSON fallback second).""" + + def __init__(self, *, keep_unknown_native_companions: bool = False) -> None: + """Configure how a mixed native batch handles unknown tool names. + + Unknown-only batches are always retained so the executor can return a + corrective error. In a mixed batch, hosts may either discard unknown + companions (the default) or retain every unambiguous native call. + """ + self.keep_unknown_native_companions = keep_unknown_native_companions + + def parse(self, response: Any, tool_names: set[str]) -> list[dict]: + # ── Strategy 1: Native function calling ────────────────────────────── + # ``response.tool_calls`` is OpenAI wire shape on a native + # ``LLMResponse`` (``{id, type, function:{name, arguments}}``) and the + # parsed ``{name, args, id}`` shape on a legacy langchain message; + # ``_normalize_native_tool_call`` accepts either and emits the parsed + # shape the executor consumes. + native_raw = list(getattr(response, "tool_calls", None) or []) + native = [ + n for n in (_normalize_native_tool_call(tc) for tc in native_raw) if n + ] + if native: + known_calls = [tc for tc in native if tc.get("name") in tool_names] + logger.debug( + "native FC: %d total, %d known", len(native), len(known_calls), + ) + if known_calls and not self.keep_unknown_native_companions: + # When the model emitted at least one executable action, drop + # unknown companions instead of turning an otherwise useful + # parallel batch into a visible failure. If every call is + # unknown, keep them so the executor can return an explicit + # correction; returning an empty list there could be mistaken + # for a completed no-tool turn. + # + # A dropped call still occupies a ``tool_call_id`` in the + # assistant history message, which the agent loop wrote from + # the raw response before calling us. ``_answer_dropped_tool_calls`` + # there answers those ids — without it an orphan id is a hard + # HTTP 400 on Azure and other providers. + unknown = [ + tc.get("name", "") + for tc in native + if tc.get("name") not in tool_names + ] + if unknown: + logger.warning( + "native FC: dropped unknown companion tool calls: %s", + unknown, + ) + return known_calls + return native + + # ── Strategy 2: JSON text fallback ─────────────────────────────────── + raw_content = getattr(response, "content", "") or "" + text = self._extract_text(raw_content) + parsed = self._parse_json_tool_calls(text, tool_names) + if parsed: + return parsed + return self._parse_mcp_tool_calls(text, tool_names) + + # ── Internal helpers ────────────────────────────────────────────────────── + + @staticmethod + def _extract_text(content: Any) -> str: + """Normalise content to a plain string. + + Handles: + - ``str`` — returned as-is. + - ``list`` of blocks (Anthropic / LangChain format) — text blocks + are concatenated. + """ + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for block in content: + if isinstance(block, str): + parts.append(block) + elif isinstance(block, dict): + text = block.get("text", "") + if isinstance(text, str): + parts.append(text) + return "\n".join(parts) + return str(content) if content else "" + + @staticmethod + def _parse_json_tool_calls(text: str, tool_names: set[str]) -> list[dict]: + """Find all … blocks and parse each as JSON.""" + results: list[dict] = [] + for match in _TOOL_CALL_RE.finditer(text): + raw = match.group(1).strip() + try: + payload = json.loads(raw) + except (json.JSONDecodeError, ValueError): + logger.debug("Skipping malformed JSON tool_call block: %r", raw[:120]) + continue + + if not isinstance(payload, dict): + logger.debug("Skipping non-object tool_call payload: %r", payload) + continue + + name = str(payload.get("tool", "") or "").strip() + if not name or name not in tool_names: + logger.debug("Skipping unknown/missing tool name: %r", name) + continue + + args = payload.get("args", {}) + if not isinstance(args, dict): + args = {} + + results.append({"name": name, "args": args}) + + logger.debug("JSON fallback: found %d tool calls", len(results)) + return results + + @staticmethod + def _parse_mcp_tool_calls(text: str, tool_names: set[str]) -> list[dict]: + """Parse Agent-Protocol style blocks.""" + if "use_mcp_tool" not in tool_names: + return [] + + results: list[dict] = [] + for idx, match in enumerate(_MCP_USE_TOOL_RE.finditer(text)): + body = match.group(1) + server_name = _extract_xml_value(body, "server_name") + tool_name = _extract_xml_value(body, "tool_name") + if not server_name or not tool_name: + logger.debug("Skipping malformed use_mcp_tool block: %r", body[:120]) + continue + + raw_args = _extract_xml_value(body, "arguments") + arguments: dict[str, Any] = {} + if raw_args: + try: + parsed = json.loads(raw_args) + except (json.JSONDecodeError, ValueError): + logger.debug( + "use_mcp_tool arguments were not JSON: %r", + raw_args[:120], + ) + parsed = {} + if isinstance(parsed, dict): + arguments = parsed + + results.append({ + "name": "use_mcp_tool", + "args": { + "server_name": server_name, + "tool_name": tool_name, + "arguments": arguments, + }, + "id": f"mcp_tc_{idx}", + }) + + logger.debug("MCP XML fallback: found %d tool calls", len(results)) + return results + + +class MultiFormatToolCallParser(DefaultToolCallParser): + """Default parser + Qwen/Seed XML + Seed FunctionCall wrapper fallbacks. + + Priority order: + 1. Native ``response.tool_calls`` (via parent) + 2. JSON ``{"tool": …}`` (via parent) + 3. Seed FunctionCall wrapper ``<|FunctionCallBegin|>[…]<|FunctionCallEnd|>`` + 4. Qwen XML ``…`` + 5. Seed XML ``…`` + + Strategies 3-5 only fire when there is no native ``tool_calls`` field — + an empty native list (all filtered out) is treated as a deliberate signal + and short-circuits the fallbacks, matching the base parser's semantics. + The FunctionCall wrapper is the exception: an unambiguous Seed-specific + marker overrides truthy-but-malformed native ``tool_calls`` from upstream + proxy repack failures. + + Leaked ``…`` / ``…`` tags are + stripped from ``response.content`` before parsing. + + Use :meth:`parse_text` to recover leaks from arbitrary text that isn't + on ``response.content`` (e.g. a model's native ```` block when it + wrote tool calls inside its private reasoning instead of using the + visible content channel). + """ + + def parse(self, response: Any, tool_names: set[str]) -> list[dict]: + # Response-level cleaning: strip leaked / tags + # in place before any parsing. Safe — these tags are private + # inner-monologue that should never have been emitted. + self._clean_leaked_content(response) + + base = super().parse(response, tool_names) + if base: + return base + + text = self._extract_text(getattr(response, "content", "") or "") + + # FunctionCall wrapper has higher priority than the native short- + # circuit: this marker is unambiguous (no false-positive matches in + # real prose) and we trust it over a possibly-empty native list + # produced by an upstream proxy that failed to repack it. Side- + # effect strip keeps history clean — that's why this branch lives + # here in ``parse()`` rather than the side-effect-free + # :meth:`parse_text`. + if text and "<|FunctionCallBegin|>" in text: + fc = _parse_fc_wrapped(text, tool_names) + if fc: + self._strip_fc_wrappers(response) + return fc + + # Native API was used (even with unknown names) — don't second-guess + # with XML regexes on residual content. + if getattr(response, "tool_calls", None): + return base + + return self.parse_text(text, tool_names) + + def parse_text(self, text: str, tool_names: set[str]) -> list[dict]: + """Recover tool calls from a raw text string (pure, no side effects). + + Used both as the content fallback in :meth:`parse` and to recover + leaks from a model's thinking block — Qwen 3.5 35B writes Hermes XML + inside ```` instead of using native ``tool_calls``. Returns + ``[]`` when no recognisable leak markers are present. + """ + if not text: + return [] + if "" in text and ( + calls := self._parse_mcp_tool_calls(text, tool_names) + ): + return calls + if ( + "" in text + and "" in text and ( + calls := _parse_fc_wrapped(text, tool_names) + ): + return calls + # Seed XML returns directly even when empty — once we recognise the + # format we don't fall through to other parsers. + if '{...`` truncated mid-stream by + # ``max_tokens`` (closing ```` lost). Try brace- + # balanced JSON extraction. Only fires when nothing else matched + # — guarded by an explicit check so we don't pay the cost on the + # common path. + if "" in text and "" not in text: + recovered = _parse_dangling_json_tool_call(text, tool_names) + if recovered: + return recovered + return [] + + @staticmethod + def _clean_leaked_content(response: Any) -> None: + """Strip leaked ````/```` tags from response.content + while preserving the extracted inner content as reasoning. + + Mutates the response in place: visible content loses the tag blocks, + and any recovered thinking/reasoning text is concatenated onto + ``response.additional_kwargs[LEAKED_REASONING_KEY]`` so downstream + consumers (trace logger, SSE UI, evidence observer) can display it. + """ + content = getattr(response, "content", None) + if content is None: + return + + recovered_parts: list[str] = [] + + if isinstance(content, str): + cleaned, reasoning = extract_leaked_reasoning(content) + if reasoning: + recovered_parts.append(reasoning) + if cleaned != content: + # A frozen/immutable response shouldn't happen here, but must + # not crash the parse if it does. + with contextlib.suppress(Exception): + response.content = cleaned + elif isinstance(content, list): + # LangChain content-blocks list: each block may be a str or dict. + changed = False + new_blocks: list[Any] = [] + for block in content: + if isinstance(block, str): + clean, reasoning = extract_leaked_reasoning(block) + if reasoning: + recovered_parts.append(reasoning) + changed = changed or clean != block + new_blocks.append(clean) + elif isinstance(block, dict) and isinstance( + block.get("text"), str, + ): + clean, reasoning = extract_leaked_reasoning(block["text"]) + if reasoning: + recovered_parts.append(reasoning) + b2 = dict(block) + b2["text"] = clean + changed = changed or b2["text"] != block["text"] + new_blocks.append(b2) + else: + new_blocks.append(block) + if changed: + with contextlib.suppress(Exception): + response.content = new_blocks + + if recovered_parts: + _attach_leaked_reasoning(response, "\n\n".join(recovered_parts)) + + @staticmethod + def _parse_qwen_xml(text: str, tool_names: set[str]) -> list[dict]: + results: list[dict] = [] + for i, match in enumerate(_QWEN_TOOL_CALL_RE.finditer(text)): + name = match.group(1) + if name not in tool_names: + logger.debug("Qwen XML: skipping unknown tool %r", name) + continue + body = match.group(2) + args: dict = {} + for pm in _QWEN_PARAM_RE.finditer(body): + key = pm.group(1) + args[key] = _coerce_param_value(pm.group(2).strip()) + results.append({"name": name, "args": args, "id": f"qwen_tc_{i}"}) + logger.debug("Qwen XML fallback: found %d tool calls", len(results)) + return results + + @staticmethod + def _parse_seed_xml(text: str, tool_names: set[str]) -> list[dict]: + results: list[dict] = [] + for i, match in enumerate(_SEED_FUNCTION_RE.finditer(text)): + name = match.group(1) + if name not in tool_names: + logger.debug("Seed XML: skipping unknown tool %r", name) + continue + body = match.group(2) + args: dict = {} + for pm in _SEED_PARAM_RE.finditer(body): + key = pm.group(1) + args[key] = _coerce_param_value(pm.group(2).strip()) + results.append({"name": name, "args": args, "id": f"seed_tc_{i}"}) + logger.debug("Seed XML fallback: found %d tool calls", len(results)) + return results + + @staticmethod + def _strip_fc_wrappers(response: Any) -> None: + """Strip ``<|FunctionCallBegin|>…<|FunctionCallEnd|>`` from + ``response.content`` after we've parsed the call out, so the + message history doesn't carry the duplicate wire format.""" + content = getattr(response, "content", None) + if not isinstance(content, str): + return + cleaned = _FC_WRAPPED_RE.sub("", content).strip() + if cleaned != content: + with contextlib.suppress(Exception): + response.content = cleaned + + +def _balance_json_object(text: str, start: int) -> int | None: + """Return the index one past the matching ``}`` for ``text[start] == '{'``. + + Brace-counts depth while respecting JSON string semantics (``"..."`` + with ``\\`` escapes) so braces inside strings don't bump the depth. + Returns ``None`` when the object never closes — i.e. truncation + happened inside the JSON. + """ + if start >= len(text) or text[start] != "{": + return None + depth = 0 + in_string = False + escape = False + for i in range(start, len(text)): + c = text[i] + if escape: + escape = False + continue + if in_string: + if c == "\\": + escape = True + elif c == '"': + in_string = False + continue + if c == '"': + in_string = True + elif c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + return i + 1 + return None + + +def _parse_dangling_json_tool_call( + text: str, tool_names: set[str], +) -> list[dict]: + """Recover a single ``{...}`` whose closing tag was + truncated by ``max_tokens``. + + Fires only when ```` is present and ```` is + not. Brace-balances the JSON body from the first ``{`` after the + opening tag; bails out when the body is genuinely incomplete + (truncation inside a string). At most one call is recovered — the + truncation point is by definition the end of useful content, so any + later calls in the same response don't exist. + """ + match = _DANGLING_TOOL_CALL_RE.search(text) + if not match: + return [] + body_start = match.start(1) + end = _balance_json_object(text, body_start) + if end is None: + logger.debug( + "Dangling : JSON body truncated mid-value, " + "cannot recover. preview=%r", text[body_start:body_start + 120], + ) + return [] + raw = text[body_start:end] + try: + payload = json.loads(raw) + except (json.JSONDecodeError, ValueError) as exc: + logger.debug( + "Dangling : balanced body did not parse " + "(%s). preview=%r", exc, raw[:120], + ) + return [] + if not isinstance(payload, dict): + return [] + name = str(payload.get("tool", "") or "").strip() + if not name or name not in tool_names: + logger.debug( + "Dangling : unknown or missing tool %r", name, + ) + return [] + args = payload.get("args", {}) + if not isinstance(args, dict): + args = {} + logger.info( + "Recovered dangling for %r (lost ; " + "%d-byte body)", name, end - body_start, + ) + return [{"name": name, "args": args, "id": "dangling_tc_0"}] + + +def _parse_fc_wrapped(text: str, tool_names: set[str]) -> list[dict]: + """Parse Seed reasoning-mode ``<|FunctionCallBegin|>[…]<|FunctionCallEnd|>``. + + Body is a JSON array of ``{"name": str, "parameters": dict}`` objects. + """ + results: list[dict] = [] + idx = 0 + for match in _FC_WRAPPED_RE.finditer(text): + try: + calls = json.loads(match.group(1)) + except (json.JSONDecodeError, ValueError) as exc: + logger.warning( + "FunctionCall wrapper: bad JSON (%s) body_preview=%r", + exc, match.group(1)[:200], + ) + continue + if not isinstance(calls, list): + logger.warning( + "FunctionCall wrapper: body not a list, got %s", + type(calls).__name__, + ) + continue + for call in calls: + if not isinstance(call, dict): + continue + name = call.get("name", "") + if name not in tool_names: + logger.warning( + "FunctionCall wrapper: skipping unknown tool %r " + "(allowed=%s)", name, sorted(tool_names), + ) + continue + args = call.get("parameters") or call.get("arguments") or {} + if not isinstance(args, dict): + continue + results.append({"name": name, "args": args, "id": f"fc_tc_{idx}"}) + idx += 1 + logger.debug("FunctionCall wrapper: found %d tool calls", len(results)) + return results + + +def _extract_xml_value(body: str, tag: str) -> str: + match = re.search( + rf"<{re.escape(tag)}>\s*(.*?)\s*", + body, + re.DOTALL | re.IGNORECASE, + ) + return match.group(1).strip() if match else "" + + +def _coerce_param_value(raw: str) -> Any: + """Try to decode a parameter value as JSON; fall back to the raw string.""" + try: + return json.loads(raw) + except (json.JSONDecodeError, ValueError): + return raw + + +def _strip_leaked_tags(text: str) -> str: + """Remove leaked inner-monologue / reasoning tags from model output.""" + cleaned, _ = extract_leaked_reasoning(text) + return cleaned + + +def _attach_leaked_reasoning(response: Any, reasoning: str) -> None: + """Append recovered reasoning to ``response.response_metadata``. + + Native ``LLMResponse`` carries salvaged reasoning on ``response_metadata`` + (where ``llm_client.extract_leaked_reasoning`` reads it); a legacy + langchain message's ``additional_kwargs`` is honoured as a fallback. + Accumulates across repeated parser invocations so we don't clobber + reasoning extracted on a prior pass (e.g. when the same message is + re-parsed as history). Silent no-op if the response is frozen. + """ + if not reasoning: + return + meta = getattr(response, "response_metadata", None) + if not isinstance(meta, dict): + meta = getattr(response, "additional_kwargs", None) + if not isinstance(meta, dict): + with contextlib.suppress(Exception): + response.response_metadata = {LEAKED_REASONING_KEY: reasoning} + return + prior = meta.get(LEAKED_REASONING_KEY, "") + if prior: + if reasoning in prior: + return + meta[LEAKED_REASONING_KEY] = f"{prior}\n\n{reasoning}" + else: + meta[LEAKED_REASONING_KEY] = reasoning + + +def extract_leaked_reasoning(text: str) -> tuple[str, str]: + """Strip leaked thinking/reasoning tags and return (cleaned, reasoning). + + Recovers inner content from every leak pattern we know about: + * ``…`` / ``…`` / ``…`` (balanced) + * ``…`` (Seed / GPT-OSS) + * ``…`` + * dangling opens truncated mid-response — the tail is taken as + reasoning, not silently dropped. + + The reasoning string is the concatenation of every captured block, joined + by blank lines. Empty captures are skipped. The cleaned text has all + matched blocks removed, matching the previous ``_strip_leaked_tags`` + contract. + """ + if not text: + return text, "" + + reasoning_parts: list[str] = [] + + def _collect(pattern: re.Pattern, src: str) -> str: + out = src + for match in pattern.finditer(src): + inner = match.group(1).strip() if match.groups() else "" + if inner: + reasoning_parts.append(inner) + out = pattern.sub("", out) + return out + + cleaned = text + # Specific "never_used" variants first — they have a different closing + # shape from the generic / pair and must not be left + # dangling after the generic strip. + cleaned = _collect(_NEVER_USED_TAG_RE, cleaned) + cleaned = _collect(_MODEL_THINKING_RE, cleaned) + cleaned = _collect(_LEAKED_TAG_RE, cleaned) + + # Dangling tails — capture the remainder as reasoning (truncated inner + # monologue), then drop it from the cleaned output. + for dangling_re in (_NEVER_USED_DANGLING_RE, _DANGLING_LEAKED_TAG_RE): + match = dangling_re.search(cleaned) + if match: + inner = match.group(1).strip() + if inner: + reasoning_parts.append(inner) + cleaned = dangling_re.sub("", cleaned) + + reasoning = "\n\n".join(p for p in reasoning_parts if p) + return cleaned, reasoning diff --git a/agent_core/runtime/loop/tool_exec.py b/agent_core/runtime/loop/tool_exec.py new file mode 100644 index 0000000..33d769e --- /dev/null +++ b/agent_core/runtime/loop/tool_exec.py @@ -0,0 +1,285 @@ +# pyright: reportUnknownArgumentType=false, reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnnecessaryIsInstance=false +"""Product-neutral parallel tool execution with explicit host hooks.""" + +from __future__ import annotations + +import asyncio +import contextlib +import time +from collections.abc import Awaitable, Callable +from contextlib import AbstractContextManager, nullcontext, suppress +from dataclasses import dataclass +from typing import Any, Protocol, runtime_checkable + +from agent_core.loop_types import ToolResult + +__all__ = [ + "PROTECTED_FANIN_TOOLS", + "TOOL_RESULT_MAX_CHARS", + "DefaultToolResultPostProcessor", + "ToolExecutionHooks", + "ToolLike", + "ToolResultPostProcessor", + "execute_tools", +] + +TOOL_RESULT_MAX_CHARS = 150_000 +_AGGREGATION_TOOLS = frozenset({"collect_reports", "collect_results"}) +_SELF_TIMING_TOOL_FLOORS: dict[str, int] = { + "run_python_code": 660, + "download_file": 660, +} +PROTECTED_FANIN_TOOLS = frozenset( + {"collect_reports", "collect_results", "delegate_subtask", "assign_task"} +) + + +@runtime_checkable +class ToolLike(Protocol): + """Small structural contract required by the execution engine.""" + + name: str + + async def ainvoke(self, args: dict[str, Any]) -> Any: ... + + +@runtime_checkable +class ToolResultPostProcessor(Protocol): + def process(self, tool_result: ToolResult) -> str: ... + + +class DefaultToolResultPostProcessor: + def __init__(self, max_chars: int | None = None) -> None: + self._max_chars = max_chars + + def process(self, tool_result: ToolResult) -> str: + content = tool_result.result + cap = self._max_chars + if cap and isinstance(content, str) and len(content) > cap: + return content[:cap] + ( + f"\n\n[... truncated {len(content) - cap} chars past " + f"{cap}-char cap]" + ) + return content if isinstance(content, str) else str(content) + + +TimeoutResolver = Callable[[str, dict[str, Any], int], float] +AwaitCall = Callable[[Awaitable[Any], str, dict[str, Any], float], Awaitable[Any]] +CallScopeFactory = Callable[ + [dict[str, Any], float], AbstractContextManager[Any] +] +ResultTransform = Callable[[str, str], str] +BatchTransform = Callable[[list[ToolResult]], list[ToolResult]] +CallObserver = Callable[[str], None] +UnknownResult = Callable[[str, tuple[str, ...]], str] +TimeoutResult = Callable[[str, float, float], str] +FailureResult = Callable[[str, Exception], str] +InterruptedResult = Callable[[str], str] + + +def _default_timeout(name: str, args: dict[str, Any], configured: int) -> float: + floor = _SELF_TIMING_TOOL_FLOORS.get(name) + if floor is not None: + return float(max(configured, floor)) + if name not in _AGGREGATION_TOOLS: + return float(configured) + try: + requested = int(args.get("timeout", 0) or 0) + except (TypeError, ValueError): + requested = 0 + return float(max(configured, requested + 5)) if requested > 0 else float(configured) + + +async def _default_await( + awaitable: Awaitable[Any], + _name: str, + _args: dict[str, Any], + timeout: float, +) -> Any: + return await asyncio.wait_for(awaitable, timeout=timeout) + + +def _default_scope( + _call: dict[str, Any], _timeout: float +) -> AbstractContextManager[Any]: + return nullcontext() + + +def _identity_result(_name: str, value: str) -> str: + if len(value) <= TOOL_RESULT_MAX_CHARS: + return value + return value[:TOOL_RESULT_MAX_CHARS] + ( + f"\n... [truncated, {len(value)} chars total]" + ) + + +def _identity_batch(results: list[ToolResult]) -> list[ToolResult]: + return results + + +def _noop_call(_name: str) -> None: + return None + + +def _unknown_result(name: str, available: tuple[str, ...]) -> str: + choices = ", ".join(available) or "(none)" + return ( + f"Error: unknown tool '{name}' is not available. " + f"Available tools: {choices}. Call one of these instead." + ) + + +def _timeout_result(name: str, elapsed_s: float, _effective_timeout: float) -> str: + return f"Error: tool '{name}' timed out after {elapsed_s:.1f}s" + + +def _failure_result(name: str, exc: Exception) -> str: + del name + return f"Error: {type(exc).__name__}: {exc}" + + +def _interrupted_result(_name: str) -> str: + return ( + "[interrupted] Waiting for sub-agent reports was cancelled because " + "a new user message arrived." + ) + + +@dataclass(frozen=True) +class ToolExecutionHooks: + """Host decisions around a shared execution lifecycle. + + The core owns dispatch, interrupt races, cancellation hygiene and result + construction. Products inject deadline policy, contextvars, metering, + spill/truncation behavior and aggregate budgeting through these hooks. + """ + + resolve_timeout: TimeoutResolver = _default_timeout + await_call: AwaitCall = _default_await + call_scope: CallScopeFactory = _default_scope + transform_result: ResultTransform = _identity_result + transform_batch: BatchTransform = _identity_batch + on_call: CallObserver = _noop_call + unknown_result: UnknownResult = _unknown_result + timeout_result: TimeoutResult = _timeout_result + failure_result: FailureResult = _failure_result + interrupted_result: InterruptedResult = _interrupted_result + + +async def execute_tools( + tool_calls: list[dict[str, Any]], + tool_map: dict[str, ToolLike], + *, + timeout: int, + turn: int, + count_offset: int, + interrupt_waiter: Callable[[dict[str, Any]], Awaitable[bool]] | None = None, + hooks: ToolExecutionHooks | None = None, +) -> list[ToolResult]: + """Execute a tool-call batch concurrently and convert every outcome. + + Exceptions and timeouts become error results; no raw tool failure escapes + into the loop. A fan-in tool may race an interrupt waiter so a follow-up + user message can wake it without abandoning already collected work. + """ + + runtime = hooks or ToolExecutionHooks() + + async def _run_one(call: dict[str, Any], index: int) -> ToolResult: + name = str(call.get("name", "") or "") + raw_args = call.get("args", {}) or {} + args = raw_args if isinstance(raw_args, dict) else {} + tool_call_id = str( + call.get("id", "") or f"call_{turn}_{count_offset + index}" + ) + start = time.monotonic() + tool = tool_map.get(name) + if tool is None: + return ToolResult( + name=name, + args=args, + result=runtime.unknown_result(name, tuple(sorted(tool_map))), + duration_ms=0, + tool_call_id=tool_call_id, + is_error=True, + ) + + effective_timeout = runtime.resolve_timeout(name, args, timeout) + runtime.on_call(name) + invoke_task: asyncio.Future[Any] | None = None + interrupt_task: asyncio.Future[bool] | None = None + woke_for_interrupt = False + try: + with runtime.call_scope( + {**call, "id": tool_call_id}, effective_timeout + ): + invocation = runtime.await_call( + tool.ainvoke(args), name, args, effective_timeout + ) + if interrupt_waiter is not None and name in _AGGREGATION_TOOLS: + invoke_task = asyncio.ensure_future(invocation) + interrupt_task = asyncio.ensure_future(interrupt_waiter(call)) + done, _ = await asyncio.wait( + {invoke_task, interrupt_task}, + return_when=asyncio.FIRST_COMPLETED, + ) + if interrupt_task in done and bool(interrupt_task.result()): + woke_for_interrupt = True + if invoke_task not in done: + invoke_task.cancel() + with suppress(asyncio.CancelledError): + await invoke_task + raw = runtime.interrupted_result(name) + else: + raw = invoke_task.result() + else: + raw = await invocation + + result = runtime.transform_result(name, str(raw) if raw is not None else "") + return ToolResult( + name=name, + args=args, + result=result, + duration_ms=int((time.monotonic() - start) * 1000), + tool_call_id=tool_call_id, + is_error=False, + interrupted=woke_for_interrupt, + ) + except TimeoutError: + elapsed = int((time.monotonic() - start) * 1000) + return ToolResult( + name=name, + args=args, + result=runtime.timeout_result( + name, elapsed / 1000, effective_timeout + ), + duration_ms=elapsed, + tool_call_id=tool_call_id, + is_error=True, + ) + except asyncio.CancelledError: + raise + except Exception as exc: + return ToolResult( + name=name, + args=args, + result=runtime.failure_result(name, exc), + duration_ms=int((time.monotonic() - start) * 1000), + tool_call_id=tool_call_id, + is_error=True, + ) + finally: + if interrupt_task is not None and not interrupt_task.done(): + interrupt_task.cancel() + with suppress(asyncio.CancelledError): + await interrupt_task + if invoke_task is not None and not invoke_task.done(): + invoke_task.cancel() + with suppress(asyncio.CancelledError): + await invoke_task + elif invoke_task is not None and not invoke_task.cancelled(): + with contextlib.suppress(Exception): + invoke_task.exception() + + tasks = [_run_one(call, index) for index, call in enumerate(tool_calls)] + return runtime.transform_batch(list(await asyncio.gather(*tasks))) diff --git a/docs/agent-loop-boundary.md b/docs/agent-loop-boundary.md new file mode 100644 index 0000000..ee7986d --- /dev/null +++ b/docs/agent-loop-boundary.md @@ -0,0 +1,28 @@ +# Agent-loop boundary + +AgentCore owns one complete logical-turn engine: + +- thinking extraction and history normalization; +- native, JSON, MCP, Qwen, Seed and wrapped function-call parsing; +- parallel tool dispatch, interruption and cancellation hygiene; +- observer lifecycle, compaction, rollback, continuation and finalization; +- the composition of the shared physical LLM-call runtime across turns. + +The products keep policy and state that cannot be portable. `AgentLoopHooks` +injects session affinity, wall-deadline lookup, provider-chain state, execution +scope storage, cancellation cleanup and spill-reference detection. +`ToolExecutionHooks` injects effective timeout calculation, wall-clamped waits, +per-call ContextVars and metering, result spilling/truncation and aggregate +result budgeting. + +Model endpoint registries are also product data. A product compatibility facade +calls `configure_model_registry()` with its packaged YAML path; AgentCore has no +knowledge of either product package. + +Mixed native function-call batches have one explicit parser decision. By +default, unknown companions are dropped when a known call is executable, while +unknown-only batches are retained for corrective tool errors. A host needing +the previous keep-all behavior constructs the parser with +`keep_unknown_native_companions=True`. In either mode, the loop answers any +dropped assistant `tool_call_id`, preventing an orphaned call from making the +next provider request invalid. diff --git a/tests/test_agent_loop_engine.py b/tests/test_agent_loop_engine.py new file mode 100644 index 0000000..94bd1de --- /dev/null +++ b/tests/test_agent_loop_engine.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from agent_core.llm import LLMResponse +from agent_core.loop_types import LoopConfig, LoopPolicy +from agent_core.runtime.loop.agent_loop import AgentLoopHooks, run_agent_loop +from agent_core.runtime.loop.model_profile import ModelProfile +from agent_core.runtime.loop.tool_exec import ToolExecutionHooks + + +class SequenceLLM: + def __init__(self, responses: list[LLMResponse]) -> None: + self.responses = responses + self.calls: list[list[dict[str, Any]]] = [] + + async def chat(self, messages, **_kwargs) -> LLMResponse: + self.calls.append(messages) + return self.responses.pop(0) + + def stream(self, messages, **_kwargs): + raise AssertionError("streaming was not requested") + + +class EchoTool: + name = "echo" + + async def ainvoke(self, args: dict[str, Any]) -> Any: + return args["value"] + + def to_openai_schema(self) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": self.name, + "description": "echo a value", + "parameters": {"type": "object"}, + }, + } + + +def _config() -> LoopConfig: + return LoopConfig( + max_turns=3, + loop_policy=LoopPolicy(no_tool_behavior="stop"), + max_llm_retries=1, + ) + + +@pytest.mark.asyncio +async def test_agent_loop_executes_tool_then_returns_final_answer() -> None: + llm = SequenceLLM( + [ + LLMResponse( + content="", + tool_calls=[ + { + "id": "tc1", + "type": "function", + "function": { + "name": "echo", + "arguments": '{"value":"hello"}', + }, + } + ], + ), + LLMResponse(content="finished"), + ] + ) + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[EchoTool()], + config=_config(), + model_profile=ModelProfile(model_id="test", provider="test"), + ) + + assert result.final_content == "finished" + assert result.tool_calls_count == 1 + assert any(message.get("role") == "tool" and message.get("content") == "hello" + for message in result.messages) + + +@pytest.mark.asyncio +async def test_runtime_hooks_wrap_scope_llm_and_tool_boundaries() -> None: + events: list[str] = [] + + def enter_scope(cfg, phase_id, metadata): + events.append(f"enter:{cfg.max_turns}:{phase_id}:{metadata['agent_id']}") + return type("Scope", (), {"metadata": metadata})(), "token" + + hooks = AgentLoopHooks( + sticky_session_enabled=lambda: events.append("sticky") or True, + wall_deadline_remaining=lambda: events.append("deadline") or None, + chain_fallback_active=lambda: False, + enter_scope=enter_scope, + exit_scope=lambda token: events.append(f"exit:{token}"), + tool_execution=ToolExecutionHooks( + on_call=lambda name: events.append(f"tool:{name}") + ), + ) + llm = SequenceLLM([LLMResponse(content="done")]) + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[], + config=LoopConfig( + max_turns=1, + task_id="task", + role_id="role", + loop_policy=LoopPolicy(no_tool_behavior="stop"), + max_llm_retries=1, + ), + runtime_hooks=hooks, + ) + + assert result.final_content == "done" + assert events[0] == "sticky" + assert events[1].startswith("enter:1:") + assert "deadline" in events + assert events[-1] == "exit:token" + + +@pytest.mark.asyncio +async def test_host_can_override_session_binding() -> None: + bound: list[str] = [] + llm = SequenceLLM([LLMResponse(content="done")]) + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[], + config=LoopConfig( + max_turns=1, + task_id="runtime-task", + llm_session_id="gateway-session", + loop_policy=LoopPolicy(no_tool_behavior="stop"), + max_llm_retries=1, + ), + runtime_hooks=AgentLoopHooks( + bind_session=lambda client, session_id: ( + bound.append(session_id) or client + ) + ), + ) + + assert result.final_content == "done" + assert bound == ["gateway-session"] diff --git a/tests/test_model_profile_native.py b/tests/test_model_profile_native.py new file mode 100644 index 0000000..0c138b9 --- /dev/null +++ b/tests/test_model_profile_native.py @@ -0,0 +1,215 @@ +"""Native (langchain-free) thinking-parse + history-normalize for the loop. + +Phase-1b additive scaffolding: ``DefaultThinkingParser`` is now native-aware +(reads reasoning from a native ``LLMResponse`` OR the legacy langchain +``additional_kwargs``), and ``NativeMessageNormalizer`` produces the OpenAI-wire +``Message`` dict that the flip will store in history. The single most important +assertion here is the **reasoning_content leak guard** (PR #209): the wire +assistant message must NOT carry a ``reasoning_content`` key. +""" +from __future__ import annotations + +from types import SimpleNamespace + +from agent_core.llm import LLMResponse +from agent_core.runtime.loop.model_profile import ( + DefaultThinkingParser, + HistoryPolicy, + ModelProfile, + NativeMessageNormalizer, + ThinkingResult, + _extract_reasoning, + _to_openai_tool_calls, +) + + +def _profile(fmt): + return ModelProfile(model_id="m", provider="p", thinking_format=fmt) + + +# ── dual-compatible reasoning extraction ──────────────────────────────────── + + +def test_extract_reasoning_from_native_llmresponse(): + r = LLMResponse(content="ans", reasoning_content="my thoughts") + assert _extract_reasoning(r) == "my thoughts" + + +def test_extract_reasoning_from_legacy_langchain_additional_kwargs(): + # A langchain AIMessage has no .reasoning_content attr; it lives in + # additional_kwargs. The dual-compat reader still finds it. + legacy = SimpleNamespace(additional_kwargs={"reasoning_content": "legacy rc"}) + assert _extract_reasoning(legacy) == "legacy rc" + + +def test_extract_reasoning_absent_returns_empty(): + assert _extract_reasoning(LLMResponse(content="ans")) == "" + assert _extract_reasoning(SimpleNamespace()) == "" + + +# ── DefaultThinkingParser on native LLMResponse ───────────────────────────── + + +def test_parser_reasoning_content_format_native(): + parser = DefaultThinkingParser() + r = LLMResponse(content="the answer", reasoning_content="deep thought") + res = parser.extract(r, _profile("reasoning_content")) + assert res.thinking == "deep thought" + assert res.visible_content == "the answer" + + +def test_parser_tag_format_prefers_typed_reasoning_channel(): + parser = DefaultThinkingParser() + # typed reasoning channel present -> used directly, content stays visible + r = LLMResponse(content="visible reply", reasoning_content="typed rc") + res = parser.extract(r, _profile("tag")) + assert res.thinking == "typed rc" + assert res.visible_content == "visible reply" + + +def test_parser_tag_format_falls_back_to_regex(): + parser = DefaultThinkingParser() + r = LLMResponse(content="inlineanswer") + res = parser.extract(r, _profile("tag")) + assert res.thinking == "inline" + assert res.visible_content == "answer" + + +# ── NativeMessageNormalizer: the leak guard ───────────────────────────────── + + +def test_to_history_does_not_leak_reasoning_content_onto_wire(): + norm = NativeMessageNormalizer() + r = LLMResponse(content="visible", reasoning_content="SECRET reasoning") + tr = ThinkingResult(thinking="SECRET reasoning", visible_content="visible") + msg = norm.to_history(r, tr, HistoryPolicy(thinking_in_history=False)) + # the PR #209 guard — reasoning never serialised onto the wire message + assert "reasoning_content" not in msg + assert msg == {"content": "visible", "role": "assistant"} + + +def test_to_history_emits_wire_tool_calls(): + norm = NativeMessageNormalizer() + r = LLMResponse(content="") + tr = ThinkingResult( + thinking="", visible_content="", + tool_calls=[{"name": "search", "args": {"q": "x"}, "id": "c1"}], + ) + msg = norm.to_history(r, tr, HistoryPolicy()) + assert msg["tool_calls"] == [{ + "type": "function", "id": "c1", + "function": {"name": "search", "arguments": '{"q": "x"}'}, + }] + assert list(msg["tool_calls"][0].keys()) == ["type", "id", "function"] + assert "reasoning_content" not in msg + + +def test_to_history_thinking_in_history_snapshots_full_content(): + norm = NativeMessageNormalizer() + r = LLMResponse(content="xanswer") + tr = ThinkingResult(thinking="x", visible_content="answer") + msg = norm.to_history( + r, tr, HistoryPolicy(thinking_in_history=True), thinking_format="tag" + ) + assert msg["content"] == "x\nanswer" + + +# ── _to_openai_tool_calls wire conversion ─────────────────────────────────── + + +def test_to_openai_tool_calls_parsed_to_wire(): + out = _to_openai_tool_calls([{"name": "f", "args": {"x": 1}, "id": "c1"}]) + assert out == [{"type": "function", "id": "c1", + "function": {"name": "f", "arguments": '{"x": 1}'}}] + + +def test_to_openai_tool_calls_passthrough_already_wire(): + wire = [{"type": "function", "id": "c1", + "function": {"name": "f", "arguments": "{}"}}] + assert _to_openai_tool_calls(wire) == wire + + +def test_to_openai_tool_calls_string_args_kept_verbatim(): + out = _to_openai_tool_calls([{"name": "f", "args": '{"x":1}', "id": "c1"}]) + assert out[0]["function"]["arguments"] == '{"x":1}' + + +def test_to_openai_tool_calls_repairs_empty_wire_arguments(): + wire = [{ + "type": "function", + "id": "c1", + "function": {"name": "bfunction", "arguments": ""}, + }] + + out = _to_openai_tool_calls(wire) + + assert out[0]["function"]["arguments"] == "{}" + + +def test_to_openai_tool_calls_repairs_truncated_parsed_arguments(): + parsed = [{"name": "bash", "args": '{"command":', "id": "c1"}] + + out = _to_openai_tool_calls(parsed) + + assert out[0]["function"]["arguments"] == "{}" + + +def test_to_openai_tool_calls_rejects_non_object_arguments(): + parsed = [{"name": "bash", "args": '["unexpected"]', "id": "c1"}] + + out = _to_openai_tool_calls(parsed) + + assert out[0]["function"]["arguments"] == "{}" + + +def test_to_openai_tool_calls_empty(): + assert _to_openai_tool_calls([]) == [] + + +# ── format-aware reasoning round-trip (replaces _ReasoningChatOpenAI) ──────── + + +def test_to_history_tag_format_inlines_reasoning_into_content(): + norm = NativeMessageNormalizer() + r = LLMResponse(content="answer", reasoning_content="my reasoning") + tr = ThinkingResult(thinking="my reasoning", visible_content="answer") + msg = norm.to_history( + r, tr, HistoryPolicy(thinking_in_history=True), thinking_format="tag" + ) + # SGLang/Qwen: reasoning inlined into content; NO bare wire field. + assert msg["content"] == "my reasoning\nanswer" + assert "reasoning_content" not in msg + + +def test_to_history_reasoning_content_format_keeps_field(): + norm = NativeMessageNormalizer() + r = LLMResponse(content="answer", reasoning_content="deep") + tr = ThinkingResult(thinking="deep", visible_content="answer") + msg = norm.to_history( + r, tr, HistoryPolicy(thinking_in_history=True), + thinking_format="reasoning_content", + ) + # DeepSeek/o-series: content untouched, reasoning_content kept on the wire. + assert msg["content"] == "answer" + assert msg["reasoning_content"] == "deep" + + +def test_to_history_none_format_drops_reasoning(): + norm = NativeMessageNormalizer() + r = LLMResponse(content="answer", reasoning_content="secret") + tr = ThinkingResult(thinking="secret", visible_content="answer") + msg = norm.to_history(r, tr, HistoryPolicy(), thinking_format="none") + assert msg == {"content": "answer", "role": "assistant"} + assert "reasoning_content" not in msg + + +def test_to_history_tag_escapes_nested_close_tag(): + norm = NativeMessageNormalizer() + r = LLMResponse(content="a", reasoning_content="x y") + tr = ThinkingResult(thinking="x y", visible_content="a") + msg = norm.to_history( + r, tr, HistoryPolicy(thinking_in_history=True), thinking_format="tag" + ) + # nested close-tag escaped so it can't terminate the wrapper early + assert "" in msg["content"] + assert msg["content"].count("") == 1 diff --git a/tests/test_multi_format_tool_call_parser.py b/tests/test_multi_format_tool_call_parser.py new file mode 100644 index 0000000..0290990 --- /dev/null +++ b/tests/test_multi_format_tool_call_parser.py @@ -0,0 +1,299 @@ +"""Tests for MultiFormatToolCallParser — Qwen/Seed XML fallbacks. + +The parser is a superset of DefaultToolCallParser. All existing +DefaultToolCallParser behavior is preserved; these tests cover only the +new XML fallback paths and guard the priority invariants. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from agent_core.runtime.loop.tool_call_parser import ( + DefaultToolCallParser, + MultiFormatToolCallParser, + ToolCallParser, +) + + +def _make_response( + *, + content: str | list | None = None, + tool_calls: list[dict] | None = None, +) -> MagicMock: + mock = MagicMock() + mock.content = content if content is not None else "" + mock.tool_calls = tool_calls if tool_calls is not None else [] + return mock + + +_ALL_TOOLS: set[str] = { + "web_search", "web_fetch", "bash", "delegate_subtask", + "create_subagent", "assign_task", "run_python_code", +} + + +class TestMultiFormatPriority: + def test_native_still_wins(self): + """Native tool_calls take priority over any XML in content.""" + parser = MultiFormatToolCallParser() + response = _make_response( + tool_calls=[{"name": "web_search", "args": {"query": "x"}, "id": "n1"}], + content='ls', + ) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["name"] == "web_search" + + def test_json_fallback_still_works(self): + """Existing JSON fallback inherited from base parser still works.""" + parser = MultiFormatToolCallParser() + response = _make_response( + content='{"tool": "web_search", "args": {"query": "hi"}}', + ) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["name"] == "web_search" + assert result[0]["args"] == {"query": "hi"} + + def test_native_with_all_unknown_tools_does_not_fall_back(self): + """If native had only unknown tools, don't second-guess with XML — the + unknown native call is returned as-is (so the executor can error on it), + and the XML web_search in content is NOT used.""" + parser = MultiFormatToolCallParser() + response = _make_response( + tool_calls=[{"name": "unknown_tool", "args": {}, "id": "n1"}], + content='x', + ) + result = parser.parse(response, _ALL_TOOLS) + assert [tc["name"] for tc in result] == ["unknown_tool"] + + +class TestQwenXmlFallback: + def test_qwen_single_call(self): + parser = MultiFormatToolCallParser() + response = _make_response(content=( + '' + 'NVIDIA H100' + '5' + '' + )) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["name"] == "web_search" + assert result[0]["args"]["query"] == "NVIDIA H100" + assert result[0]["args"]["num"] == 5 # coerced from JSON-parseable string + + def test_qwen_multiple_calls(self): + parser = MultiFormatToolCallParser() + content = ( + '' + 'AMD' + '' + 'thinking text\n' + '' + 'https://example.com' + '' + ) + response = _make_response(content=content) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 2 + names = {tc["name"] for tc in result} + assert names == {"web_search", "web_fetch"} + + def test_qwen_unknown_tool_filtered(self): + parser = MultiFormatToolCallParser() + response = _make_response(content=( + '' + '1' + '' + )) + result = parser.parse(response, _ALL_TOOLS) + assert result == [] + + def test_qwen_emits_langchain_shape(self): + parser = MultiFormatToolCallParser() + response = _make_response(content=( + '' + 'echo hi' + '' + )) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + tc = result[0] + assert set(tc.keys()) >= {"name", "args", "id"} + assert isinstance(tc["args"], dict) + + +class TestSeedXmlFallback: + def test_seed_single_call(self): + parser = MultiFormatToolCallParser() + response = _make_response(content=( + '' + 'AMD MI300X' + '3' + '' + )) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["name"] == "web_search" + assert result[0]["args"]["query"] == "AMD MI300X" + assert result[0]["args"]["num"] == 3 + + def test_seed_nested_json_param(self): + """Seed parameters containing JSON lists are decoded to real lists.""" + parser = MultiFormatToolCallParser() + response = _make_response(content=( + '' + '["foo", "bar"]' + '' + )) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["args"]["q"] == ["foo", "bar"] + + def test_seed_unknown_tool_filtered(self): + parser = MultiFormatToolCallParser() + response = _make_response(content=( + '' + '1' + '' + )) + result = parser.parse(response, _ALL_TOOLS) + assert result == [] + + def test_seed_with_tag_hint(self): + """Seed-flavored content with hint also triggers fallback.""" + parser = MultiFormatToolCallParser() + content = ( + '' + '' + 'pwd' + '' + '' + ) + response = _make_response(content=content) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["name"] == "bash" + + +class TestMalformedInput: + def test_empty_response(self): + parser = MultiFormatToolCallParser() + result = parser.parse(_make_response(content="", tool_calls=[]), _ALL_TOOLS) + assert result == [] + + def test_plain_text_no_xml(self): + parser = MultiFormatToolCallParser() + result = parser.parse( + _make_response(content="Just an answer, no tool call."), _ALL_TOOLS, + ) + assert result == [] + + def test_partial_qwen_tag_is_ignored(self): + """Unclosed without matching → no parse.""" + parser = MultiFormatToolCallParser() + result = parser.parse( + _make_response(content="incomplete"), + _ALL_TOOLS, + ) + assert result == [] + + +class TestDanglingJsonRecovery: + """Recover ``{...}`` truncated mid-stream by ``max_tokens``. + + Real failure mode observed on qwen3.5-397b heavy_mode_smoke when a + long ``assign_task`` payload pushed the closing ```` past + the response cap. JSON body is preserved up to the close brace; only + the tag is missing. Recovery extracts the balanced JSON object. + """ + + def test_recovers_complete_json_missing_close_tag(self): + parser = MultiFormatToolCallParser() + result = parser.parse( + _make_response(content=( + '{"tool": "web_search", "args": ' + '{"query": "NVIDIA H100"}}' + )), + _ALL_TOOLS, + ) + assert len(result) == 1 + assert result[0]["name"] == "web_search" + assert result[0]["args"] == {"query": "NVIDIA H100"} + + def test_recovers_nested_json_missing_close_tag(self): + parser = MultiFormatToolCallParser() + result = parser.parse( + _make_response(content=( + '{"tool": "assign_task", "args": ' + '{"tasks": [{"agent": "researcher", "prompt": "find X"}]}}' + )), + _ALL_TOOLS, + ) + assert len(result) == 1 + assert result[0]["name"] == "assign_task" + assert result[0]["args"]["tasks"][0]["agent"] == "researcher" + + def test_does_not_recover_truncation_inside_string(self): + """Truncation mid-string is unrecoverable — string never closes.""" + parser = MultiFormatToolCallParser() + result = parser.parse( + _make_response(content=( + '{"tool": "assign_task", "args": ' + '{"tasks": [{"agent": "x", "prompt": "Research the 96th Aca' + )), + _ALL_TOOLS, + ) + assert result == [] + + def test_does_not_recover_unknown_tool(self): + parser = MultiFormatToolCallParser() + result = parser.parse( + _make_response(content=( + '{"tool": "nope_not_a_tool", "args": {}}' + )), + _ALL_TOOLS, + ) + assert result == [] + + def test_skipped_when_close_tag_present(self): + """Closed ``...`` goes through the regular + JSON path, not the dangling recovery. Guard against + double-parsing.""" + parser = MultiFormatToolCallParser() + result = parser.parse( + _make_response(content=( + '{"tool": "web_search", "args": ' + '{"query": "x"}}' + )), + _ALL_TOOLS, + ) + assert len(result) == 1 + # ID is set by the JSON path (no ``id`` field) — dangling + # recovery would have used ``dangling_tc_0``. + assert result[0].get("id") != "dangling_tc_0" + + def test_native_still_beats_dangling(self): + parser = MultiFormatToolCallParser() + result = parser.parse( + _make_response( + tool_calls=[ + {"name": "web_fetch", "args": {"url": "u"}, "id": "n1"}, + ], + content='{"tool": "web_search", "args": {"query": "x"}}', + ), + _ALL_TOOLS, + ) + assert len(result) == 1 + assert result[0]["name"] == "web_fetch" + + +class TestProtocolCompliance: + def test_is_tool_call_parser(self): + assert isinstance(MultiFormatToolCallParser(), ToolCallParser) + + def test_is_subclass_of_default(self): + assert issubclass(MultiFormatToolCallParser, DefaultToolCallParser) + diff --git a/tests/test_thinking_history_policy.py b/tests/test_thinking_history_policy.py new file mode 100644 index 0000000..587c486 --- /dev/null +++ b/tests/test_thinking_history_policy.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import pytest + +from agent_core.llm import LLMResponse +from agent_core.runtime.loop import context_budget +from agent_core.runtime.loop.model_profile import ( + DefaultThinkingParser, + HistoryPolicy, + ModelProfile, + NativeMessageNormalizer, + resolve_history_policy, +) + + +def _history( + response: LLMResponse, + *, + thinking_format: str = "tag", + policy: HistoryPolicy, +) -> dict: + profile = ModelProfile( + model_id="test-model", + provider="test", + thinking_format=thinking_format, + ) + parsed = DefaultThinkingParser().extract(response, profile) + return NativeMessageNormalizer().to_history( + response, + parsed, + policy, + thinking_format, + ) + + +@pytest.mark.parametrize( + ("config", "enabled", "cap"), + [ + ({}, False, None), + ({"thinking_in_history": False}, False, None), + ({"thinking_in_history": True}, True, None), + ({"thinking_in_history": True, "thinking_history_max_tokens": 0}, True, None), + ({"thinking_history_max_tokens": 8192}, True, 8192), + ({"thinking_history_max_tokens": 8192.0}, True, 8192), + # A quoted YAML "false" must disable, not read as a truthy string. + ({"thinking_in_history": "false"}, False, None), + ({"thinking_in_history": "true"}, True, None), + ({"thinking_in_history": "no", "thinking_history_max_tokens": 8192}, False, None), + ( + {"thinking_in_history": True, "thinking_history_max_tokens": "8192"}, + True, + 8192, + ), + ( + {"thinking_in_history": False, "thinking_history_max_tokens": 8192}, + False, + None, + ), + ], +) +def test_resolve_history_policy( + config: dict, + enabled: bool, + cap: int | None, +) -> None: + policy = resolve_history_policy(config) + + assert policy.thinking_in_history is enabled + assert policy.thinking_history_max_tokens == cap + + +@pytest.mark.parametrize("cap", [-1, "many", [], True, False, 8192.5, {}]) +def test_resolve_history_policy_rejects_invalid_cap(cap: object) -> None: + with pytest.raises(ValueError, match="thinking_history_max_tokens"): + resolve_history_policy({"thinking_history_max_tokens": cap}) + + +@pytest.mark.parametrize("flag", ["maybe", 3, []]) +def test_resolve_history_policy_rejects_invalid_flag(flag: object) -> None: + with pytest.raises(ValueError, match="thinking_in_history"): + resolve_history_policy({"thinking_in_history": flag}) + + +def test_disabled_policy_omits_tag_reasoning() -> None: + history = _history( + LLMResponse(content="answer", reasoning_content="private reasoning"), + policy=HistoryPolicy(thinking_in_history=False), + ) + + assert history == {"content": "answer", "role": "assistant"} + + +def test_enabled_policy_keeps_full_tag_reasoning() -> None: + history = _history( + LLMResponse(content="answer", reasoning_content="full reasoning"), + policy=HistoryPolicy(thinking_in_history=True), + ) + + assert history == { + "content": "full reasoning\nanswer", + "role": "assistant", + } + + +def test_capped_policy_keeps_reasoning_tail(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(context_budget, "estimate_tokens", len) + + history = _history( + LLMResponse(content="answer", reasoning_content="discard-KEEP"), + policy=HistoryPolicy( + thinking_in_history=True, + thinking_history_max_tokens=4, + ), + ) + + assert history == { + "content": "KEEP\nanswer", + "role": "assistant", + } + + +def test_inline_tag_reasoning_is_not_duplicated() -> None: + history = _history( + LLMResponse(content="reasoning\nanswer"), + policy=HistoryPolicy(thinking_in_history=True), + ) + + assert history == { + "content": "reasoning\nanswer", + "role": "assistant", + } + + +def test_disabled_policy_omits_reasoning_content_field() -> None: + history = _history( + LLMResponse(content="answer", reasoning_content="private reasoning"), + thinking_format="reasoning_content", + policy=HistoryPolicy(thinking_in_history=False), + ) + + assert history == {"content": "answer", "role": "assistant"} + + +def test_signed_content_blocks_ignore_text_history_policy() -> None: + blocks = [ + {"type": "thinking", "thinking": "reasoning", "signature": "signed"}, + {"type": "text", "text": "answer"}, + ] + history = _history( + LLMResponse(content=blocks), + thinking_format="content_block", + policy=HistoryPolicy( + thinking_in_history=False, + thinking_history_max_tokens=1, + ), + ) + + assert history == {"content": blocks, "role": "assistant"} + + +def test_multiple_inline_think_blocks_are_all_retained() -> None: + """A turn that reopens must not lose the later blocks. + + ``to_history`` rebuilds the message from the parsed result, so any block the + parser drops is reasoning gone from history. The visible text around a block + must also stay separated ("ax\nb" must not become "ab"). + """ + history = _history( + LLMResponse(content="A\nstep1B\nstep2"), + policy=HistoryPolicy(thinking_in_history=True), + ) + + assert history == { + "content": "A\nB\nstep1\nstep2", + "role": "assistant", + } + + +def test_multiple_inline_think_blocks_are_omitted_when_disabled() -> None: + history = _history( + LLMResponse(content="A\nstep1B\nstep2"), + policy=HistoryPolicy(thinking_in_history=False), + ) + + assert history == {"content": "step1\nstep2", "role": "assistant"} + + +def test_capped_policy_applies_to_reasoning_content_format( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(context_budget, "estimate_tokens", len) + + history = _history( + LLMResponse(content="answer", reasoning_content="discard-KEEP"), + thinking_format="reasoning_content", + policy=HistoryPolicy( + thinking_in_history=True, + thinking_history_max_tokens=4, + ), + ) + + assert history == { + "content": "answer", + "role": "assistant", + "reasoning_content": "KEEP", + } + + diff --git a/tests/test_tool_call_parser_generic.py b/tests/test_tool_call_parser_generic.py new file mode 100644 index 0000000..0b83e1b --- /dev/null +++ b/tests/test_tool_call_parser_generic.py @@ -0,0 +1,228 @@ +"""Tests for the generic ToolCallParser (native FC + JSON fallback).""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from agent_core.runtime.loop.tool_call_parser import ( + DefaultToolCallParser, + ToolCallParser, +) + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +def _make_response( + *, + content: str | list | None = None, + tool_calls: list[dict] | None = None, +) -> MagicMock: + """Build a fake AIMessage-style response object.""" + mock = MagicMock() + mock.content = content if content is not None else "" + mock.tool_calls = tool_calls if tool_calls is not None else [] + return mock + + +_ALL_TOOLS: set[str] = { + "web_search", + "web_fetch", + "bash", + "delegate_subtask", + "use_mcp_tool", +} + + +# ── Tests ──────────────────────────────────────────────────────────────────── + + +class TestNativeFunctionCalling: + def test_native_fc_parsed(self): + """Native tool_calls are returned correctly.""" + parser = DefaultToolCallParser() + response = _make_response( + tool_calls=[ + {"name": "web_search", "args": {"query": "AI chips 2025"}, "id": "tc1"} + ] + ) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["name"] == "web_search" + assert result[0]["args"] == {"query": "AI chips 2025"} + + def test_native_fc_keeps_unknown_tools(self): + """Native tool calls are passed through even when the name is unknown, + so the executor can return an explicit "unknown tool" error the model + can recover from (silently dropping them stranded the turn as empty).""" + parser = DefaultToolCallParser(keep_unknown_native_companions=True) + response = _make_response( + tool_calls=[ + {"name": "web_search", "args": {"query": "test"}, "id": "tc1"}, + {"name": "unknown_tool", "args": {}, "id": "tc2"}, + ] + ) + result = parser.parse(response, _ALL_TOOLS) + assert [tc["name"] for tc in result] == ["web_search", "unknown_tool"] + + def test_native_fc_takes_priority(self): + """When both native tool_calls and JSON text are present, native wins.""" + parser = DefaultToolCallParser() + response = _make_response( + content='{"tool": "bash", "args": {"command": "ls"}}', + tool_calls=[ + {"name": "web_search", "args": {"query": "from native"}, "id": "tc1"} + ], + ) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["name"] == "web_search" + # bash from JSON text is NOT returned + assert all(tc["name"] != "bash" for tc in result) + + +class TestJsonFallback: + def test_json_fallback_single(self): + """A single block is parsed correctly.""" + parser = DefaultToolCallParser() + response = _make_response( + content='{"tool": "web_search", "args": {"query": "NVIDIA H100"}}', + ) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["name"] == "web_search" + assert result[0]["args"]["query"] == "NVIDIA H100" + + def test_json_fallback_multiple(self): + """Two blocks in the same response are both parsed.""" + parser = DefaultToolCallParser() + content = ( + "" + '{"tool": "web_search", "args": {"query": "NVIDIA H100"}}' + "" + "\nSome thinking text in between.\n" + "" + '{"tool": "web_fetch", "args": {"url": "https://example.com"}}' + "" + ) + response = _make_response(content=content) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 2 + names = {tc["name"] for tc in result} + assert names == {"web_search", "web_fetch"} + + def test_json_fallback_filters_unknown(self): + """Unknown tool names in JSON blocks are skipped.""" + parser = DefaultToolCallParser() + content = ( + '{"tool": "web_search", "args": {"query": "test"}}' + '{"tool": "ghost_tool", "args": {}}' + ) + response = _make_response(content=content) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["name"] == "web_search" + + def test_malformed_json_skipped(self): + """Broken JSON inside → empty list, no exception raised.""" + parser = DefaultToolCallParser() + response = _make_response( + content="{bad json here!!!!}", + ) + result = parser.parse(response, _ALL_TOOLS) + assert result == [] + + def test_list_content_handled(self): + """Anthropic-style list content (content=[{type:text, text:...}]) is handled.""" + parser = DefaultToolCallParser() + # Simulate Anthropic content format: list of blocks + content_blocks = [ + {"type": "text", "text": "Let me search for that."}, + { + "type": "text", + "text": '{"tool": "web_search", "args": {"query": "AI"}}', + }, + ] + response = _make_response(content=content_blocks) + result = parser.parse(response, _ALL_TOOLS) + assert len(result) == 1 + assert result[0]["name"] == "web_search" + + +class TestMCPToolFallback: + def test_use_mcp_tool_xml_fallback(self): + parser = DefaultToolCallParser() + response = _make_response( + content=( + "" + "deepwiki" + "read_repo" + '{"repo":"modelcontextprotocol/servers"}' + "" + ), + ) + + result = parser.parse(response, _ALL_TOOLS) + + assert result == [{ + "name": "use_mcp_tool", + "args": { + "server_name": "deepwiki", + "tool_name": "read_repo", + "arguments": {"repo": "modelcontextprotocol/servers"}, + }, + "id": "mcp_tc_0", + }] + + def test_use_mcp_tool_requires_router_tool_to_be_allowed(self): + parser = DefaultToolCallParser() + response = _make_response( + content=( + "" + "deepwiki" + "read_repo" + "{}" + "" + ), + ) + + result = parser.parse(response, {"web_search"}) + + assert result == [] + + +class TestEdgeCases: + def test_empty_response(self): + """No content and no tool_calls → empty list.""" + parser = DefaultToolCallParser() + response = _make_response(content="", tool_calls=[]) + result = parser.parse(response, _ALL_TOOLS) + assert result == [] + + def test_empty_tool_names_set_keeps_native_calls(self): + """Native calls are passed through even with an empty tool_names set — + the executor surfaces the "unknown tool" error (text/JSON parsing still + filters, but native intent is never silently dropped).""" + parser = DefaultToolCallParser() + response = _make_response( + tool_calls=[{"name": "web_search", "args": {}, "id": "tc1"}] + ) + result = parser.parse(response, set()) + assert [tc["name"] for tc in result] == ["web_search"] + + def test_content_with_no_tool_call_tags(self): + """Plain text without tags → empty list.""" + parser = DefaultToolCallParser() + response = _make_response(content="This is just a text answer, no tool call.") + result = parser.parse(response, _ALL_TOOLS) + assert result == [] + + +class TestProtocolCompliance: + def test_protocol_compliance(self): + """DefaultToolCallParser satisfies the ToolCallParser Protocol.""" + parser = DefaultToolCallParser() + assert isinstance(parser, ToolCallParser) + + def test_protocol_has_parse_method(self): + """ToolCallParser Protocol requires a parse method.""" + assert hasattr(ToolCallParser, "parse") diff --git a/tests/test_tool_call_parser_mixed_native.py b/tests/test_tool_call_parser_mixed_native.py new file mode 100644 index 0000000..561c636 --- /dev/null +++ b/tests/test_tool_call_parser_mixed_native.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from agent_core.llm import LLMResponse +from agent_core.runtime.loop.agent_loop import _answer_dropped_tool_calls +from agent_core.runtime.loop.model_profile import ( + DefaultThinkingParser, + HistoryPolicy, + ModelProfile, + NativeMessageNormalizer, +) +from agent_core.runtime.loop.tool_call_parser import DefaultToolCallParser + + +def _native_call(name: str, call_id: str) -> dict: + return { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": "{}"}, + } + + +def test_native_parser_drops_unknown_companion_when_known_call_exists() -> None: + response = LLMResponse(tool_calls=[ + _native_call("glob_search", "known"), + _native_call("read_file_stub", "unknown"), + ]) + + calls = DefaultToolCallParser().parse(response, {"glob_search"}) + + assert [call["name"] for call in calls] == ["glob_search"] + + +def test_native_parser_keeps_all_unknown_calls_for_explicit_correction() -> None: + response = LLMResponse(tool_calls=[ + _native_call("read_file_stub", "unknown"), + ]) + + calls = DefaultToolCallParser().parse(response, {"glob_search"}) + + assert [call["name"] for call in calls] == ["read_file_stub"] + + +def _assistant_turn(response: LLMResponse) -> dict: + """The history message the agent loop writes before it parses.""" + profile = ModelProfile(model_id="gpt-4o", provider="openai") + thinking = DefaultThinkingParser().extract(response, profile) + return NativeMessageNormalizer().to_history( + response, thinking, HistoryPolicy(), profile.thinking_format, + ) + + +def test_a_dropped_companion_call_still_gets_a_tool_response() -> None: + """An orphan ``tool_call_id`` is a hard HTTP 400 on Azure and others. + + The assistant history message is written from the raw response, so it keeps + the dropped call's id; without a matching tool message the very next + request is malformed and the run dies instead of recovering. + """ + response = LLMResponse(tool_calls=[ + _native_call("glob_search", "known"), + _native_call("read_file_stub", "unknown"), + ]) + tool_names = {"glob_search"} + history = _assistant_turn(response) + messages = [history] + parsed = DefaultToolCallParser().parse(response, tool_names) + + _answer_dropped_tool_calls(messages, history, parsed, tool_names) + + executed = {call["id"] for call in parsed} + answered = { + message["tool_call_id"] for message in messages + if message.get("role") == "tool" + } + recorded = {call["id"] for call in history["tool_calls"]} + assert recorded <= executed | answered + assert answered == {"unknown"} + # The correction the executor would have produced is not lost either, so + # the model learns the name is wrong instead of reissuing it every turn. + assert "read_file_stub" in messages[-1]["content"] + + +def test_executed_calls_are_left_for_the_executor_to_answer() -> None: + response = LLMResponse(tool_calls=[_native_call("glob_search", "known")]) + tool_names = {"glob_search"} + history = _assistant_turn(response) + messages = [history] + parsed = DefaultToolCallParser().parse(response, tool_names) + + _answer_dropped_tool_calls(messages, history, parsed, tool_names) + + assert messages == [history] + diff --git a/tests/test_tool_exec.py b/tests/test_tool_exec.py new file mode 100644 index 0000000..3e9b65c --- /dev/null +++ b/tests/test_tool_exec.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import asyncio +from contextlib import contextmanager +from dataclasses import replace +from typing import Any + +import pytest + +from agent_core.runtime.loop.tool_exec import ToolExecutionHooks, execute_tools + + +class FakeTool: + def __init__(self, name: str, result: Any = "ok", *, delay: float = 0) -> None: + self.name = name + self.result = result + self.delay = delay + + async def ainvoke(self, args: dict[str, Any]) -> Any: + await asyncio.sleep(self.delay) + if isinstance(self.result, Exception): + raise self.result + return self.result + + +@pytest.mark.asyncio +async def test_parallel_batch_converts_success_unknown_and_failure() -> None: + calls = [ + {"name": "ok", "args": {"x": 1}, "id": "a"}, + {"name": "missing", "args": {}, "id": "b"}, + {"name": "bad", "args": {}, "id": "c"}, + ] + results = await execute_tools( + calls, + {"ok": FakeTool("ok", "done"), "bad": FakeTool("bad", ValueError("boom"))}, + timeout=1, + turn=2, + count_offset=4, + ) + + assert [result.tool_call_id for result in results] == ["a", "b", "c"] + assert results[0].result == "done" + assert results[0].is_error is False + assert "unknown tool 'missing'" in results[1].result + assert "bad, ok" in results[1].result + assert results[1].is_error is True + assert "boom" in results[2].result + + +@pytest.mark.asyncio +async def test_host_hooks_own_timeout_context_and_result_policies() -> None: + events: list[str] = [] + + @contextmanager + def scope(call: dict[str, Any], timeout: float): + events.append(f"enter:{call['id']}:{timeout}") + try: + yield + finally: + events.append("exit") + + async def await_call(awaitable, name, args, timeout): + events.append(f"await:{name}:{args['x']}:{timeout}") + return await awaitable + + hooks = ToolExecutionHooks( + resolve_timeout=lambda _name, _args, configured: configured + 7, + await_call=await_call, + call_scope=scope, + on_call=lambda name: events.append(f"meter:{name}"), + transform_result=lambda name, value: f"{name}={value}", + transform_batch=lambda results: [replace(results[0], result="batch")], + ) + results = await execute_tools( + [{"name": "echo", "args": {"x": 3}, "id": "tc"}], + {"echo": FakeTool("echo", "value")}, + timeout=5, + turn=1, + count_offset=0, + hooks=hooks, + ) + + assert results[0].result == "batch" + assert events == ["meter:echo", "enter:tc:12", "await:echo:3:12", "exit"] + + +@pytest.mark.asyncio +async def test_fan_in_interrupt_cancels_invocation_and_returns_result() -> None: + async def interrupt(_call: dict[str, Any]) -> bool: + await asyncio.sleep(0) + return True + + results = await execute_tools( + [{"name": "collect_reports", "args": {}, "id": "fan"}], + {"collect_reports": FakeTool("collect_reports", delay=10)}, + timeout=20, + turn=1, + count_offset=0, + interrupt_waiter=interrupt, + ) + + assert results[0].interrupted is True + assert results[0].result.startswith("[interrupted]") + + +@pytest.mark.asyncio +async def test_external_cancellation_is_not_converted_to_tool_error() -> None: + task = asyncio.create_task( + execute_tools( + [{"name": "slow", "args": {}, "id": "slow"}], + {"slow": FakeTool("slow", delay=10)}, + timeout=20, + turn=1, + count_offset=0, + ) + ) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task From c25195e43099f22a2eeb05390d5ae5fe15dff59c Mon Sep 17 00:00:00 2001 From: zhanghanduo Date: Tue, 1 Sep 2026 11:14:21 +0800 Subject: [PATCH 2/2] fix(loop): close the tool-call answering invariant and isolate host hooks Review of the extracted agent-loop engine found the "every dropped assistant tool_call_id gets an answer" invariant held on the normal paths but not on observer-intervention paths, plus several hook boundaries where a host mistake became an engine failure. Correctness: - tool_exec: an interrupt waiter resolving to False (no observer asked to interrupt) or raising cancelled the fan-in tool anyway and reported its collected work as "interrupted by a follow-up user message". Only a genuine request now cancels, and a tool that finished in the same wake-up keeps its result. - agent_loop: skip_tool_execution, and continue_to_next_turn without pop_last_message, left the assistant tool_calls unanswered -> hard 400 on the next request. Both now synthesise answers, inserted beneath the assistant message so an injected user message cannot split the pair. - agent_loop: synthetic and executed tool_call_ids were numbered off different lists and collided once an observer skipped an earlier text-parsed call. Ids are now assigned once, before the split. - tool_exec: a non-positive result cap (the usual "unlimited" sentinel) sliced from the end and silently dropped every result's tail. - model_profile: a non-string `format` in the registry raised TypeError and aborted the whole load; a missing PyYAML disabled registry inference in total silence. Both are now per-row warnings / a loud error, and pyyaml is declared as a `registry` extra. - model_profile: tag-format models that fill the typed reasoning channel AND leave inline tags duplicated the reasoning in history and kept it there with thinking_in_history=False. - agent_loop: the zero-filled usage fallback overwrote the previous turn's real token counts, so the context-overflow guard computed its estimate off 0 and never fired. - agent_loop: _pop_last_assistant_turn was a silent no-op once a user message had been injected mid-turn, letting a rollback plus continue_to_next_turn replay the same turn until the attempt buffer ran out. Boundaries: - tool_exec: resolve_timeout, on_call, transform_batch and the result formatters ran outside the guarded region, so one raising metering hook killed the whole batch and the surrounding loop. They now fall back to core behavior with a warning. - tool_exec: the fan-in tool names eligible for interruption are now a ToolExecutionHooks field instead of hardcoded product names. - agent_loop: bind_session and sticky_session_enabled both owned session affinity and resolved silently; bind_session now explicitly wins, warns, and is typed LLMClient -> LLMClient rather than Any -> Any. - agent_loop: HistoryPolicy.tool_result_max_chars was read by nobody. It now applies when the caller actually supplies a policy (never by dataclass default, which would newly truncate existing runs). - model_profile: MessageNormalizer.to_history declared three parameters while the engine calls it with four, so the Protocol was unimplementable. 21 regression tests, each verified to fail before its fix. 404 passed, ruff clean, pyright strict clean. Co-Authored-By: Claude Opus 5 (1M context) --- agent_core/runtime/loop/agent_loop.py | 174 +++++++- agent_core/runtime/loop/model_profile.py | 47 +- agent_core/runtime/loop/tool_exec.py | 147 ++++++- docs/agent-loop-boundary.md | 15 +- pyproject.toml | 5 + tests/test_agent_loop_engine.py | 527 ++++++++++++++++++++++- tests/test_model_registry_loading.py | 103 +++++ tests/test_thinking_history_policy.py | 38 ++ tests/test_tool_exec.py | 140 ++++++ uv.lock | 54 ++- 10 files changed, 1203 insertions(+), 47 deletions(-) create mode 100644 tests/test_model_registry_loading.py diff --git a/agent_core/runtime/loop/agent_loop.py b/agent_core/runtime/loop/agent_loop.py index 06b4675..e4646f6 100644 --- a/agent_core/runtime/loop/agent_loop.py +++ b/agent_core/runtime/loop/agent_loop.py @@ -136,8 +136,13 @@ async def _no_cancel_cleanup() -> None: class AgentLoopHooks: """Product-owned runtime state injected around the shared loop engine.""" + # Session affinity has exactly one owner per run. ``bind_session``, when + # supplied, fully REPLACES the built-in ``bind_session_id`` binding and + # therefore also replaces ``sticky_session_enabled`` -- the engine does not + # consult the flag a host binding is free to ignore. Supplying both is a + # configuration error and is reported once at loop start. sticky_session_enabled: Callable[[], bool] | None = None - bind_session: Callable[[Any, str], Any] | None = None + bind_session: Callable[[LLMClient, str], LLMClient] | None = None wall_deadline_remaining: Callable[[], float | None] = _no_deadline chain_fallback_active: Callable[[], bool] = _false enter_scope: Callable[ @@ -228,6 +233,12 @@ async def run_agent_loop( # Pin one conversation to one upstream worker when affinity is enabled. llm_session_id = cfg.llm_session_id or cfg.task_id if runtime.bind_session is not None: + if runtime.sticky_session_enabled is not None: + logger.warning( + "AgentLoopHooks got both bind_session and " + "sticky_session_enabled; bind_session owns session affinity " + "and the sticky flag is ignored" + ) llm_with_session = runtime.bind_session(llm, llm_session_id) else: llm_with_session = bind_session_id( @@ -262,6 +273,7 @@ async def run_agent_loop( normalizer, tool_map, tool_names, llm_with_session, llm_with_tools, messages, metadata, on_turn_complete, pause_check, scope=scope, runtime_hooks=runtime, + tool_result_cap=_effective_tool_result_cap(cfg, history_policy), ) except asyncio.CancelledError: # The loop task was cancelled mid-flight (wall deadline, fan-out @@ -332,6 +344,7 @@ async def _run_loop_inner( pause_check: PauseCheckHook | None = None, scope: Any = None, runtime_hooks: AgentLoopHooks | None = None, + tool_result_cap: int | None = None, ) -> AgentLoopResult: """Inner loop extracted so run_agent_loop can wrap with ExecutionScope.""" runtime = runtime_hooks or AgentLoopHooks() @@ -478,10 +491,20 @@ async def _run_loop_inner( cfg, obs, tool_map, messages, metadata, turn, total_tool_calls, ctx, parsed_calls, runtime.tool_execution, runtime.body_has_spill_reference, + result_max_chars=tool_result_cap, ) total_tool_calls += tool_calls_executed if stop_reason: break + else: + # An observer suppressed dispatch but the assistant message with + # its ``tool_calls`` is already in history; leaving those ids + # unanswered would make the next provider request invalid. + _answer_unexecuted_tool_calls( + messages, + "tool execution was suppressed for this turn; re-issue it if " + "still needed.", + ) stop_reason = _handle_context_overflow( cfg, messages, turn, last_input_tokens, last_output_tokens @@ -727,6 +750,51 @@ def _answer_dropped_tool_calls( answered.add(call_id) +def _answer_unexecuted_tool_calls(messages: list[Message], detail: str) -> None: + """Answer every ``tool_call_id`` in the last assistant message that no + tool message replies to. + + :func:`_answer_dropped_tool_calls` covers calls the engine itself refused + to dispatch, but it runs *before* execution and therefore counts the + surviving ``parsed_calls`` as answered-by-construction. When an observer + intervention keeps the assistant message yet skips execution + (``skip_tool_execution``, or ``continue_to_next_turn`` without + ``pop_last_message``), that assumption breaks and the next provider + request carries unanswered ids -- a hard 400 on OpenAI, Azure and + Anthropic alike. This closes the invariant on those paths. + + Answers are *inserted* directly beneath the assistant message rather + than appended, because a tool message separated from its call by an + injected user message is just as invalid as a missing one. + """ + + idx = len(messages) - 1 + while idx >= 0 and not is_assistant_msg(messages[idx]): + idx -= 1 + if idx < 0: + return + recorded = messages[idx].get("tool_calls") or [] + if not recorded: + return + answered = { + message.get("tool_call_id") + for message in messages[idx + 1:] + if is_tool_msg(message) + } + insert_at = idx + 1 + while insert_at < len(messages) and is_tool_msg(messages[insert_at]): + insert_at += 1 + for call in recorded: + call_id = call.get("id") + if not call_id or call_id in answered: + continue + messages.insert(insert_at, tool_msg( + f"[tool call not executed] {detail}", call_id + )) + insert_at += 1 + answered.add(call_id) + + def _pop_last_assistant_turn(messages: list[Message]) -> None: """Remove the assistant message a rollback observer rejected, in full. @@ -743,20 +811,31 @@ def _pop_last_assistant_turn(messages: list[Message]) -> None: Only this turn's tail is in scope: real tool results are appended later, in ``_execute_tool_calls``. - If no assistant message sits beneath the trailing tool messages, the - tail is not this turn's shape at all — leave history untouched rather - than popping an unrelated prefix. + The tail is not always tool messages either: an interrupt notice or an + observer injection appends a ``user`` message after the assistant one. + Stopping at the first non-tool message there made this a silent no-op, so + a rollback paired with ``continue_to_next_turn`` replayed the same content + forever until the attempt buffer ran out. Search past anything for the + assistant message, and remove it together with the tool answers that + belong to it while leaving later injections in place -- they carry + information the rejected assistant turn does not. + + If there is no assistant message at all, the tail is not this turn's + shape: leave history untouched rather than popping an unrelated prefix. """ idx = len(messages) - 1 - while idx >= 0 and is_tool_msg(messages[idx]): + while idx >= 0 and not is_assistant_msg(messages[idx]): idx -= 1 - if idx < 0 or not is_assistant_msg(messages[idx]): + if idx < 0: logger.warning( - "_pop_last_assistant_turn: no assistant message beneath the " - "trailing tool messages; leaving history unchanged" + "_pop_last_assistant_turn: no assistant message in history; " + "leaving it unchanged" ) return - del messages[idx:] + end = idx + 1 + while end < len(messages) and is_tool_msg(messages[end]): + end += 1 + del messages[idx:end] async def _process_llm_response( @@ -816,12 +895,15 @@ async def _process_llm_response( if isinstance(tc, dict) and tc.get("id") } for blocked in blocked_landing_calls: - call_id = str(blocked.get("id") or "") - if call_id and call_id in native_ids: + # Deliberately not named ``call_id``: that parameter holds the + # LLM call id used by the attempt events, and rebinding it here + # left every later read pointing at the last blocked tool call. + blocked_id = str(blocked.get("id") or "") + if blocked_id and blocked_id in native_ids: messages.append(tool_msg( "[tool call blocked] final turn accepts only " "workflow-approved landing tools; report current progress.", - call_id, + blocked_id, )) parsed_calls = allowed_calls @@ -852,8 +934,17 @@ async def _process_llm_response( "reasoning_tokens": 0, "estimated": True, } if usage: - last_input_tokens = int(usage.get("prompt_tokens", 0) or 0) - last_output_tokens = int(usage.get("completion_tokens", 0) or 0) + prompt_tokens = int(usage.get("prompt_tokens", 0) or 0) + completion_tokens = int(usage.get("completion_tokens", 0) or 0) + # The zero-filled fallback above records model identity for cost + # attribution; it carries no token counts. Letting its zeros overwrite + # the previous turn's real numbers would make the context-overflow + # guard compute an estimate of 0 and never fire, so a gateway that + # omits usage on some routes turns a clean ``context_limit_reached`` + # into a provider-side rejection. Keep the last known values instead. + if prompt_tokens or completion_tokens: + last_input_tokens = prompt_tokens + last_output_tokens = completion_tokens leaked_reasoning = extract_leaked_reasoning(response) rmd = getattr(response, "response_metadata", None) or {} @@ -887,6 +978,14 @@ async def _process_llm_response( if merged_llm.inject_messages: for msg_text in merged_llm.inject_messages: messages.append(user_msg(msg_text)) + # The turn is being replayed without executing its calls. When the + # observer also popped the assistant message this is a no-op; when it + # did not, these ids would otherwise reach the next request unanswered. + _answer_unexecuted_tool_calls( + messages, + "the turn was restarted before this call ran; re-issue it if " + "still needed.", + ) return ( parsed_calls, ctx, stop_reason, True, False, last_input_tokens, last_output_tokens, @@ -908,26 +1007,65 @@ async def _process_llm_response( ) +def _effective_tool_result_cap( + cfg: LoopConfig, policy: HistoryPolicy | None +) -> int | None: + """Resolve the ToolMessage character cap from both places it is declared. + + ``LoopConfig.tool_result_max_chars`` is the per-run override and wins when + set. ``HistoryPolicy.tool_result_max_chars`` is the *role* value (15_000, + the sub-agent number) and used to be read by nobody, so a host configuring + it saw results bounded only by the 150K spill ceiling -- ten times the + intended size entering history, which is what the overflow guard exists to + prevent. + + Only a policy the caller actually supplied is consulted. The dataclass + default is 15_000, so falling back to it for callers that pass no policy + would newly truncate results those runs previously kept in full. + """ + + if cfg.tool_result_max_chars is not None: + return cfg.tool_result_max_chars + if policy is None: + return None + cap = policy.tool_result_max_chars + return cap if cap and cap > 0 else None + + async def _execute_tool_calls( cfg: LoopConfig, obs: list, tool_map: dict[str, ToolLike], messages: list[Message], metadata: dict[str, Any], turn: int, total_tool_calls: int, ctx: TurnContext, parsed_calls: list[dict], execution_hooks: ToolExecutionHooks, body_has_spill_reference: Callable[[str], bool], + *, + result_max_chars: int | None = None, ) -> tuple[str, int]: executable: list[tuple[int, dict]] = [] synthetic: list[tuple[int, ToolResult]] = [] for idx, tc in enumerate(parsed_calls): + # Text-mode calls arrive without a provider id. Assign it here, once, + # off the ``parsed_calls`` index -- before the batch is split into + # skipped and executable halves. ``execute_tools`` numbers off its own + # sub-list, so letting it fill the gap made the two halves collide on + # the same ``tool_call_id`` whenever an observer skipped an earlier + # call. Observers also see the final id this way. + if not tc.get("id"): + tc = {**tc, "id": f"call_{turn}_{total_tool_calls + idx}"} tcv = await notify_tool_call(obs, ctx, tc) if tcv.metadata_updates: metadata.update(tcv.metadata_updates) if tcv.rewrite_args is not None: tc = {**tc, "args": tcv.rewrite_args} if tcv.skip_with_result is not None: + raw_args = tc.get("args") synthetic.append((idx, ToolResult( name=str(tc.get("name", "") or ""), - args=tc.get("args", {}) if isinstance(tc.get("args"), dict) else {}, + args=raw_args if isinstance(raw_args, dict) else {}, result=tcv.skip_with_result, duration_ms=0, - tool_call_id=str(tc.get("id") or f"call_{turn}_{idx}"), is_error=False, + tool_call_id=str( + tc.get("id") or f"call_{turn}_{total_tool_calls + idx}" + ), + is_error=False, ))) else: executable.append((idx, tc)) @@ -969,7 +1107,9 @@ async def _execute_tool_calls( "dropping the unfilled slot(s)", len(results), len(ordered), ) - processor = cfg.tool_result_post_processor or DefaultToolResultPostProcessor(cfg.tool_result_max_chars) + processor = cfg.tool_result_post_processor or DefaultToolResultPostProcessor( + result_max_chars + ) can_recover = "recover_result" in tool_map for tr_result in results: tr_result = await notify_tool_result(obs, ctx, tr_result) diff --git a/agent_core/runtime/loop/model_profile.py b/agent_core/runtime/loop/model_profile.py index 4665318..457a4af 100644 --- a/agent_core/runtime/loop/model_profile.py +++ b/agent_core/runtime/loop/model_profile.py @@ -98,7 +98,18 @@ def _load_thinking_format_patterns() -> ( return () try: import yaml - except ImportError: # pragma: no cover — PyYAML is a project dep + except ImportError: + # AgentCore declares no dependencies, so PyYAML is genuinely optional + # here. Failing silently would leave every model on the caller's + # default format -- for a content_block provider that means reasoning + # is inlined as and the signed blocks are dropped -- while the + # host believes its configured registry is in effect. + logger.error( + "model registry %s is configured but PyYAML is not installed; " + "thinking-format inference is disabled and every model falls back " + "to the caller-supplied default. Install PyYAML to enable it.", + _registry_path, + ) return () try: raw = yaml.safe_load(_registry_path.read_text(encoding="utf-8")) or {} @@ -120,7 +131,17 @@ def _load_thinking_format_patterns() -> ( continue pattern_str = entry.get("pattern") fmt = entry.get("format") - if not pattern_str or fmt not in _VALID_FORMATS: + # ``fmt in _VALID_FORMATS`` raises TypeError for a list or dict, which + # an indentation typo (``format: [content_block]``) produces easily and + # which would abort the whole load. That is exactly what + # :func:`is_thinking_format` exists to prevent. + if not isinstance(pattern_str, str) or not pattern_str: + logger.warning( + "model_registry.yaml: skipped entry with a non-string " + "pattern: %r", entry, + ) + continue + if not is_thinking_format(fmt): logger.warning( "model_registry.yaml: skipped invalid entry %r", entry, ) @@ -333,8 +354,14 @@ def to_history( response: Any, thinking_result: ThinkingResult, policy: HistoryPolicy, + thinking_format: ThinkingFormat, ) -> Message: - """Return the message form appropriate for the conversation history.""" + """Return the message form appropriate for the conversation history. + + The engine passes the resolved ``thinking_format`` as a fourth + argument; the Protocol declared three, so no implementation of it as + written could actually be used. + """ ... @@ -381,9 +408,19 @@ def extract(self, response: Any, profile: ModelProfile) -> ThinkingResult: # case this format originally handled. typed_rc = _extract_reasoning(response) if typed_rc: + # The two channels are not exclusive: SGLang qwen3 fills the + # typed field AND can leave inline tags in ``content``. Keeping + # the raw text as "visible" would replay the same reasoning + # twice in history (``to_history`` re-wraps ``thinking``), and + # would leave inline reasoning in history even when the policy + # explicitly disables it. Strip the tags either way; fold any + # inline blocks into the reasoning so nothing is lost. + inline = _THINK_RE.findall(raw) + visible = _THINK_RE.sub("\n", raw).strip() if inline else raw + thinking = "\n".join([typed_rc, *inline]) if inline else typed_rc return ThinkingResult( - thinking=typed_rc, - visible_content=raw, + thinking=thinking, + visible_content=visible, tool_calls=tool_calls, ) # Every block, not just the first: a turn can reopen diff --git a/agent_core/runtime/loop/tool_exec.py b/agent_core/runtime/loop/tool_exec.py index 33d769e..6956f2a 100644 --- a/agent_core/runtime/loop/tool_exec.py +++ b/agent_core/runtime/loop/tool_exec.py @@ -5,6 +5,7 @@ import asyncio import contextlib +import logging import time from collections.abc import Awaitable, Callable from contextlib import AbstractContextManager, nullcontext, suppress @@ -13,6 +14,8 @@ from agent_core.loop_types import ToolResult +logger = logging.getLogger(__name__) + __all__ = [ "PROTECTED_FANIN_TOOLS", "TOOL_RESULT_MAX_CHARS", @@ -54,13 +57,18 @@ def __init__(self, max_chars: int | None = None) -> None: def process(self, tool_result: ToolResult) -> str: content = tool_result.result + if not isinstance(content, str): + content = str(content) cap = self._max_chars - if cap and isinstance(content, str) and len(content) > cap: - return content[:cap] + ( - f"\n\n[... truncated {len(content) - cap} chars past " - f"{cap}-char cap]" - ) - return content if isinstance(content, str) else str(content) + # A non-positive cap is the conventional "unlimited" sentinel. Letting + # it through would slice from the end (``content[:-3]``) and silently + # drop the tail of every tool result. + if cap is None or cap <= 0 or len(content) <= cap: + return content + return content[:cap] + ( + f"\n\n[... truncated {len(content) - cap} chars past " + f"{cap}-char cap]" + ) TimeoutResolver = Callable[[str, dict[str, Any], int], float] @@ -145,6 +153,55 @@ def _interrupted_result(_name: str) -> str: ) +def _safe_hook[T]( + what: str, + call: Callable[[], T], + fallback: Callable[[], T], +) -> T: + """Run a host hook, falling back to core behavior if it raises. + + Metering, timeout policy and batch budgeting are host concerns layered + around execution; a broken one must not turn a whole tool batch -- or the + surrounding agent loop -- into a failure. Mirrors the attempt-observer + rule in ``_call.py``: passive observability never breaks a valid call. + """ + + try: + return call() + except asyncio.CancelledError: + raise + except Exception: + logger.warning( + "tool execution hook %s raised; falling back to core default", + what, + exc_info=True, + ) + return fallback() + + +def _interrupt_requested(task: asyncio.Future[bool], name: str) -> bool: + """Did the waiter actually ask for an interrupt? + + A waiter that returns ``False`` means "no interrupt was requested", and a + waiter that raises is a broken host observer -- neither is a user + interrupt. Treating either as one would cancel a healthy fan-in tool and + report collected work as abandoned, so both resolve to ``False`` here. + """ + + try: + return bool(task.result()) + except asyncio.CancelledError: + # The waiter itself was cancelled; that is not a user interrupt. + return False + except Exception: + logger.warning( + "tool interrupt waiter for %r raised; treating as no interrupt", + name, + exc_info=True, + ) + return False + + @dataclass(frozen=True) class ToolExecutionHooks: """Host decisions around a shared execution lifecycle. @@ -164,6 +221,11 @@ class ToolExecutionHooks: timeout_result: TimeoutResult = _timeout_result failure_result: FailureResult = _failure_result interrupted_result: InterruptedResult = _interrupted_result + # Which tools may be woken by an interrupt waiter. The default names are + # the ones this engine was extracted from; a host whose fan-in tool is + # called something else has to be able to say so, or its interrupt waiter + # is never consulted no matter that it was supplied. + aggregation_tools: frozenset[str] = _AGGREGATION_TOOLS async def execute_tools( @@ -198,14 +260,24 @@ async def _run_one(call: dict[str, Any], index: int) -> ToolResult: return ToolResult( name=name, args=args, - result=runtime.unknown_result(name, tuple(sorted(tool_map))), + result=_safe_hook( + "unknown_result", + lambda: runtime.unknown_result( + name, tuple(sorted(tool_map)) + ), + lambda: _unknown_result(name, tuple(sorted(tool_map))), + ), duration_ms=0, tool_call_id=tool_call_id, is_error=True, ) - effective_timeout = runtime.resolve_timeout(name, args, timeout) - runtime.on_call(name) + effective_timeout = _safe_hook( + "resolve_timeout", + lambda: runtime.resolve_timeout(name, args, timeout), + lambda: float(timeout), + ) + _safe_hook("on_call", lambda: runtime.on_call(name), lambda: None) invoke_task: asyncio.Future[Any] | None = None interrupt_task: asyncio.Future[bool] | None = None woke_for_interrupt = False @@ -216,16 +288,33 @@ async def _run_one(call: dict[str, Any], index: int) -> ToolResult: invocation = runtime.await_call( tool.ainvoke(args), name, args, effective_timeout ) - if interrupt_waiter is not None and name in _AGGREGATION_TOOLS: + if ( + interrupt_waiter is not None + and name in runtime.aggregation_tools + ): invoke_task = asyncio.ensure_future(invocation) interrupt_task = asyncio.ensure_future(interrupt_waiter(call)) - done, _ = await asyncio.wait( - {invoke_task, interrupt_task}, - return_when=asyncio.FIRST_COMPLETED, - ) - if interrupt_task in done and bool(interrupt_task.result()): - woke_for_interrupt = True - if invoke_task not in done: + pending: set[asyncio.Future[Any]] = { + invoke_task, + interrupt_task, + } + while pending: + done, pending = await asyncio.wait( + pending, return_when=asyncio.FIRST_COMPLETED + ) + # Completed work always wins: if the tool finished in + # the same wake-up as the waiter, keep its result. + if invoke_task in done: + break + # A waiter that finished without requesting an + # interrupt is not a reason to abandon the tool -- + # keep waiting on the invocation alone. + if interrupt_task in done and _interrupt_requested( + interrupt_task, name + ): + woke_for_interrupt = True + break + if woke_for_interrupt: invoke_task.cancel() with suppress(asyncio.CancelledError): await invoke_task @@ -250,8 +339,14 @@ async def _run_one(call: dict[str, Any], index: int) -> ToolResult: return ToolResult( name=name, args=args, - result=runtime.timeout_result( - name, elapsed / 1000, effective_timeout + result=_safe_hook( + "timeout_result", + lambda: runtime.timeout_result( + name, elapsed / 1000, effective_timeout + ), + lambda: _timeout_result( + name, elapsed / 1000, effective_timeout + ), ), duration_ms=elapsed, tool_call_id=tool_call_id, @@ -260,10 +355,15 @@ async def _run_one(call: dict[str, Any], index: int) -> ToolResult: except asyncio.CancelledError: raise except Exception as exc: + failure = exc return ToolResult( name=name, args=args, - result=runtime.failure_result(name, exc), + result=_safe_hook( + "failure_result", + lambda: runtime.failure_result(name, failure), + lambda: _failure_result(name, failure), + ), duration_ms=int((time.monotonic() - start) * 1000), tool_call_id=tool_call_id, is_error=True, @@ -282,4 +382,9 @@ async def _run_one(call: dict[str, Any], index: int) -> ToolResult: invoke_task.exception() tasks = [_run_one(call, index) for index, call in enumerate(tool_calls)] - return runtime.transform_batch(list(await asyncio.gather(*tasks))) + results = list(await asyncio.gather(*tasks)) + return _safe_hook( + "transform_batch", + lambda: runtime.transform_batch(results), + lambda: results, + ) diff --git a/docs/agent-loop-boundary.md b/docs/agent-loop-boundary.md index ee7986d..757e090 100644 --- a/docs/agent-loop-boundary.md +++ b/docs/agent-loop-boundary.md @@ -10,7 +10,10 @@ AgentCore owns one complete logical-turn engine: The products keep policy and state that cannot be portable. `AgentLoopHooks` injects session affinity, wall-deadline lookup, provider-chain state, execution -scope storage, cancellation cleanup and spill-reference detection. +scope storage, cancellation cleanup and spill-reference detection. Session +affinity has a single owner per run: `bind_session` fully replaces the built-in +binding, and therefore also replaces `sticky_session_enabled`, which is ignored +(with a warning) when both are supplied. `ToolExecutionHooks` injects effective timeout calculation, wall-clamped waits, per-call ContextVars and metering, result spilling/truncation and aggregate result budgeting. @@ -26,3 +29,13 @@ the previous keep-all behavior constructs the parser with `keep_unknown_native_companions=True`. In either mode, the loop answers any dropped assistant `tool_call_id`, preventing an orphaned call from making the next provider request invalid. + +That invariant also covers observer interventions that keep the assistant +message but bypass execution -- `skip_tool_execution`, and +`continue_to_next_turn` without `pop_last_message`. Synthetic answers are +inserted directly beneath the assistant message, so an injected user message +never separates a call from its reply. Host hooks are advisory: a raising +`resolve_timeout`, `on_call`, `transform_batch` or result formatter is logged +and falls back to core behavior rather than failing the batch, and a fan-in +interrupt waiter that returns `False` or raises leaves the tool running instead +of discarding its collected work. diff --git a/pyproject.toml b/pyproject.toml index 6da2bdd..73d7850 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,16 @@ dependencies = [] [project.optional-dependencies] tokenizer = ["tiktoken>=0.12.0"] +# ``configure_model_registry()`` reads a host-supplied YAML file. The core has +# no required dependencies, so the parser is an explicit extra rather than an +# assumption: without it registry inference is disabled and logged loudly. +registry = ["pyyaml>=6.0"] dev = [ "pytest>=9.0.2", "pytest-asyncio>=1.3.0", "ruff>=0.7", "pyright[nodejs]>=1.1.400", + "pyyaml>=6.0", ] [build-system] diff --git a/tests/test_agent_loop_engine.py b/tests/test_agent_loop_engine.py index 94bd1de..ea2c3aa 100644 --- a/tests/test_agent_loop_engine.py +++ b/tests/test_agent_loop_engine.py @@ -1,13 +1,20 @@ from __future__ import annotations +import asyncio +import json from typing import Any import pytest from agent_core.llm import LLMResponse -from agent_core.loop_types import LoopConfig, LoopPolicy +from agent_core.loop_types import ( + Intervention, + LoopConfig, + LoopPolicy, + ToolCallIntervention, +) from agent_core.runtime.loop.agent_loop import AgentLoopHooks, run_agent_loop -from agent_core.runtime.loop.model_profile import ModelProfile +from agent_core.runtime.loop.model_profile import HistoryPolicy, ModelProfile from agent_core.runtime.loop.tool_exec import ToolExecutionHooks @@ -153,3 +160,519 @@ async def test_host_can_override_session_binding() -> None: assert result.final_content == "done" assert bound == ["gateway-session"] + + +def _orphan_tool_call_ids(messages: list[dict[str, Any]]) -> set[str]: + """Ids an assistant message announces that no tool message answers. + + Providers reject these with a hard 400, so any request the loop builds must + have an empty orphan set. + """ + announced: set[str] = set() + answered: set[str] = set() + for message in messages: + if message.get("role") == "assistant": + for call in message.get("tool_calls") or []: + if call.get("id"): + announced.add(str(call["id"])) + elif message.get("role") == "tool" and message.get("tool_call_id"): + answered.add(str(message["tool_call_id"])) + return announced - answered + + +def _misplaced_tool_messages(messages: list[dict[str, Any]]) -> list[int]: + """Indices of tool messages not reachable from the assistant call above. + + A tool message separated from its ``tool_calls`` by a user message is as + invalid as a missing one, so injection must not split the pair. + """ + bad: list[int] = [] + for index, message in enumerate(messages): + if message.get("role") != "tool": + continue + cursor = index - 1 + while cursor >= 0 and messages[cursor].get("role") == "tool": + cursor -= 1 + if cursor < 0 or messages[cursor].get("role") != "assistant": + bad.append(index) + return bad + + +def _native_call(call_id: str, value: str) -> dict[str, Any]: + return { + "id": call_id, + "type": "function", + "function": { + "name": "echo", + "arguments": json.dumps({"value": value}), + }, + } + + +@pytest.mark.asyncio +async def test_skip_tool_execution_still_answers_the_announced_calls() -> None: + """``skip_tool_execution`` must not strand the assistant's tool_call_id.""" + + class SkipExecution: + # Only critical observers have their Intervention collected; + # non-critical hooks run detached and their return value is dropped. + critical = True + + async def on_llm_response(self, ctx: Any) -> Intervention: + if ctx.tool_calls: + return Intervention(skip_tool_execution=True) + return Intervention() + + llm = SequenceLLM( + [ + LLMResponse(content="", tool_calls=[_native_call("tc1", "hello")]), + LLMResponse(content="finished"), + ] + ) + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[EchoTool()], + config=_config(), + observers=[SkipExecution()], + model_profile=ModelProfile(model_id="test", provider="test"), + ) + + assert result.final_content == "finished" + assert len(llm.calls) == 2 + assert _orphan_tool_call_ids(llm.calls[1]) == set() + assert _misplaced_tool_messages(llm.calls[1]) == [] + assert _orphan_tool_call_ids(result.messages) == set() + + +@pytest.mark.asyncio +async def test_continue_to_next_turn_without_pop_answers_the_calls() -> None: + """A replayed turn keeps the assistant message, so it must answer it too.""" + + class ReplayOnce: + critical = True + + def __init__(self) -> None: + self.fired = False + + async def on_llm_response(self, ctx: Any) -> Intervention: + if ctx.tool_calls and not self.fired: + self.fired = True + return Intervention( + continue_to_next_turn=True, + inject_messages=["reconsider that call"], + ) + return Intervention() + + llm = SequenceLLM( + [ + LLMResponse(content="", tool_calls=[_native_call("tc1", "hello")]), + LLMResponse(content="finished"), + ] + ) + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[EchoTool()], + config=_config(), + observers=[ReplayOnce()], + model_profile=ModelProfile(model_id="test", provider="test"), + ) + + assert result.final_content == "finished" + assert len(llm.calls) == 2 + assert _orphan_tool_call_ids(llm.calls[1]) == set() + assert _misplaced_tool_messages(llm.calls[1]) == [] + + +@pytest.mark.asyncio +async def test_skipping_a_text_parsed_call_does_not_reuse_the_synthetic_id() -> None: + """Text-mode calls carry no provider id, so the engine mints one. + + The skipped and executed halves used to number off different lists, which + made them collide on the same ``tool_call_id`` once an earlier call was + short-circuited. + """ + + class SkipFirst: + def __init__(self) -> None: + self.seen = 0 + + async def on_tool_call( + self, _ctx: Any, _tool_call: dict[str, Any] + ) -> ToolCallIntervention: + self.seen += 1 + if self.seen == 1: + return ToolCallIntervention(skip_with_result="skipped by policy") + return ToolCallIntervention() + + text_calls = ( + '{"tool": "echo", "args": {"value": "a"}}' + '{"tool": "echo", "args": {"value": "b"}}' + ) + llm = SequenceLLM( + [LLMResponse(content=text_calls), LLMResponse(content="finished")] + ) + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[EchoTool()], + config=_config(), + observers=[SkipFirst()], + model_profile=ModelProfile(model_id="test", provider="test"), + ) + + assert result.final_content == "finished" + tool_ids = [ + message.get("tool_call_id") + for message in result.messages + if message.get("role") == "tool" + ] + assert len(tool_ids) == 2 + assert len(set(tool_ids)) == 2, f"colliding tool_call_ids: {tool_ids}" + bodies = [ + message.get("content") + for message in result.messages + if message.get("role") == "tool" + ] + assert "skipped by policy" in bodies + assert "b" in bodies + + +@pytest.mark.asyncio +async def test_bind_session_takes_over_and_warns_about_the_sticky_flag( + caplog: pytest.LogCaptureFixture, +) -> None: + """Two hooks for one concern must not resolve silently.""" + bound: list[str] = [] + sticky_consulted: list[bool] = [] + llm = SequenceLLM([LLMResponse(content="done")]) + + with caplog.at_level("WARNING"): + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[], + config=LoopConfig( + max_turns=1, + task_id="runtime-task", + llm_session_id="gateway-session", + loop_policy=LoopPolicy(no_tool_behavior="stop"), + max_llm_retries=1, + ), + runtime_hooks=AgentLoopHooks( + bind_session=lambda client, session_id: ( + bound.append(session_id) or client + ), + sticky_session_enabled=lambda: ( + sticky_consulted.append(True) or True + ), + ), + ) + + assert result.final_content == "done" + assert bound == ["gateway-session"] + assert sticky_consulted == [] + assert any( + "bind_session owns session affinity" in record.message + for record in caplog.records + ) + + +@pytest.mark.asyncio +async def test_history_policy_tool_result_cap_is_honoured() -> None: + """``HistoryPolicy.tool_result_max_chars`` used to be read by nobody.""" + + class BigTool: + name = "echo" + + async def ainvoke(self, args: dict[str, Any]) -> Any: + return "x" * 500 + + def to_openai_schema(self) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": self.name, + "description": "big", + "parameters": {"type": "object"}, + }, + } + + llm = SequenceLLM( + [ + LLMResponse(content="", tool_calls=[_native_call("tc1", "hello")]), + LLMResponse(content="finished"), + ] + ) + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[BigTool()], + config=_config(), + history_policy=HistoryPolicy(tool_result_max_chars=100), + model_profile=ModelProfile(model_id="test", provider="test"), + ) + + assert result.final_content == "finished" + body = next( + str(message.get("content") or "") + for message in result.messages + if message.get("role") == "tool" + ) + assert "truncated" in body + assert len(body) < 400 + + +@pytest.mark.asyncio +async def test_loop_config_cap_overrides_the_policy_default() -> None: + llm = SequenceLLM( + [ + LLMResponse(content="", tool_calls=[_native_call("tc1", "hello")]), + LLMResponse(content="finished"), + ] + ) + + config = _config() + config.tool_result_max_chars = 0 # explicit "no cap" + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[EchoTool()], + config=config, + history_policy=HistoryPolicy(tool_result_max_chars=2), + model_profile=ModelProfile(model_id="test", provider="test"), + ) + + body = next( + str(message.get("content") or "") + for message in result.messages + if message.get("role") == "tool" + ) + assert body == "hello" + + +@pytest.mark.asyncio +async def test_synthesised_zero_usage_does_not_reset_the_token_estimate() -> None: + """A gateway that omits usage must not silently disable the overflow guard. + + The zero-filled fallback exists for cost attribution only; letting its + zeros overwrite the previous turn's real counts made + ``_handle_context_overflow`` compute its estimate off 0 and never fire. + + Numbers are chosen so the guard's verdict differs *only* because of the + carried-over counts: floor is 1000 (max_completion_tokens=0, empty summary + prompt), turn 1 reports 900+50 and appends a tiny tool result, turn 2 + reports no usage and appends a ~1506-token one. + + turn 1 : 950 + 7 + 1000 = 1957 < 3000 (continues) + turn 2, zeros : 0 + 1506 + 1000 = 2506 < 3000 (guard misses) + turn 2, preserved : 950 + 1506 + 1000 = 3456 >= 3000 (guard fires) + """ + + class SizedTool: + name = "echo" + + async def ainvoke(self, args: dict[str, Any]) -> Any: + return args["value"] + + def to_openai_schema(self) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": self.name, + "description": "echo", + "parameters": {"type": "object"}, + }, + } + + first = LLMResponse(content="", tool_calls=[_native_call("tc1", "hello")]) + first.usage = {"prompt_tokens": 900, "completion_tokens": 50} + second = LLMResponse( + content="", tool_calls=[_native_call("tc2", "x" * 4000)] + ) + second.usage = None + llm = SequenceLLM([first, second, LLMResponse(content="finished")]) + + config = _config() + config.context_overflow_guard = True + config.max_context_length = 3000 + config.max_completion_tokens = 0 + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[SizedTool()], + config=config, + model_profile=ModelProfile(model_id="test", provider="test"), + ) + + assert result.stopped_by == "context_limit_reached" + + +@pytest.mark.asyncio +async def test_host_named_fan_in_tool_can_be_interrupted() -> None: + """A host whose fan-in tool is not named ``collect_reports``. + + Its interrupt waiter used to be ignored no matter that it was supplied. + """ + + class GatherTool: + name = "gather_subagent_reports" + + async def ainvoke(self, args: dict[str, Any]) -> Any: + await asyncio.sleep(10) + return "never" + + def to_openai_schema(self) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": self.name, + "description": "fan in", + "parameters": {"type": "object"}, + }, + } + + class Interrupter: + async def wait_for_tool_interrupt( + self, _ctx: Any, _tool_call: dict[str, Any] + ) -> bool: + await asyncio.sleep(0) + return True + + call = { + "id": "tc1", + "type": "function", + "function": {"name": "gather_subagent_reports", "arguments": "{}"}, + } + llm = SequenceLLM( + [ + LLMResponse(content="", tool_calls=[call]), + LLMResponse(content="finished"), + ] + ) + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[GatherTool()], + config=_config(), + observers=[Interrupter()], + runtime_hooks=AgentLoopHooks( + tool_execution=ToolExecutionHooks( + aggregation_tools=frozenset({"gather_subagent_reports"}) + ) + ), + model_profile=ModelProfile(model_id="test", provider="test"), + ) + + body = next( + str(message.get("content") or "") + for message in result.messages + if message.get("role") == "tool" + ) + assert body.startswith("[interrupted]") + + +@pytest.mark.asyncio +async def test_rollback_after_an_injected_message_still_drops_the_turn() -> None: + """``pop_last_message`` must not be a no-op once a user message is injected. + + ``on_tool_wait_interrupted`` appends a user message *inside* the turn, so at + end-of-turn rollback the tail is [assistant, tool, user]. Stopping at the + first non-tool message made the pop a silent no-op, and paired with + ``continue_to_next_turn`` the same assistant turn replayed until the + attempt buffer ran out. + """ + + class InterruptThenRollback: + critical = True + + def __init__(self) -> None: + self.rolled_back = False + + async def wait_for_tool_interrupt( + self, _ctx: Any, _tool_call: dict[str, Any] + ) -> bool: + await asyncio.sleep(0) + return not self.rolled_back + + async def on_tool_wait_interrupted(self, _ctx: Any) -> Intervention: + return Intervention(inject_messages=["a new user message arrived"]) + + async def on_turn_end(self, _ctx: Any) -> Intervention: + if not self.rolled_back: + self.rolled_back = True + return Intervention( + pop_last_message=True, continue_to_next_turn=True + ) + return Intervention() + + class FanInTool: + name = "collect_reports" + + async def ainvoke(self, args: dict[str, Any]) -> Any: + await asyncio.sleep(10) + return "never" + + def to_openai_schema(self) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": self.name, + "description": "fan in", + "parameters": {"type": "object"}, + }, + } + + fan_call = { + "id": "tc1", + "type": "function", + "function": {"name": "collect_reports", "arguments": "{}"}, + } + llm = SequenceLLM( + [ + LLMResponse(content="", tool_calls=[fan_call]), + LLMResponse(content="finished"), + ] + ) + + result = await run_agent_loop( + system_prompt="system", + user_message="start", + llm=llm, + tools=[FanInTool()], + config=_config(), + observers=[InterruptThenRollback()], + model_profile=ModelProfile(model_id="test", provider="test"), + ) + + assert result.final_content == "finished" + assert len(llm.calls) == 2 + # The rejected assistant turn and its tool reply are gone... + assert not any(message.get("tool_calls") for message in llm.calls[1]), ( + llm.calls[1] + ) + assert not any( + message.get("role") == "tool" for message in llm.calls[1] + ), llm.calls[1] + # ...while the injected notice that replaced it survives. + assert any( + "a new user message arrived" in str(message.get("content") or "") + for message in llm.calls[1] + ) + assert _orphan_tool_call_ids(llm.calls[1]) == set() diff --git a/tests/test_model_registry_loading.py b/tests/test_model_registry_loading.py new file mode 100644 index 0000000..9f0561b --- /dev/null +++ b/tests/test_model_registry_loading.py @@ -0,0 +1,103 @@ +"""Loader robustness for the host-configured model registry. + +The registry is the PR's headline host-configuration seam and had no coverage: +a malformed row or a missing YAML parser decided every model's thinking format +silently. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from agent_core.runtime.loop.model_profile import ( + configure_model_registry, + infer_thinking_format, + reset_thinking_format_cache, +) + + +@pytest.fixture(autouse=True) +def _restore_registry(): + yield + configure_model_registry(None) + reset_thinking_format_cache() + + +def _write(tmp_path: Path, body: str) -> Path: + path = tmp_path / "model_registry.yaml" + path.write_text(body, encoding="utf-8") + configure_model_registry(path) + return path + + +def test_valid_registry_drives_inference(tmp_path: Path) -> None: + _write( + tmp_path, + "thinking_formats:\n" + " - pattern: '^claude-'\n" + " format: content_block\n", + ) + assert infer_thinking_format("claude-opus-5", default="tag") == "content_block" + assert infer_thinking_format("qwen3-32b", default="tag") == "tag" + + +def test_unhashable_format_value_skips_the_row_instead_of_crashing( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """``format: [content_block]`` is an easy indentation typo. + + ``fmt in _VALID_FORMATS`` raises ``TypeError: unhashable type`` for a list, + which aborted the whole load and took every other row down with it. + """ + _write( + tmp_path, + "thinking_formats:\n" + " - pattern: '^qwen'\n" + " format: [content_block]\n" + " - pattern: '^claude-'\n" + " format: content_block\n", + ) + with caplog.at_level("WARNING"): + assert infer_thinking_format("qwen3-32b", default="tag") == "tag" + # The healthy row still loaded. + assert ( + infer_thinking_format("claude-opus-5", default="tag") == "content_block" + ) + assert any("skipped invalid entry" in r.message for r in caplog.records) + + +def test_non_string_pattern_skips_the_row(tmp_path: Path) -> None: + _write( + tmp_path, + "thinking_formats:\n" + " - pattern: {oops: 1}\n" + " format: content_block\n" + " - pattern: '^claude-'\n" + " format: content_block\n", + ) + assert infer_thinking_format("claude-opus-5", default="tag") == "content_block" + + +def test_missing_yaml_parser_is_reported_not_silently_ignored( + tmp_path: Path, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + """Silently returning the default hides a fully disabled registry. + + For a ``content_block`` provider that fallback inlines reasoning as + ```` and drops the signed blocks, while the host believes its + configured registry is in effect. + """ + _write( + tmp_path, + "thinking_formats:\n - pattern: '^claude-'\n format: content_block\n", + ) + reset_thinking_format_cache() + monkeypatch.setitem(sys.modules, "yaml", None) + + with caplog.at_level("ERROR"): + assert infer_thinking_format("claude-opus-5", default="tag") == "tag" + + assert any("PyYAML is not installed" in r.message for r in caplog.records) diff --git a/tests/test_thinking_history_policy.py b/tests/test_thinking_history_policy.py index 587c486..2916eb7 100644 --- a/tests/test_thinking_history_policy.py +++ b/tests/test_thinking_history_policy.py @@ -206,3 +206,41 @@ def test_capped_policy_applies_to_reasoning_content_format( } + + +def test_typed_reasoning_and_inline_tags_do_not_duplicate_reasoning() -> None: + """SGLang qwen3 can fill the typed channel AND leave inline tags. + + Keeping the raw text as visible content replayed the same reasoning twice + in history, and left inline reasoning there even when the policy disabled + it. + """ + response = LLMResponse( + content="inline\nanswer", + reasoning_content="typed rc", + ) + history = _history( + response, policy=HistoryPolicy(thinking_in_history=True) + ) + content = str(history.get("content") or "") + + assert content.count("") == 1 + assert "typed rc" in content + assert "inline" in content + assert "answer" in content + + +def test_disabled_thinking_history_strips_inline_tags_from_the_typed_path() -> None: + response = LLMResponse( + content="inline\nanswer", + reasoning_content="typed rc", + ) + history = _history( + response, policy=HistoryPolicy(thinking_in_history=False) + ) + content = str(history.get("content") or "") + + assert "" not in content + assert "inline" not in content + assert "typed rc" not in content + assert "answer" in content diff --git a/tests/test_tool_exec.py b/tests/test_tool_exec.py index 3e9b65c..a09160a 100644 --- a/tests/test_tool_exec.py +++ b/tests/test_tool_exec.py @@ -118,3 +118,143 @@ async def test_external_cancellation_is_not_converted_to_tool_error() -> None: task.cancel() with pytest.raises(asyncio.CancelledError): await task + + +@pytest.mark.asyncio +async def test_waiter_returning_false_leaves_fan_in_tool_running() -> None: + """A waiter resolving to ``False`` is not an interrupt. + + ``_wait_for_tool_interrupt`` returns ``False`` whenever no observer asked + to interrupt, so racing it must not cancel the fan-in tool nor report the + collected work as abandoned. + """ + + async def no_interrupt(_call: dict[str, Any]) -> bool: + await asyncio.sleep(0) + return False + + results = await execute_tools( + [{"name": "collect_reports", "args": {}, "id": "fan"}], + {"collect_reports": FakeTool("collect_reports", "gathered", delay=0.02)}, + timeout=20, + turn=1, + count_offset=0, + interrupt_waiter=no_interrupt, + ) + + assert results[0].result == "gathered" + assert results[0].interrupted is False + assert results[0].is_error is False + + +@pytest.mark.asyncio +async def test_raising_waiter_neither_interrupts_nor_fails_the_tool() -> None: + async def broken(_call: dict[str, Any]) -> bool: + await asyncio.sleep(0) + raise RuntimeError("observer exploded") + + results = await execute_tools( + [{"name": "collect_reports", "args": {}, "id": "fan"}], + {"collect_reports": FakeTool("collect_reports", "gathered", delay=0.02)}, + timeout=20, + turn=1, + count_offset=0, + interrupt_waiter=broken, + ) + + assert results[0].result == "gathered" + assert results[0].interrupted is False + assert results[0].is_error is False + + +@pytest.mark.asyncio +async def test_real_interrupt_still_wins_over_a_slow_tool() -> None: + async def interrupt(_call: dict[str, Any]) -> bool: + await asyncio.sleep(0) + return True + + results = await execute_tools( + [{"name": "collect_reports", "args": {}, "id": "fan"}], + {"collect_reports": FakeTool("collect_reports", delay=10)}, + timeout=20, + turn=1, + count_offset=0, + interrupt_waiter=interrupt, + ) + + assert results[0].interrupted is True + assert results[0].result.startswith("[interrupted]") + + +@pytest.mark.asyncio +async def test_finished_tool_result_wins_when_interrupt_lands_together() -> None: + """Work that already completed is never thrown away for an interrupt.""" + + async def interrupt(_call: dict[str, Any]) -> bool: + await asyncio.sleep(0.02) + return True + + results = await execute_tools( + [{"name": "collect_reports", "args": {}, "id": "fan"}], + {"collect_reports": FakeTool("collect_reports", "gathered")}, + timeout=20, + turn=1, + count_offset=0, + interrupt_waiter=interrupt, + ) + + assert results[0].result == "gathered" + assert results[0].interrupted is False + + +@pytest.mark.asyncio +async def test_raising_host_hooks_fall_back_instead_of_failing_the_batch() -> None: + """Metering and budgeting are advisory; a broken one must not kill tools.""" + + def boom(*_args: Any, **_kwargs: Any) -> Any: + raise RuntimeError("hook exploded") + + hooks = ToolExecutionHooks( + resolve_timeout=boom, + on_call=boom, + transform_batch=boom, + unknown_result=boom, + failure_result=boom, + ) + results = await execute_tools( + [ + {"name": "ok", "args": {}, "id": "a"}, + {"name": "missing", "args": {}, "id": "b"}, + {"name": "bad", "args": {}, "id": "c"}, + ], + {"ok": FakeTool("ok", "done"), "bad": FakeTool("bad", ValueError("boom"))}, + timeout=1, + turn=1, + count_offset=0, + hooks=hooks, + ) + + assert [result.tool_call_id for result in results] == ["a", "b", "c"] + assert results[0].result == "done" + assert "unknown tool 'missing'" in results[1].result + assert "boom" in results[2].result + + +def test_non_positive_result_cap_means_unlimited_not_tail_trimming() -> None: + """``-1`` is a common "no cap" sentinel; it must not slice from the end.""" + from agent_core.loop_types import ToolResult + from agent_core.runtime.loop.tool_exec import DefaultToolResultPostProcessor + + body = "abcdefghij" + result = ToolResult( + name="t", args={}, result=body, duration_ms=0, + tool_call_id="x", is_error=False, + ) + + assert DefaultToolResultPostProcessor(None).process(result) == body + assert DefaultToolResultPostProcessor(0).process(result) == body + assert DefaultToolResultPostProcessor(-3).process(result) == body + + capped = DefaultToolResultPostProcessor(4).process(result) + assert capped.startswith("abcd") + assert "truncated 6 chars" in capped diff --git a/uv.lock b/uv.lock index 6af0bef..c62a421 100644 --- a/uv.lock +++ b/uv.lock @@ -12,8 +12,12 @@ dev = [ { name = "pyright", extra = ["nodejs"] }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pyyaml" }, { name = "ruff" }, ] +registry = [ + { name = "pyyaml" }, +] tokenizer = [ { name = "tiktoken" }, ] @@ -31,10 +35,12 @@ requires-dist = [ { name = "pyright", extras = ["nodejs"], marker = "extra == 'dev'", specifier = ">=1.1.400" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3.0" }, + { name = "pyyaml", marker = "extra == 'dev'", specifier = ">=6.0" }, + { name = "pyyaml", marker = "extra == 'registry'", specifier = ">=6.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.7" }, { name = "tiktoken", marker = "extra == 'tokenizer'", specifier = ">=0.12.0" }, ] -provides-extras = ["tokenizer", "dev"] +provides-extras = ["tokenizer", "registry", "dev"] [package.metadata.requires-dev] dev = [ @@ -310,6 +316,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "regex" version = "2026.8.31"