diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ee6a3d1..f7656556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,10 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres WAF checklist row are gone. ### Fixed +- **Observe tool rows inherit explicit runtime evidence from their agent invocation.** + Tool spans that omit provider metadata are now correlated to the matching + `invoke_agent` span within the same project, agent, and trace, so Hosted + Agents no longer appear as `Unknown` in the Tools table. - **Observe now exposes trustworthy aggregates with metadata-only drill-through.** Overview and Agents use workspace-compatible percentile expressions, Models excludes agent and tool spans from inference counts, and sortable counters in diff --git a/src/agentops/agent/observe/queries.py b/src/agentops/agent/observe/queries.py index 5abf496c..3d745a75 100644 --- a/src/agentops/agent/observe/queries.py +++ b/src/agentops/agent/observe/queries.py @@ -438,7 +438,19 @@ def build_tools_query( '| extend tool_name = tostring(Properties["gen_ai.tool.name"])', '| extend operation_name = tostring(Properties["gen_ai.operation.name"])', ] - aggregate_lines = ["base", "| where isnotempty(tool_name)"] + aggregate_lines = [ + "base", + "| where isnotempty(tool_name)", + "| join kind=leftouter runtime_evidence " + "on project_resource_id, agent_key, OperationId", + "| extend agent_id = iff(isnotempty(agent_id), agent_id, runtime_agent_id), " + "agent_name = iff(isnotempty(agent_name), agent_name, runtime_agent_name), " + "provider_name = iff(isnotempty(provider_name), " + "provider_name, runtime_provider_name), " + "system = iff(isnotempty(system), system, runtime_system)", + "| project-away project_resource_id1, agent_key1, OperationId1, " + "runtime_agent_id, runtime_agent_name, runtime_provider_name, runtime_system", + ] if filters.tool_name: aggregate_lines.append( f"| where tool_name == '{_kql_escape(filters.tool_name)}'" @@ -447,8 +459,12 @@ def build_tools_query( "| summarize invocations = count(), " "failures = countif(Success == false), " "p95_latency_ms = percentile(DurationMs, 95), " - "last_seen = max(TimeGenerated) " - "by project_resource_id, agent_key, agent_id, agent_name, provider_name, system, tool_name" + "last_seen = max(TimeGenerated), " + "agent_id = take_anyif(agent_id, isnotempty(agent_id)), " + "agent_name = take_anyif(agent_name, isnotempty(agent_name)), " + "provider_name = take_anyif(provider_name, isnotempty(provider_name)), " + "system = take_anyif(system, isnotempty(system)) " + "by project_resource_id, agent_key, tool_name" ) unattributed_count = ( "0" @@ -457,9 +473,17 @@ def build_tools_query( ) return "\n".join( [ - f"let base = {base_lines[0]}", - *base_lines[1:-1], - f"{base_lines[-1]};", + "let base = materialize(", + *base_lines, + ");", + "let runtime_evidence = base", + '| where operation_name == "invoke_agent"', + "| summarize " + "runtime_agent_id = take_anyif(agent_id, isnotempty(agent_id)), " + "runtime_agent_name = take_anyif(agent_name, isnotempty(agent_name)), " + "runtime_provider_name = take_anyif(provider_name, isnotempty(provider_name)), " + "runtime_system = take_anyif(system, isnotempty(system)) " + "by project_resource_id, agent_key, OperationId;", f"let unattributed_count = {unattributed_count};", f"let agg = {aggregate_lines[0]}", *aggregate_lines[1:-1], diff --git a/tests/unit/test_observe_queries.py b/tests/unit/test_observe_queries.py index c7a45a2e..c804117b 100644 --- a/tests/unit/test_observe_queries.py +++ b/tests/unit/test_observe_queries.py @@ -744,7 +744,8 @@ def test_tools_query_uses_tool_metadata_without_reading_tool_content() -> None: query = build_tools_query(_filters(tool_name="lookup'o''ticket")) assert query.startswith( - "let base = union withsource=TelemetryTable AppDependencies, AppRequests" + "let base = materialize(\n" + "union withsource=TelemetryTable AppDependencies, AppRequests" ) assert 'Properties["gen_ai.tool.name"]' in query assert 'Properties["gen_ai.operation.name"]' in query @@ -754,7 +755,19 @@ def test_tools_query_uses_tool_metadata_without_reading_tool_content() -> None: assert "lookup\\'o\\'\\'ticket" in query assert "AppGenAIContent" not in query assert "gen_ai.tool.message" not in query - assert "provider_name, system, tool_name" in query + assert ( + "| join kind=leftouter runtime_evidence " + "on project_resource_id, agent_key, OperationId" + ) in query + assert ( + 'runtime_evidence = base\n| where operation_name == "invoke_agent"' + ) in query + assert ( + "provider_name = iff(isnotempty(provider_name), " + "provider_name, runtime_provider_name)" + ) in query + assert "provider_name = take_anyif(provider_name, isnotempty(provider_name))" in query + assert "by project_resource_id, agent_key, tool_name" in query assert "| sort by invocations desc" in query assert f"| take {MAX_ROWS_PER_QUERY}" in query diff --git a/tests/unit/test_observe_service.py b/tests/unit/test_observe_service.py index 9821cf81..5d2bc122 100644 --- a/tests/unit/test_observe_service.py +++ b/tests/unit/test_observe_service.py @@ -655,6 +655,7 @@ def test_normalize_tool_row_preserves_source_and_omits_token_attribution() -> No "tool_name": "search", "agent_key": "agent-1", "agent_id": "agent-1", + "provider_name": "microsoft.agent_framework", "invocations": 4, "failures": 1, "p95_latency_ms": None, @@ -665,6 +666,7 @@ def test_normalize_tool_row_preserves_source_and_omits_token_attribution() -> No assert tool.source_id == source.source_id assert tool.tool_name == "search" + assert tool.source_kind == "foundry_hosted" assert tool.p95_latency_ms is None assert not hasattr(tool, "input_tokens") assert not hasattr(tool, "output_tokens")