fix(pydantic_ai): report structured token usage instead of stringifying to evaluation_context (#28)

sdk/python/src/ecp/adaptors/pydantic_ai.py

@@ -46,17 +46,29 @@ def step(self, input_text: str) -> Result:
         # 1. Capture thoughts and tool calls from messages
         self._capture_from_result(result)
 
-        # 2. Capture usage metadata as a thought
+        # 2. Capture usage metadata
+        usage_payload = None
         try:
             usage = result.usage()
             if usage:
-                usage_str = f"Usage: {usage.input_tokens} input, {usage.output_tokens} output tokens ({usage.requests} requests)"
-                self.captured_thoughts.append(usage_str)
+                payload = {}
+                if getattr(usage, "input_tokens", None) is not None:
+                    payload["input_tokens"] = usage.input_tokens
+                elif getattr(usage, "request_tokens", None) is not None:
+                    payload["input_tokens"] = usage.request_tokens
+                if getattr(usage, "output_tokens", None) is not None:
+                    payload["output_tokens"] = usage.output_tokens
+                elif getattr(usage, "response_tokens", None) is not None:
+                    payload["output_tokens"] = usage.response_tokens
+                if getattr(usage, "total_tokens", None) is not None:
+                    payload["total_tokens"] = usage.total_tokens
+                if payload:
+                    usage_payload = payload
         except Exception:
             pass
 
         # 3. Format public output
-        # If the result has structured 'data', we prefer that. 
+        # If the result has structured 'data', we prefer that.
         # If it's a Pydantic model, dump it correctly.
         public_output = ""
         try:
@@ -77,6 +89,7 @@ def step(self, input_text: str) -> Result:
             public_output=public_output,
             evaluation_context="\n".join(self.captured_thoughts) if self.captured_thoughts else None,
             tool_calls=self.captured_tool_calls or None,
+            usage=usage_payload,
         )
 
     def _capture_from_result(self, result: Any) -> None:
@@ -98,7 +111,7 @@ def _capture_from_result(self, result: Any) -> None:
         # The last ModelResponse is the final one containing public_output
         responses = [m for m in messages if getattr(m, "kind", None) == "response"]
         last_response = responses[-1] if responses else None
-        
+
         for msg in responses:
             parts = getattr(msg, "parts", [])
             for part in parts:
@@ -109,24 +122,24 @@ def _capture_from_result(self, result: Any) -> None:
                     content = getattr(part, "content", "")
                     if content:
                         self.captured_thoughts.append(content)
-                
+
                 # Capture reasoning TextParts
                 elif part_kind == "text":
                     content = getattr(part, "content", "")
                     # A text part is a thought if:
                     # 1. It's in a message that also has tool calls
                     # 2. It's in a message that is NOT the very last response of the run
                     has_tool_calls = any(getattr(p, "part_kind", "") in ("tool-call", "tool_call") for p in parts)
-                    
+
                     if msg is not last_response or has_tool_calls:
                         if content and content.strip():
                             self.captured_thoughts.append(content.strip())
-                
+
                 # Capture tool calls
                 elif part_kind in ("tool-call", "tool_call"):
                     tool_name = getattr(part, "tool_name", None)
                     args = {}
-                    
+
                     if hasattr(part, "args_as_dict"):
                         try:
                             args = part.args_as_dict()

sdk/python/tests/fixtures/adapters/pydantic_ai_calculator.json

@@ -158,7 +158,7 @@
   "expected": {
     "status": "done",
     "public_output": "The answer is 120.",
-    "evaluation_context": "The user wants 15 times 8. Use the calculator tool.\nUsage: 61 input, 14 output tokens (2 requests)",
+    "evaluation_context": "The user wants 15 times 8. Use the calculator tool.",
     "tool_calls": [
       {
         "name": "calculator",
@@ -167,7 +167,10 @@
         }
       }
     ],
-    "usage": null
+    "usage": {
+      "input_tokens": 61,
+      "output_tokens": 14
+    }
   },
   "events": [],
   "adapter_kwargs": {

sdk/python/tests/test_adapters.py

@@ -93,6 +93,30 @@ def usage(self):
         result = adapter.step("2+2")
         self.assertEqual(result.tool_calls[0]["name"], "calculator")
         self.assertEqual(result.tool_calls[0]["arguments"]["expression"], "2+2")
+        self.assertEqual(result.usage, {"input_tokens": 1, "output_tokens": 1})
+        self.assertIsNone(result.evaluation_context)
+
+    def test_pydantic_ai_structured_usage_reported(self):
+        from ecp.adaptors.pydantic_ai import ECPPydanticAIAdapter
+
+        class _RunResult:
+            data = "done"
+            output = "done"
+
+            def new_messages(self):
+                return []
+
+            def usage(self):
+                return SimpleNamespace(input_tokens=100, output_tokens=50, total_tokens=150, requests=1)
+
+        fake_agent = SimpleNamespace(run_sync=lambda *_args, **_kwargs: _RunResult())
+        adapter = ECPPydanticAIAdapter(fake_agent)
+        result = adapter.step("hi")
+        self.assertEqual(
+            result.usage,
+            {"input_tokens": 100, "output_tokens": 50, "total_tokens": 150},
+        )
+        self.assertIsNone(result.evaluation_context)
 
 
 if __name__ == "__main__":