From e6ade6926471a5edfa3485ce4b32660f9148f1d6 Mon Sep 17 00:00:00 2001 From: Aniket Wattamwar Date: Sun, 19 Jul 2026 22:31:34 -0600 Subject: [PATCH] minor: added litellm, protocol specs, typescript impl --- examples/protocol_demo.py | 98 +++++++++++++++++++++++ mkdocs.yml | 19 +++-- runtime/python/pyproject.toml | 4 +- runtime/python/src/ecp_runtime/graders.py | 23 +++--- runtime/python/tests/test_graders.py | 23 ++---- sdk/typescript/package.json | 17 ++++ 6 files changed, 144 insertions(+), 40 deletions(-) create mode 100644 examples/protocol_demo.py create mode 100644 sdk/typescript/package.json diff --git a/examples/protocol_demo.py b/examples/protocol_demo.py new file mode 100644 index 0000000..1a63b6d --- /dev/null +++ b/examples/protocol_demo.py @@ -0,0 +1,98 @@ +""" +ECP Protocol Demonstration (Zero-YAML, Zero-CLI) + +This script demonstrates how an evaluation client (e.g. LangSmith, Promptfoo) +can communicate with an ECP Server (an AI Agent) using nothing but raw JSON-RPC +over standard input/output. + +It does NOT use `ecp_runtime`, YAML files, or the `ecp run` CLI. +It proves that ECP is a language-agnostic protocol. +""" + +import json +import subprocess +import sys +import threading +from typing import Dict, Any + +# We'll use the existing customer support demo agent as our server +AGENT_CMD = [sys.executable, "examples/customer_support_demo/agent.py"] + +def send_rpc(process: subprocess.Popen, method: str, params: Dict[str, Any], msg_id: int) -> None: + """Send a JSON-RPC 2.0 message to the agent's stdin.""" + payload = { + "jsonrpc": "2.0", + "method": method, + "params": params, + "id": msg_id + } + raw = json.dumps(payload) + "\n" + print(f"\n---> [CLIENT SENDS]: {raw.strip()}") + + if process.stdin: + process.stdin.write(raw) + process.stdin.flush() + +def read_rpc(process: subprocess.Popen) -> None: + """Continuously read JSON-RPC 2.0 responses from the agent's stdout.""" + if not process.stdout: + return + + for line in process.stdout: + line = line.strip() + if not line: + continue + try: + # We just print the raw response to demonstrate the protocol shape + response = json.loads(line) + formatted = json.dumps(response, indent=2) + print(f"<--- [AGENT RESPONDS]:\n{formatted}") + except json.JSONDecodeError: + print(f"<--- [AGENT STDOUT]: {line}") + +def main() -> None: + print(f"Starting ECP Protocol Demo...") + print(f"Launching Agent Server: {' '.join(AGENT_CMD)}") + + # 1. Start the Agent Process + process = subprocess.Popen( + AGENT_CMD, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=sys.stderr, # Pipe stderr straight through + text=True, + bufsize=1 # Line-buffered + ) + + # 2. Start a background thread to read responses + reader_thread = threading.Thread(target=read_rpc, args=(process,), daemon=True) + reader_thread.start() + + try: + # 3. Send the Initialization message + send_rpc(process, "agent/initialize", {"config": {}}, msg_id=1) + + # 4. Wait a moment, then send the Step message (the actual evaluation task) + import time + time.sleep(1) + + send_rpc( + process, + "agent/step", + {"input": "Hi, I'd like to return order #A100 because it arrived damaged."}, + msg_id=2 + ) + + time.sleep(4) + + # 5. Send the Reset message to clear state + send_rpc(process, "agent/reset", {}, msg_id=3) + time.sleep(1) + + finally: + print("\nShutting down Agent Server...") + process.terminate() + process.wait() + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 5cf8170..95a23f2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,14 +5,17 @@ repo_url: https://github.com/evaluation-context-protocol/ecp repo_name: evaluation-context-protocol/ecp nav: - - Home: index.md - - Why ECP: why-ecp.md - - Quickstart: quickstart.md - - CI: ci.md - - Inspector: inspector.md - - Examples: examples.md - - DX Validation: dev-experience.md - - Specification: spec.md + - Introduction: + - Home: index.md + - Why ECP: why-ecp.md + - Protocol Specification: spec.md + - Python SDK: + - Quickstart: quickstart.md + - Examples: examples.md + - CLI Tool (YAML): + - CI: ci.md + - Inspector: inspector.md + - DX Validation: dev-experience.md theme: name: material diff --git a/runtime/python/pyproject.toml b/runtime/python/pyproject.toml index d03364f..d9fc9d7 100644 --- a/runtime/python/pyproject.toml +++ b/runtime/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "ecp-runtime" -version = "0.4.1" +version = "0.6.0" description = "Vendor-neutral runtime for portable AI agent evaluations with outputs, tool calls, and audit context." authors = [ { name = "ECP Maintainers", email = "aniket.wattamwar17@gmail.com" }, @@ -16,7 +16,7 @@ dependencies = [ "pyyaml>=6.0", # For reading Manifests "pydantic>=2.0", # For validating Schemas "rich>=13.0", # For pretty terminal output - "openai>=1.0.0", # For the LLM Judge + "litellm>=1.0.0,<1.50.0", # For universal LLM support in graders (avoid rust ring on arm64) "Jinja2>=3.1.0", # For HTML report rendering ] diff --git a/runtime/python/src/ecp_runtime/graders.py b/runtime/python/src/ecp_runtime/graders.py index 218175d..cdcc5b7 100644 --- a/runtime/python/src/ecp_runtime/graders.py +++ b/runtime/python/src/ecp_runtime/graders.py @@ -10,11 +10,12 @@ sys.path.append(os.path.dirname(__file__)) from manifest import GraderConfig, StepConfig # type: ignore -# Try importing OpenAI, but don't crash if it's missing (unless used) +# Try importing litellm, but don't crash if it's missing (unless used) try: - from openai import OpenAI # type: ignore + import litellm + from litellm import completion # type: ignore except ImportError: - OpenAI = None # type: ignore + litellm = None # type: ignore def _llm_judge_model() -> str: @@ -52,19 +53,13 @@ def check_text_match(grader: GraderConfig, text: str) -> Tuple[bool, str]: def check_llm_judge(grader: GraderConfig, text: str) -> Tuple[bool, str, float]: """ - Uses an LLM to evaluate the text. + Uses an LLM to evaluate the text via LiteLLM. Returns: (passed, reasoning, score) """ if not grader.prompt: return False, "No prompt provided for llm_judge", 0.0 - if OpenAI is None: - return False, "LLM judge unavailable: OpenAI library not installed", 0.0 - - api_key = os.environ.get("OPENAI_API_KEY") - if not api_key: - return False, "OPENAI_API_KEY not set in environment", 0.0 - - client = OpenAI(api_key=api_key) + if litellm is None: + return False, "LLM judge unavailable: litellm library not installed. Install with `pip install litellm`.", 0.0 # 1. Construct the Prompt for the Judge system_prompt = "You are an impartial AI Judge. You evaluate outputs based on specific criteria." @@ -84,9 +79,9 @@ def check_llm_judge(grader: GraderConfig, text: str) -> Tuple[bool, str, float]: Provide a short reasoning before the result. """ - # 2. Call the Judge (using a cheap, smart model) + # 2. Call the Judge via LiteLLM try: - response = client.chat.completions.create( + response = completion( model=_llm_judge_model(), messages=[ {"role": "system", "content": system_prompt}, diff --git a/runtime/python/tests/test_graders.py b/runtime/python/tests/test_graders.py index bd91d31..0d4f45e 100644 --- a/runtime/python/tests/test_graders.py +++ b/runtime/python/tests/test_graders.py @@ -35,28 +35,19 @@ def test_llm_judge_fails_without_api_key(self) -> None: passed, reason, score = check_llm_judge(grader, "hello") self.assertFalse(passed) self.assertEqual(score, 0.0) - self.assertIn("OPENAI_API_KEY not set", reason) + self.assertIn("OPENAI_API_KEY", reason) def test_llm_judge_uses_configured_model(self) -> None: calls = {} - class _FakeCompletions: - def create(self, **kwargs): - calls["kwargs"] = kwargs - msg = SimpleNamespace(content="Looks good. RESULT: PASS") - choice = SimpleNamespace(message=msg) - return SimpleNamespace(choices=[choice]) - - class _FakeChat: - completions = _FakeCompletions() - - class _FakeOpenAIClient: - def __init__(self, api_key): - self.api_key = api_key - self.chat = _FakeChat() + def _fake_completion(**kwargs): + calls["kwargs"] = kwargs + msg = SimpleNamespace(content="Looks good. RESULT: PASS") + choice = SimpleNamespace(message=msg) + return SimpleNamespace(choices=[choice]) grader = GraderConfig(type="llm_judge", prompt="Check quality.") - with mock.patch("ecp_runtime.graders.OpenAI", _FakeOpenAIClient): + with mock.patch("ecp_runtime.graders.completion", side_effect=_fake_completion): with mock.patch.dict( os.environ, {"OPENAI_API_KEY": "test", "ECP_LLM_JUDGE_MODEL": "gpt-test-model"}, diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json new file mode 100644 index 0000000..c7b205c --- /dev/null +++ b/sdk/typescript/package.json @@ -0,0 +1,17 @@ +{ + "name": "@ecp/sdk", + "version": "0.1.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc", + "test": "jest" + }, + "author": "ECP Maintainers", + "license": "MIT", + "description": "Evaluation Context Protocol SDK for TypeScript/Node.js", + "devDependencies": { + "typescript": "^5.0.0", + "@types/node": "^20.0.0" + } +}