minor: added litellm, protocol specs, typescript impl

examples/protocol_demo.py

@@ -0,0 +1,98 @@
+"""
+ECP Protocol Demonstration (Zero-YAML, Zero-CLI)
+
+This script demonstrates how an evaluation client (e.g. LangSmith, Promptfoo)
+can communicate with an ECP Server (an AI Agent) using nothing but raw JSON-RPC
+over standard input/output. 
+
+It does NOT use `ecp_runtime`, YAML files, or the `ecp run` CLI. 
+It proves that ECP is a language-agnostic protocol.
+"""
+
+import json
+import subprocess
+import sys
+import threading
+from typing import Dict, Any
+
+# We'll use the existing customer support demo agent as our server
+AGENT_CMD = [sys.executable, "examples/customer_support_demo/agent.py"]
+
+def send_rpc(process: subprocess.Popen, method: str, params: Dict[str, Any], msg_id: int) -> None:
+    """Send a JSON-RPC 2.0 message to the agent's stdin."""
+    payload = {
+        "jsonrpc": "2.0",
+        "method": method,
+        "params": params,
+        "id": msg_id
+    }
+    raw = json.dumps(payload) + "\n"
+    print(f"\n---> [CLIENT SENDS]: {raw.strip()}")
+
+    if process.stdin:
+        process.stdin.write(raw)
+        process.stdin.flush()
+
+def read_rpc(process: subprocess.Popen) -> None:
+    """Continuously read JSON-RPC 2.0 responses from the agent's stdout."""
+    if not process.stdout:
+        return
+
+    for line in process.stdout:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            # We just print the raw response to demonstrate the protocol shape
+            response = json.loads(line)
+            formatted = json.dumps(response, indent=2)
+            print(f"<--- [AGENT RESPONDS]:\n{formatted}")
+        except json.JSONDecodeError:
+            print(f"<--- [AGENT STDOUT]: {line}")
+
+def main() -> None:
+    print(f"Starting ECP Protocol Demo...")
+    print(f"Launching Agent Server: {' '.join(AGENT_CMD)}")
+
+    # 1. Start the Agent Process
+    process = subprocess.Popen(
+        AGENT_CMD,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=sys.stderr,  # Pipe stderr straight through
+        text=True,
+        bufsize=1 # Line-buffered
+    )
+
+    # 2. Start a background thread to read responses
+    reader_thread = threading.Thread(target=read_rpc, args=(process,), daemon=True)
+    reader_thread.start()
+
+    try:
+        # 3. Send the Initialization message
+        send_rpc(process, "agent/initialize", {"config": {}}, msg_id=1)
+
+        # 4. Wait a moment, then send the Step message (the actual evaluation task)
+        import time
+        time.sleep(1)
+
+        send_rpc(
+            process, 
+            "agent/step", 
+            {"input": "Hi, I'd like to return order #A100 because it arrived damaged."}, 
+            msg_id=2
+        )
+
+        time.sleep(4)
+
+        # 5. Send the Reset message to clear state
+        send_rpc(process, "agent/reset", {}, msg_id=3)
+        time.sleep(1)
+
+    finally:
+        print("\nShutting down Agent Server...")
+        process.terminate()
+        process.wait()
+
+if __name__ == "__main__":
+    main()

mkdocs.yml

@@ -5,14 +5,17 @@ repo_url: https://github.com/evaluation-context-protocol/ecp
 repo_name: evaluation-context-protocol/ecp
 
 nav:
-  - Home: index.md
-  - Why ECP: why-ecp.md
-  - Quickstart: quickstart.md
-  - CI: ci.md
-  - Inspector: inspector.md
-  - Examples: examples.md
-  - DX Validation: dev-experience.md
-  - Specification: spec.md
+  - Introduction:
+      - Home: index.md
+      - Why ECP: why-ecp.md
+  - Protocol Specification: spec.md
+  - Python SDK:
+      - Quickstart: quickstart.md
+      - Examples: examples.md
+  - CLI Tool (YAML):
+      - CI: ci.md
+      - Inspector: inspector.md
+      - DX Validation: dev-experience.md
 
 theme:
   name: material

runtime/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ecp-runtime"
-version = "0.5.0"
+version = "0.6.0"
 description = "Vendor-neutral runtime for portable AI agent evaluations with outputs, tool calls, and audit context."
 authors = [
   { name = "ECP Maintainers", email = "aniket.wattamwar17@gmail.com" },
@@ -16,7 +16,7 @@ dependencies = [
     "pyyaml>=6.0",          # For reading Manifests
     "pydantic>=2.0",        # For validating Schemas
     "rich>=13.0",           # For pretty terminal output
-    "openai>=1.0.0",        # For the LLM Judge
+    "litellm>=1.0.0,<1.50.0", # For universal LLM support in graders (avoid rust ring on arm64)
   "Jinja2>=3.1.0",        # For HTML report rendering
 ]
 

runtime/python/src/ecp_runtime/graders.py

@@ -10,11 +10,12 @@
     sys.path.append(os.path.dirname(__file__))
     from manifest import GraderConfig, StepConfig  # type: ignore
 
-# Try importing OpenAI, but don't crash if it's missing (unless used)
+# Try importing litellm, but don't crash if it's missing (unless used)
 try:
-    from openai import OpenAI  # type: ignore
+    import litellm
+    from litellm import completion  # type: ignore
 except ImportError:
-    OpenAI = None  # type: ignore
+    litellm = None  # type: ignore
 
 
 def _llm_judge_model() -> str:
@@ -52,19 +53,13 @@ def check_text_match(grader: GraderConfig, text: str) -> Tuple[bool, str]:
 
 def check_llm_judge(grader: GraderConfig, text: str) -> Tuple[bool, str, float]:
     """
-    Uses an LLM to evaluate the text.
+    Uses an LLM to evaluate the text via LiteLLM.
     Returns: (passed, reasoning, score)
     """
     if not grader.prompt:
         return False, "No prompt provided for llm_judge", 0.0
-    if OpenAI is None:
-        return False, "LLM judge unavailable: OpenAI library not installed", 0.0
-
-    api_key = os.environ.get("OPENAI_API_KEY")
-    if not api_key:
-        return False, "OPENAI_API_KEY not set in environment", 0.0
-
-    client = OpenAI(api_key=api_key)
+    if litellm is None:
+        return False, "LLM judge unavailable: litellm library not installed. Install with `pip install litellm`.", 0.0
 
     # 1. Construct the Prompt for the Judge
     system_prompt = "You are an impartial AI Judge. You evaluate outputs based on specific criteria."
@@ -84,9 +79,9 @@ def check_llm_judge(grader: GraderConfig, text: str) -> Tuple[bool, str, float]:
     Provide a short reasoning before the result.
     """
 
-    # 2. Call the Judge (using a cheap, smart model)
+    # 2. Call the Judge via LiteLLM
     try:
-        response = client.chat.completions.create(
+        response = completion(
             model=_llm_judge_model(),
             messages=[
                 {"role": "system", "content": system_prompt},

runtime/python/tests/test_graders.py

@@ -35,28 +35,19 @@ def test_llm_judge_fails_without_api_key(self) -> None:
             passed, reason, score = check_llm_judge(grader, "hello")
         self.assertFalse(passed)
         self.assertEqual(score, 0.0)
-        self.assertIn("OPENAI_API_KEY not set", reason)
+        self.assertIn("OPENAI_API_KEY", reason)
 
     def test_llm_judge_uses_configured_model(self) -> None:
         calls = {}
 
-        class _FakeCompletions:
-            def create(self, **kwargs):
-                calls["kwargs"] = kwargs
-                msg = SimpleNamespace(content="Looks good. RESULT: PASS")
-                choice = SimpleNamespace(message=msg)
-                return SimpleNamespace(choices=[choice])
-
-        class _FakeChat:
-            completions = _FakeCompletions()
-
-        class _FakeOpenAIClient:
-            def __init__(self, api_key):
-                self.api_key = api_key
-                self.chat = _FakeChat()
+        def _fake_completion(**kwargs):
+            calls["kwargs"] = kwargs
+            msg = SimpleNamespace(content="Looks good. RESULT: PASS")
+            choice = SimpleNamespace(message=msg)
+            return SimpleNamespace(choices=[choice])
 
         grader = GraderConfig(type="llm_judge", prompt="Check quality.")
-        with mock.patch("ecp_runtime.graders.OpenAI", _FakeOpenAIClient):
+        with mock.patch("ecp_runtime.graders.completion", side_effect=_fake_completion):
             with mock.patch.dict(
                 os.environ,
                 {"OPENAI_API_KEY": "test", "ECP_LLM_JUDGE_MODEL": "gpt-test-model"},

sdk/typescript/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@ecp/sdk",
+  "version": "0.1.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "test": "jest"
+  },
+  "author": "ECP Maintainers",
+  "license": "MIT",
+  "description": "Evaluation Context Protocol SDK for TypeScript/Node.js",
+  "devDependencies": {
+    "typescript": "^5.0.0",
+    "@types/node": "^20.0.0"
+  }
+}