From 9f0152d3191848747c73654f573a904b3e82109c Mon Sep 17 00:00:00 2001 From: Kyle Verhoog Date: Tue, 18 Aug 2026 14:13:18 -0400 Subject: [PATCH] Add LLM Obs evaluations to DDClient Evaluation metrics, datasets, experiments and evaluator publishing are now client methods rather than something you reach into ddtrace for. submit_evaluation infers metric_type from the value (bool before int, since bool is an int subclass) and defaults to the active LLM Obs span, so scoring a workflow is a one-liner. It also accepts a live span or the dict from export_span(), and export_span() raises rather than returning None when nothing is in scope, so that's caught to give a useful error. Datasets/experiments need an app key on top of the api key, added as llmobs_app_key (DD_APP_KEY), alongside llmobs_project_name (DD_LLMOBS_PROJECT_NAME) for the project they organize under. Both are passed through to LLMObs.enable. datadog.llmobs re-exports the evaluator base classes, built-ins, judge helpers and dataset types so evaluator code doesn't import ddtrace. It is intentionally not imported by the package __init__, keeping the ddtrace.llmobs import cost off clients with LLM Obs disabled. Also fixes the return type comment on trace(): ddtrace.Span does not exist at runtime in ddtrace 4, it's ddtrace.trace.Span. Co-Authored-By: Claude Opus 5 --- datadog/_client.py | 292 ++++++++++++++++++++++++++++++++++++++++++++- datadog/llmobs.py | 63 ++++++++++ examples/evals.py | 123 +++++++++++++++++++ readme.md | 65 +++++++++- 4 files changed, 537 insertions(+), 6 deletions(-) create mode 100644 datadog/llmobs.py create mode 100644 examples/evals.py diff --git a/datadog/_client.py b/datadog/_client.py index 9bd96d5..28e8c8a 100644 --- a/datadog/_client.py +++ b/datadog/_client.py @@ -4,7 +4,21 @@ import shutil import subprocess import time -from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Dict, + List, + Literal, + Optional, + Sequence, + Tuple, + Type, + Union, + cast, +) import ddtrace @@ -12,15 +26,35 @@ from ddtrace.internal.utils.formats import asbool from ddtrace.profiling import Profiler from ddtrace.runtime import RuntimeMetrics +from ddtrace.trace import Span from ddtrace._logger import DD_LOG_FORMAT from ._metrics import MetricsClient from ._logging import V2LogWriter +if TYPE_CHECKING: + from ddtrace.llmobs._experiment import Dataset + from ddtrace.llmobs._experiment import DatasetRecordNew + from ddtrace.llmobs._experiment import Experiment + from ddtrace.llmobs._experiment import ExperimentResult + from ddtrace.llmobs._experiment import SyncExperiment + + logger = logging.getLogger(__name__) TraceSampleRule = Tuple[str, str, float] +# The value of an evaluation metric, and the metric_type it implies. +EvalValue = Union[str, int, float, bool, Dict[str, Any]] +MetricType = Literal["categorical", "score", "boolean", "json"] +# A span to join an evaluation to: a live span, or the dict produced by +# ``export_span()`` (which is what crosses a process boundary). +EvalSpan = Union[Span, Dict[str, str]] +# Experiment building blocks. Kept loose here; ``datadog.llmobs`` re-exports +# the real ddtrace types for annotating user code. +Task = Callable[..., Any] +Evaluator = Any +DEFAULT_PROJECT_NAME = "default" # recursive types aren't supported (yet): https://github.com/python/mypy/issues/731 # _JSON = Union[str, float, int, List["_JSON"], Dict[str, "_JSON"], None] @@ -50,6 +84,7 @@ def __bool__(self): llmobs_enabled=False, llmobs_agentless_enabled=False, llmobs_integrations_enabled=True, + llmobs_project_name=DEFAULT_PROJECT_NAME, ) # type: Dict[str, Any] @@ -81,6 +116,8 @@ def __init__( llmobs_ml_app=_sentinel, # type: Union[_Sentinel, str] llmobs_agentless_enabled=_sentinel, # type: Union[_Sentinel, bool] llmobs_integrations_enabled=_sentinel, # type: Union[_Sentinel, bool] + llmobs_app_key=_sentinel, # type: Union[_Sentinel, str] + llmobs_project_name=_sentinel, # type: Union[_Sentinel, str] default_config=_DEFAULT_CONFIG, # type: Dict[str, Any] ): # type: (...) -> None @@ -241,6 +278,36 @@ def __init__( ) self.llmobs_integrations_enabled = llmobs_integrations_enabled + # The app key is only needed by the eval APIs (datasets, experiments, + # publishing evaluators) which talk to the Datadog API directly, so an + # empty one is not an error until one of those is used. + if isinstance(llmobs_app_key, _Sentinel): + llmobs_app_key = os.getenv("DD_APP_KEY", "") + self.llmobs_app_key = cast(str, llmobs_app_key) + + if isinstance(llmobs_project_name, _Sentinel): + llmobs_project_name = os.getenv( + "DD_LLMOBS_PROJECT_NAME", default_config["llmobs_project_name"] + ) + self.llmobs_project_name = cast(str, llmobs_project_name) + + +def _infer_metric_type(label, value): + # type: (str, EvalValue) -> MetricType + # bool first: it is a subclass of int and would otherwise read as a score. + if isinstance(value, bool): + return "boolean" + if isinstance(value, (int, float)): + return "score" + if isinstance(value, str): + return "categorical" + if isinstance(value, dict): + return "json" + raise ValueError( + "Cannot infer the metric type of the %r evaluation from a %s value. " + "Pass metric_type= explicitly." % (label, type(value).__name__) + ) + class DDAgent: def __init__(self, version: str, config: DDConfig): @@ -362,13 +429,15 @@ def __init__( agentless_enabled=config.llmobs_agentless_enabled, site=config.site, api_key=config.api_key, + app_key=config.llmobs_app_key, + project_name=config.llmobs_project_name, service=config.service, env=config.env, ) self._llmobs = LLMObs def trace(self, *args, **kwargs): - # type: (...) -> ddtrace.Span + # type: (...) -> Span return self._tracer.trace(*args, **kwargs) def traced(self, *args, **kwargs): @@ -386,6 +455,17 @@ def _require_llmobs(self): ) return self._llmobs + def _require_llmobs_app_key(self): + # Datasets, experiments and remote evaluators are read/written through + # the Datadog API, which needs an app key on top of the api key. + llmobs = self._require_llmobs() + if not self._config.llmobs_app_key: + raise RuntimeError( + "An app key is required for the datasets/experiments API. Pass " + "llmobs_app_key= to DDConfig (or set DD_APP_KEY)." + ) + return llmobs + # -- LLM Observability decorators --------------------------------- # Use as ``@ddclient.workflow(name="…")`` / ``@ddclient.llm(...)`` etc. # Each proxies to ``ddtrace.llmobs.decorators.``, which produces @@ -437,12 +517,214 @@ def annotate(self, *args, **kwargs): def annotation_context(self, *args, **kwargs): return self._require_llmobs().annotation_context(*args, **kwargs) - def submit_evaluation(self, *args, **kwargs): - return self._require_llmobs().submit_evaluation(*args, **kwargs) - def export_span(self, *args, **kwargs): return self._require_llmobs().export_span(*args, **kwargs) + # -- LLM Observability evaluations -------------------------------- + # Evaluations come in two flavours: metrics submitted against spans that + # already happened (``submit_evaluation``), and experiments, which run a + # task over a dataset and score each row with evaluators. + + def submit_evaluation( + self, + label: str, + value: EvalValue, + metric_type: Optional[MetricType] = None, + span: Optional[EvalSpan] = None, + span_with_tag_value: Optional[Dict[str, str]] = None, + tags: Optional[Dict[str, str]] = None, + metadata: Optional[Dict[str, object]] = None, + assessment: Optional[Literal["pass", "fail"]] = None, + reasoning: Optional[str] = None, + timestamp_ms: Optional[int] = None, + eval_scope: Literal["span", "trace"] = "span", + ) -> None: + """Attach an evaluation metric to a span. + + ``metric_type`` is inferred from ``value`` unless given, and ``span`` + defaults to the span currently being traced, so the common case is + just ``ddclient.submit_evaluation("relevance", 0.9)``. A span from a + previous process can be joined by passing the dict that + ``export_span()`` produced, or by tag with ``span_with_tag_value``. + """ + llmobs = self._require_llmobs() + if metric_type is None: + metric_type = _infer_metric_type(label, value) + if span is None and span_with_tag_value is None: + # export_span() raises rather than returning None when there is no + # LLM Obs span in scope (a plain traced span doesn't count). + from ddtrace.llmobs._llmobs import LLMObsExportSpanError + + try: + span = llmobs.export_span() + except LLMObsExportSpanError: + span = None + if span is None: + raise ValueError( + "No LLM Obs span is currently active to attach the %r " + "evaluation to. Pass span= (a span, or the dict from " + "export_span()) or span_with_tag_value=." % label + ) + elif isinstance(span, Span): + span = llmobs.export_span(span) + llmobs.submit_evaluation( + label=label, + metric_type=metric_type, + value=value, + span=span, + span_with_tag_value=span_with_tag_value, + tags=tags, + metadata=metadata, + assessment=assessment, + reasoning=reasoning, + timestamp_ms=timestamp_ms, + eval_scope=eval_scope, + ) + + def get_spans(self, *args, **kwargs) -> List[Dict[str, Any]]: + """Query already-submitted LLM Obs spans, e.g. to evaluate them offline.""" + return self._require_llmobs().get_spans(*args, **kwargs) + + def publish_evaluator( + self, + evaluator: Evaluator, + eval_name: Optional[str] = None, + variable_mapping: Optional[Dict[str, str]] = None, + ) -> Dict[str, str]: + """Publish an evaluator so Datadog runs it against live spans.""" + return self._require_llmobs_app_key().publish_evaluator( + evaluator=evaluator, + ml_app=self._config.llmobs_ml_app, + eval_name=eval_name, + variable_mapping=variable_mapping, + ) + + # -- LLM Observability datasets ----------------------------------- + + def create_dataset( + self, + name: str, + records: Optional[List["DatasetRecordNew"]] = None, + description: str = "", + project_name: Optional[str] = None, + **kwargs, + ) -> "Dataset": + return self._require_llmobs_app_key().create_dataset( + dataset_name=name, + project_name=self._project_name(project_name), + description=description, + records=records, + **kwargs, + ) + + def create_dataset_from_csv( + self, + csv_path: str, + name: str, + input_data_columns: List[str], + expected_output_columns: Optional[List[str]] = None, + description: str = "", + project_name: Optional[str] = None, + **kwargs, + ) -> "Dataset": + return self._require_llmobs_app_key().create_dataset_from_csv( + csv_path=csv_path, + dataset_name=name, + input_data_columns=input_data_columns, + expected_output_columns=expected_output_columns, + description=description, + project_name=self._project_name(project_name), + **kwargs, + ) + + def pull_dataset( + self, + name: str, + project_name: Optional[str] = None, + version: Optional[int] = None, + tags: Optional[List[str]] = None, + ) -> "Dataset": + return self._require_llmobs_app_key().pull_dataset( + dataset_name=name, + project_name=self._project_name(project_name), + version=version, + tags=tags, + ) + + # -- LLM Observability experiments -------------------------------- + + def experiment( + self, + name: str, + task: Task, + dataset: "Dataset", + evaluators: Sequence[Evaluator], + description: str = "", + project_name: Optional[str] = None, + **kwargs, + ) -> "SyncExperiment": + """Build an experiment. Call ``.run()`` on it, or use ``run_experiment``.""" + return self._require_llmobs_app_key().experiment( + name=name, + task=task, + dataset=dataset, + evaluators=evaluators, + description=description, + project_name=self._project_name(project_name), + **kwargs, + ) + + def async_experiment( + self, + name: str, + task: Callable[..., Awaitable[Any]], + dataset: "Dataset", + evaluators: Sequence[Evaluator], + description: str = "", + project_name: Optional[str] = None, + **kwargs, + ) -> "Experiment": + """``experiment`` for a coroutine task.""" + return self._require_llmobs_app_key().async_experiment( + name=name, + task=task, + dataset=dataset, + evaluators=evaluators, + description=description, + project_name=self._project_name(project_name), + **kwargs, + ) + + def run_experiment( + self, + name: str, + task: Task, + dataset: "Dataset", + evaluators: Sequence[Evaluator], + jobs: int = 1, + raise_errors: bool = False, + sample_size: Optional[int] = None, + **kwargs, + ) -> "ExperimentResult": + """Build and run an experiment in one call, returning its results.""" + experiment = self.experiment( + name=name, + task=task, + dataset=dataset, + evaluators=evaluators, + **kwargs, + ) + return experiment.run( + jobs=jobs, raise_errors=raise_errors, sample_size=sample_size + ) + + def pull_experiment(self, experiment_id: str) -> "SyncExperiment": + return self._require_llmobs_app_key().pull_experiment(experiment_id) + + def _project_name(self, project_name=None): + # type: (Optional[str]) -> str + return project_name or self._config.llmobs_project_name + def _dd_log(self, log_level, msg, tags=_sentinel): # TODO: timestamp log = { diff --git a/datadog/llmobs.py b/datadog/llmobs.py new file mode 100644 index 0000000..e9bcf1a --- /dev/null +++ b/datadog/llmobs.py @@ -0,0 +1,63 @@ +"""Types for writing evaluators, datasets and experiments. + +Everything here is re-exported from ddtrace so that user code annotating an +evaluator or building dataset records has a single import surface:: + + from datadog.llmobs import EvaluatorContext, EvaluatorResult, LLMJudge + +This module is deliberately not imported by ``datadog/__init__.py``: pulling +in ``ddtrace.llmobs`` has a cost, and a client with LLM Obs turned off should +not pay it. +""" + +from ddtrace.llmobs import BaseAsyncEvaluator +from ddtrace.llmobs import BaseAsyncSummaryEvaluator +from ddtrace.llmobs import BaseEvaluator +from ddtrace.llmobs import BaseSummaryEvaluator +from ddtrace.llmobs import BooleanStructuredOutput +from ddtrace.llmobs import CategoricalStructuredOutput +from ddtrace.llmobs import Dataset +from ddtrace.llmobs import DatasetRecord +from ddtrace.llmobs import EvaluatorContext +from ddtrace.llmobs import EvaluatorResult +from ddtrace.llmobs import LLMJudge +from ddtrace.llmobs import MultiEvaluatorResult +from ddtrace.llmobs import Prompt +from ddtrace.llmobs import RemoteEvaluator +from ddtrace.llmobs import RemoteEvaluatorError +from ddtrace.llmobs import ScoreStructuredOutput +from ddtrace.llmobs import SummaryEvaluatorContext +from ddtrace.llmobs.evaluators import JSONEvaluator +from ddtrace.llmobs.evaluators import LengthEvaluator +from ddtrace.llmobs.evaluators import RegexMatchEvaluator +from ddtrace.llmobs.evaluators import SemanticSimilarityEvaluator +from ddtrace.llmobs.evaluators import StringCheckEvaluator + +__all__ = [ + # writing your own evaluator + "BaseEvaluator", + "BaseAsyncEvaluator", + "BaseSummaryEvaluator", + "BaseAsyncSummaryEvaluator", + "EvaluatorContext", + "EvaluatorResult", + "MultiEvaluatorResult", + "SummaryEvaluatorContext", + # llm-as-a-judge + "LLMJudge", + "BooleanStructuredOutput", + "CategoricalStructuredOutput", + "ScoreStructuredOutput", + "RemoteEvaluator", + "RemoteEvaluatorError", + # built-in evaluators + "JSONEvaluator", + "LengthEvaluator", + "RegexMatchEvaluator", + "SemanticSimilarityEvaluator", + "StringCheckEvaluator", + # datasets + "Dataset", + "DatasetRecord", + "Prompt", +] diff --git a/examples/evals.py b/examples/evals.py new file mode 100644 index 0000000..087c44f --- /dev/null +++ b/examples/evals.py @@ -0,0 +1,123 @@ +"""LLM Observability evaluations. + +Two halves, and they need different credentials: + + * submitting evaluation metrics against spans your app produced needs only + an api key (plus an agent, or agentless mode):: + + DD_API_KEY=... python examples/evals.py + + * datasets and experiments talk to the Datadog API directly and also need + an app key:: + + DD_API_KEY=... DD_APP_KEY=... python examples/evals.py +""" + +import os + +from datadog import DDClient, DDConfig +from datadog.llmobs import BaseEvaluator, EvaluatorContext, EvaluatorResult +from datadog.llmobs import StringCheckEvaluator + +ddcfg = DDConfig( + service="evals-demo", + env="dev", + version="0.0.1", + llmobs_enabled=True, + llmobs_project_name="evals-demo", + # llmobs_app_key="...", # or DD_APP_KEY, needed below the fold + # llmobs_agentless_enabled=True, +) +ddclient = DDClient(config=ddcfg) + + +# -- evaluating spans as they happen --------------------------------------- +# metric_type is inferred from the value and the evaluation attaches to the +# span currently being traced, so scoring a workflow is a one-liner. + + +@ddclient.workflow(name="rag.answer") +def answer(question: str) -> str: + output = "Scranton is in Pennsylvania." + ddclient.annotate(input_data=question, output_data=output) + + ddclient.submit_evaluation("relevance", 0.92) # score + ddclient.submit_evaluation("tone", "formal") # categorical + ddclient.submit_evaluation( # boolean, with the reasoning behind it + "grounded", + True, + assessment="pass", + reasoning="every claim appears in the retrieved docs", + tags={"evaluator": "self"}, + ) + return output + + +# A span from another process can be joined after the fact by handing back +# the dict export_span() produced, or by a tag the span carries: +# +# ddclient.submit_evaluation("thumbs_up", True, span=exported) +# ddclient.submit_evaluation( +# "thumbs_up", True, span_with_tag_value={"tag_key": "request_id", +# "tag_value": "abc123"} +# ) + + +# -- experiments ----------------------------------------------------------- +# An experiment runs a task over every record of a dataset and scores the +# results with evaluators. Evaluators are plain callables or BaseEvaluator +# subclasses; ddtrace ships a handful of built-ins. + + +class AnswerLength(BaseEvaluator): + def __init__(self, limit: int = 80): + super().__init__(name="answer_length") + self.limit = limit + + def evaluate(self, context: EvaluatorContext): + length = len(str(context.output_data)) + return EvaluatorResult( + value=length, + assessment="pass" if length <= self.limit else "fail", + reasoning="%d characters (limit %d)" % (length, self.limit), + ) + + +def task(input_data, config): + return "%s is in Pennsylvania." % input_data["city"] + + +def run_experiment(): + dataset = ddclient.create_dataset( + name="cities", + description="where are these places", + records=[ + { + "input_data": {"city": "Scranton"}, + "expected_output": "Scranton is in Pennsylvania.", + }, + { + "input_data": {"city": "Bethlehem"}, + "expected_output": "Bethlehem is in Pennsylvania.", + }, + ], + ) + + result = ddclient.run_experiment( + name="city-lookup", + task=task, + dataset=dataset, + evaluators=[AnswerLength(), StringCheckEvaluator(operation="eq")], + ) + return result + + +if __name__ == "__main__": + print(answer("Where is Scranton?")) + + if os.getenv("DD_APP_KEY"): + print(run_experiment()) + else: + print("set DD_APP_KEY to also run the experiment half of this example") + + ddclient.flush() diff --git a/readme.md b/readme.md index 0bb88ca..4536b3f 100644 --- a/readme.md +++ b/readme.md @@ -39,6 +39,8 @@ ddcfg = DDConfig( runtime_metrics_enabled=True, llmobs_enabled=True, llmobs_ml_app="my-python-service", + llmobs_project_name="my-project", # experiments/datasets live in a project + # llmobs_app_key="...", # required by the eval APIs # llmobs_agentless_enabled=True, # send straight to Datadog if no local agent # llmobs_integrations_enabled=True, # auto-instrument openai/anthropic/etc. (default) ) @@ -76,9 +78,20 @@ ddclient.flush_profiles() @ddclient.llm(model_name="gpt-4o-mini") # decorator (also: .embedding, .retrieval, .llm_agent) ddclient.annotate(input_data=..., output_data=..., metadata=..., tags=...) ddclient.annotation_context(...) -ddclient.submit_evaluation(span_context=..., label=..., value=...) ddclient.export_span(span) ddclient.flush() # also flushes llm obs + +# evaluations +ddclient.submit_evaluation(label="relevance", value=0.9) +ddclient.get_spans(...) # find spans to evaluate offline +ddclient.publish_evaluator(evaluator) # let Datadog run it on live spans +ddclient.create_dataset(name=..., records=[...]) +ddclient.create_dataset_from_csv(csv_path=..., name=..., input_data_columns=[...]) +ddclient.pull_dataset(name=...) +ddclient.experiment(name=..., task=..., dataset=..., evaluators=[...]) +ddclient.async_experiment(...) # for a coroutine task +ddclient.run_experiment(...) # build + run, returns results +ddclient.pull_experiment(experiment_id) ``` @@ -136,6 +149,56 @@ Agent runner. See [`examples/llmobs.py`](examples/llmobs.py) for an end-to-end run. +### evaluations + +Scoring what the app produced is part of observing it, so evaluations are +client methods too. `metric_type` is inferred from the value and the +evaluation attaches to the span being traced, which makes the common case a +one-liner: + +```python +@ddclient.workflow(name="rag.answer") +def answer(question: str) -> str: + output = generate(question) + ddclient.submit_evaluation("relevance", 0.92) # score + ddclient.submit_evaluation("tone", "formal") # categorical + ddclient.submit_evaluation("grounded", True, # boolean + assessment="pass", + reasoning="every claim is in the docs") + return output +``` + +A span from another process is joined by handing back what `export_span()` +produced, or by a tag it carries (`span_with_tag_value=`). + +Experiments run a task over a dataset and score every row: + +```python +from datadog.llmobs import BaseEvaluator, EvaluatorContext, EvaluatorResult + + +class AnswerLength(BaseEvaluator): + def evaluate(self, context: EvaluatorContext): + return EvaluatorResult(value=len(str(context.output_data))) + + +dataset = ddclient.create_dataset(name="cities", records=[...]) +results = ddclient.run_experiment( + name="city-lookup", + task=task, + dataset=dataset, + evaluators=[AnswerLength()], +) +``` + +Datasets and experiments read and write through the Datadog API, so they +need an app key (`llmobs_app_key=` or `DD_APP_KEY`) on top of the api key, +and they organize under `llmobs_project_name=` (or `DD_LLMOBS_PROJECT_NAME`). +Evaluator base classes, the built-in evaluators, the llm-as-a-judge helpers +and the dataset types are re-exported from `datadog.llmobs`, so evaluator +code never imports `ddtrace` itself. See [`examples/evals.py`](examples/evals.py). + + ## open questions/concerns