Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat: add offline jev routing and guardrail evaluation runner (#8230)
Brooke Ballhaus5d58569ae2a2
diff --git a/lib/crewai/tests/test_jev_evaluation_runner.py b/lib/crewai/tests/test_jev_evaluation_runner.pynew file mode 100644index 000000000..9ca829461--- /dev/null+++ b/lib/crewai/tests/test_jev_evaluation_runner.py@@ -0,0 +1,655 @@+"""Offline checks for the Jev routing and guardrail evaluation runner."""++import importlib.util+import json+from pathlib import Path+import sys+from types import SimpleNamespace++import pytest+++def _load_runner():+ path = Path(__file__).resolve().parents[3] / "scripts" / "jev_evaluation_runner.py"+ spec = importlib.util.spec_from_file_location("jev_evaluation_runner", path)+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return module+++runner = _load_runner()+GuardrailRecord = runner.GuardrailRecord+RouteRecord = runner.RouteRecord+collect_live_decisions = runner.collect_live_decisions+evaluate = runner.evaluate+load_dataset = runner.load_dataset+++EXAMPLE = [+ {+ "id": "r1",+ "kind": "route",+ "state": "Refund for order 1234",+ "expected_route": "billing",+ "decision": {+ "choice": "billing",+ "confidence": 0.91,+ "probabilities": {"billing": 0.95, "tech": 0.05},+ },+ "metadata": {+ "latency_ms": 212.0,+ "usage": {"input_tokens": 40, "output_tokens": 1},+ },+ },+ {+ "id": "r2",+ "kind": "route",+ "state": "App crashes on login",+ "expected_route": "tech",+ "decision": {+ "choice": "tech",+ "confidence": 0.62,+ "probabilities": {"billing": 0.19, "tech": 0.81},+ },+ },+ {+ "id": "r3",+ "kind": "route",+ "state": "Change my plan",+ "expected_route": "billing",+ "decision": {+ "choice": "tech",+ "confidence": 0.88,+ "probabilities": {"billing": 0.06, "tech": 0.94},+ },+ },+ {+ "id": "r4",+ "kind": "route",+ "state": "Where is my invoice?",+ "expected_route": "billing",+ "decision": None,+ "error": "TypeSafeAPITimeoutError: request timed out",+ },+ {+ "id": "g1",+ "kind": "guardrail",+ "state": "Paris is the capital of France.",+ "expected": "accept",+ "decision": {"noul": 0.97},+ },+ {+ "id": "g2",+ "kind": "guardrail",+ "state": "The moon is made of cheese.",+ "expected": "reject",+ "decision": {"noul": 0.8},+ },+ {+ "id": "g3",+ "kind": "guardrail",+ "state": "Water boils at 100 C at sea level.",+ "expected": "accept",+ "decision": {"noul": 0.79},+ },+ {+ "id": "g4",+ "kind": "guardrail",+ "state": "2 + 2 = 5",+ "expected": "reject",+ "decision": {"noul": 1.4},+ },+]+++def write_jsonl(tmp_path, records, name="data.jsonl"):+ path = tmp_path / name+ path.write_text("".join(json.dumps(r) + "\n" for r in records), encoding="utf-8")+ return path+++def route(decision, expected="billing", **extra):+ return RouteRecord(+ id="r",+ kind="route",+ state="s",+ expected_route=expected,+ decision=decision,+ **extra,+ )+++def guardrail(decision, expected="accept", **extra):+ return GuardrailRecord(+ id="g",+ kind="guardrail",+ state="s",+ expected=expected,+ decision=decision,+ **extra,+ )+++def outcome_of(record, **thresholds):+ kwargs = {+ "min_confidence": 0.7,+ "noul_threshold": 0.8,+ "routes": {"billing", "tech"},+ } | thresholds+ return evaluate([record], **kwargs).outcomes[0]+++# Metric denominators+++def test_example_dataset_report(tmp_path):+ report = evaluate(+ load_dataset(write_jsonl(tmp_path, EXAMPLE)),+ min_confidence=0.7,+ noul_threshold=0.8,+ )++ assert report.routing.model_dump() == {+ "total": 4,+ "decided": 2,+ "abstained": 1,+ "failed": 1,+ "correct": 1,+ "accuracy": 0.25,+ "coverage": 0.5,+ "abstention_rate": 0.25,+ }+ assert report.guardrail.model_dump() == {+ "total": 4,+ "decided": 3,+ "failed": 1,+ "accepted": 2,+ "rejected": 1,+ "false_accepts": 1,+ "false_accept_denominator": 1,+ "false_accept_rate": 1.0,+ "false_rejects": 1,+ "false_reject_denominator": 2,+ "false_reject_rate": 0.5,+ }+ assert report.usage.model_dump() == {+ "input_tokens": 40,+ "input_token_records": 1,+ "output_tokens": 1,+ "output_token_records": 1,+ }+ assert report.latency.model_dump() == {"count": 1, "mean_ms": 212.0}+ assert [o.outcome for o in report.outcomes] == [+ "correct",+ "abstained",+ "incorrect",+ "failed",+ "accepted",+ "accepted",+ "rejected",+ "failed",+ ]+++def test_failed_guardrails_are_excluded_from_false_rate_denominators():+ records = [+ guardrail(None, expected="reject"),+ guardrail({"noul": 0.9}, expected="accept"),+ ]+ report = evaluate(records, min_confidence=0.5, noul_threshold=0.8)++ assert report.guardrail.failed == 1+ assert report.guardrail.false_accept_denominator == 0+ assert report.guardrail.false_accept_rate is None+ assert report.guardrail.false_reject_denominator == 1+ assert report.guardrail.false_reject_rate == 0.0+++def test_empty_dataset_reports_none_rates():+ report = evaluate([], min_confidence=0.5, noul_threshold=0.5)++ assert report.routing.accuracy is None+ assert report.routing.coverage is None+ assert report.guardrail.false_reject_rate is None+ assert report.latency.mean_ms is None+++# Threshold boundaries+++ ("confidence", "expected"),+ [(0.69, "abstained"), (0.7, "correct"), (0.71, "correct")],+)+def test_choice_threshold_is_inclusive(confidence, expected):+ record = route({"choice": "billing", "confidence": confidence, "probabilities": {}})++ assert outcome_of(record, min_confidence=0.7).outcome == expected+++ ("noul", "threshold", "expected"),+ [+ (0.79, 0.8, "rejected"),+ (0.8, 0.8, "accepted"),+ (0.81, 0.8, "accepted"),+ (0.0, 0.0, "accepted"),+ (1.0, 1.0, "accepted"),+ ],+)+def test_noul_threshold_is_inclusive(noul, threshold, expected):+ assert (+ outcome_of(guardrail({"noul": noul}), noul_threshold=threshold).outcome+ == expected+ )++[email protected]("bad", [float("nan"), float("inf"), -0.1, 1.1, True, "0.5"])+def test_invalid_thresholds_raise(bad):+ with pytest.raises(ValueError, match="noul_threshold"):+ evaluate([], min_confidence=0.5, noul_threshold=bad)+ with pytest.raises(ValueError, match="min_confidence"):+ evaluate([], min_confidence=bad, noul_threshold=0.5)+++# Missing and invalid decisions+++ ("decision", "extra", "reason"),+ [+ (None, {}, "missing decision"),+ (+ {"choice": "billing", "confidence": 0.9, "probabilities": {}},+ {"error": "boom"},+ "recorded error: boom",+ ),+ (+ {"choice": "sales", "confidence": 0.9, "probabilities": {}},+ {},+ "not a known route",+ ),+ (+ {"choice": "billing", "confidence": 1.2, "probabilities": {}},+ {},+ "invalid ChoiceDecision: confidence",+ ),+ (+ {"choice": "billing", "confidence": "0.9", "probabilities": {}},+ {},+ "invalid ChoiceDecision: confidence",+ ),+ (+ {"choice": "billing", "probabilities": {}},+ {},+ "invalid ChoiceDecision: confidence",+ ),+ (+ {"choice": "billing", "confidence": 0.9, "probabilities": {"billing": 1.5}},+ {},+ "invalid ChoiceDecision: probabilities.billing",+ ),+ ],+)+def test_unusable_route_decisions_fail(decision, extra, reason):+ result = outcome_of(route(decision, **extra))++ assert result.outcome == "failed"+ assert reason in result.reason+++ "decision",+ [+ None,+ {"noul": -0.1},+ {"noul": 1.4},+ {"noul": True},+ {"noul": "0.9"},+ {"confidence": 0.9},+ ],+)+def test_unusable_guardrail_decisions_fail_and_never_accept(decision):+ assert outcome_of(guardrail(decision, expected="reject")).outcome == "failed"+++def test_recorded_guardrail_error_fails_even_with_a_passing_decision():+ result = outcome_of(guardrail({"noul": 0.99}, error="boom"))++ assert (result.outcome, result.reason) == ("failed", "recorded error: boom")+++def test_route_confidence_is_never_used_as_noul():+ record = guardrail(+ {"choice": "accept", "confidence": 0.99, "probabilities": {"accept": 1.0}}+ )++ assert outcome_of(record).outcome == "failed"+++# Invalid inputs+++ ("lines", "message"),+ [+ (['{"id": "a"', ""], "line 1"),+ (['["not", "an", "object"]'], "Invalid record at line 1"),+ (+ ['{"kind": "route", "state": "s", "expected_route": "b"}'],+ "Invalid record at line 1",+ ),+ (['{"id": "a", "kind": "other", "state": "s"}'], "Invalid record at line 1"),+ (+ ['{"id": "a", "kind": "guardrail", "state": "s", "expected": "maybe"}'],+ "Invalid record at line 1",+ ),+ (+ [json.dumps(EXAMPLE[0]), json.dumps(EXAMPLE[0])],+ "Duplicate record id 'r1' at line 2",+ ),+ (+ [+ '{"id": "a", "kind": "route", "state": "s", "expected_route": "b", "notes": "x"}'+ ],+ "Invalid record at line 1",+ ),+ (+ ['{"id": "a", "kind": "route", "expected_route": "b"}'],+ "Invalid record at line 1",+ ),+ (+ [+ '{"id": "a", "kind": "route", "state": "s", "expected_route": "b", "metadata": {"latency_ms": -1}}'+ ],+ "Invalid record at line 1",+ ),+ ],+)+def test_malformed_jsonl_raises_with_location(tmp_path, lines, message):+ path = tmp_path / "bad.jsonl"+ path.write_text("\n".join(lines), encoding="utf-8")++ with pytest.raises(ValueError, match=message):+ load_dataset(path)++[email protected]("name", ["data.json", "data.jsonl"])+def test_json_and_jsonl_with_bom_and_blank_lines_are_accepted(tmp_path, name):+ content = (+ json.dumps(EXAMPLE)+ if name.endswith(".json")+ else "\n\n".join(json.dumps(r) for r in EXAMPLE)+ )+ (tmp_path / name).write_text(content, encoding="utf-8-sig")++ assert [r.id for r in load_dataset(tmp_path / name)] == [r["id"] for r in EXAMPLE]+++def test_jsonl_strings_may_contain_unicode_line_separators(tmp_path):+ record = {"id": "a", "kind": "route", "state": "a\u2028b\x85c", "expected_route": "b"}+ path = tmp_path / "data.jsonl"+ path.write_text(json.dumps(record, ensure_ascii=False), encoding="utf-8")++ assert [r.state for r in load_dataset(path)] == ["a\u2028b\x85c"]+++ ("name", "content", "message"),+ [+ ("data.json", "{}", "must be an array"),+ ("data.json", "[1,", "Invalid JSON"),+ ("data.json", '[{"id": "a"}]', "Invalid record at index 0"),+ ("data.csv", "", "Unsupported"),+ ],+)+def test_wrong_json_shape_or_extension_raises(tmp_path, name, content, message):+ (tmp_path / name).write_text(content, encoding="utf-8")++ with pytest.raises(ValueError, match=message):+ load_dataset(tmp_path / name)+++# Command line++CLI_THRESHOLDS = ["--min-confidence", "0.7", "--noul-threshold", "0.8"]+++def test_main_prints_the_report_as_json(tmp_path, capsys):+ path = write_jsonl(tmp_path, EXAMPLE)++ assert runner.main([str(path), *CLI_THRESHOLDS]) == 0+ expected = evaluate(load_dataset(path), min_confidence=0.7, noul_threshold=0.8)+ assert json.loads(capsys.readouterr().out) == expected.model_dump(mode="json")+++def test_main_reports_invalid_input_as_a_usage_error(tmp_path, capsys):+ path = tmp_path / "bad.jsonl"+ path.write_text("{", encoding="utf-8")++ with pytest.raises(SystemExit) as exit_info:+ runner.main([str(path), *CLI_THRESHOLDS])++ assert exit_info.value.code == 2+ assert "Invalid JSONL at line 1" in capsys.readouterr().err+++# Usage and latency+++def test_missing_usage_and_latency_are_not_counted_as_zero():+ records = [+ guardrail(+ {"noul": 0.9}, metadata={"usage": {"input_tokens": 10}, "latency_ms": 100.0}+ ),+ guardrail({"noul": 0.9}, metadata={"usage": {"output_tokens": 2}}),+ guardrail({"noul": 0.9}),+ ]+ report = evaluate(records, min_confidence=0.5, noul_threshold=0.5)++ assert report.usage.model_dump() == {+ "input_tokens": 10,+ "input_token_records": 1,+ "output_tokens": 2,+ "output_token_records": 1,+ }+ assert report.latency.model_dump() == {"count": 1, "mean_ms": 100.0}+++# Live collection with a fake client+++class FakeClient:+ def __init__(self, *results):+ self.results = list(results)+ self.calls = []+ self.closed = False++ def system_one(self, state, questions, *, model=None):+ self.calls.append((state, questions, model))+ result = self.results.pop(0)+ if isinstance(result, Exception):+ raise result+ return result++ def close(self):+ self.closed = True+++LIVE_KWARGS = {+ "route_instructions": "Pick a team",+ "route_criteria": {"billing": "Payments", "tech": "Bugs"},+ "guardrail_instructions": "Is it true?",+}+++def test_live_collection_builds_questions_and_records_metadata(monkeypatch):+ ticks = iter([1.0, 1.25, 2.0, 2.5])+ monkeypatch.setattr(+ runner, "time", SimpleNamespace(perf_counter=lambda: next(ticks))+ )+ choice = SimpleNamespace(+ choice="tech", confidence=0.8, probabilities={"billing": 0.1, "tech": 0.9}+ )+ client = FakeClient(+ SimpleNamespace(+ model="jev-1",+ usage=SimpleNamespace(input_tokens=5, output_tokens=1),+ answers={"route": choice},+ ),+ SimpleNamespace(+ model="jev-1", usage=None, answers={"guardrail": SimpleNamespace(noul=0.9)}+ ),+ )+ records = collect_live_decisions(+ [route(None), guardrail(None)], client=client, model="jev", **LIVE_KWARGS+ )++ assert client.calls[0] == (+ "s",+ {+ "route": {+ "type": "choice",+ "instructions": "Pick a team",+ "criteria": {"billing": "Payments", "tech": "Bugs"},+ }+ },+ "jev",+ )+ assert client.calls[1] == (+ "s",+ {"guardrail": {"type": "noul", "instructions": "Is it true?"}},+ "jev",+ )+ assert records[0].decision == {+ "choice": "tech",+ "confidence": 0.8,+ "probabilities": {"billing": 0.1, "tech": 0.9},+ }+ assert records[0].metadata.model_dump() == {+ "model": "jev-1",+ "usage": {"input_tokens": 5, "output_tokens": 1},+ "latency_ms": 250.0,+ }+ assert records[1].decision == {"noul": 0.9}+ assert records[1].metadata.latency_ms == 500.0+ assert not client.closed+++def test_live_client_errors_become_failed_records():+ client = FakeClient(RuntimeError("timeout"), SimpleNamespace(answers={}))+ records = collect_live_decisions(+ [guardrail(None), guardrail(None)], client=client, **LIVE_KWARGS+ )+ report = evaluate(records, min_confidence=0.5, noul_threshold=0.5)++ assert records[0].error == "RuntimeError: timeout"+ assert records[0].metadata is None+ assert records[1].error == "no answer returned for 'guardrail'"+ assert [o.outcome for o in report.outcomes] == ["failed", "failed"]+++ ("response", "error"),+ [+ (SimpleNamespace(answers=["guardrail"]), "no answer returned for 'guardrail'"),+ (+ SimpleNamespace(+ usage=SimpleNamespace(input_tokens=-1, output_tokens=1),+ answers={"guardrail": SimpleNamespace(noul=0.9)},+ ),+ "invalid usage: input_tokens: Input should be greater than or equal to 0",+ ),+ ],+)+def test_malformed_live_responses_become_failed_records(response, error):+ [record] = collect_live_decisions(+ [guardrail(None)], client=FakeClient(response), **LIVE_KWARGS+ )++ assert record.error == error+ assert record.decision is None+ assert record.metadata.latency_ms is not None+++ ("record", "answers", "error"),+ [+ (+ guardrail(None),+ {+ "guardrail": SimpleNamespace(+ type="choice", choice="tech", confidence=0.9, probabilities={}+ )+ },+ "expected a 'noul' answer for 'guardrail', got 'choice'",+ ),+ (+ guardrail(None),+ {"guardrail": SimpleNamespace(type="choice", choice="tech", noul=0.9)},+ "expected a 'noul' answer for 'guardrail', got 'choice'",+ ),+ (+ route(None),+ {"route": SimpleNamespace(type="noul", noul=0.9)},+ "expected a 'choice' answer for 'route', got 'noul'",+ ),+ ],+)+def test_live_answer_of_the_wrong_type_fails(record, answers, error):+ client = FakeClient(SimpleNamespace(answers=answers))+ [record] = collect_live_decisions([record], client=client, **LIVE_KWARGS)++ assert record.error == error+ assert record.decision is None+ assert outcome_of(record).outcome == "failed"++[email protected]("probabilities", [42, 0, False, "", []])+def test_malformed_probabilities_fail_without_stopping_collection(probabilities):+ bad = SimpleNamespace(choice="tech", confidence=0.9, probabilities=probabilities)+ good = SimpleNamespace(choice="tech", confidence=0.9, probabilities={"tech": 0.9})+ client = FakeClient(+ SimpleNamespace(answers={"route": bad}),+ SimpleNamespace(answers={"route": good}),+ )+ records = collect_live_decisions(+ [route(None, expected="tech"), route(None, expected="tech")],+ client=client,+ **LIVE_KWARGS,+ )++ assert [outcome_of(r).outcome for r in records] == ["failed", "correct"]+ assert "probabilities" in outcome_of(records[0]).reason+++def test_live_answer_without_probabilities_or_usage():+ answer = SimpleNamespace(choice="tech", confidence=0.8, probabilities=None)+ client = FakeClient(SimpleNamespace(answers={"route": answer}))+ [record] = collect_live_decisions([route(None)], client=client, **LIVE_KWARGS)++ assert record.decision["probabilities"] == {}+ assert record.metadata.usage is None+ assert record.metadata.model is None+++ "override",+ [+ {"route_instructions": " "},+ {"guardrail_instructions": ""},+ {"route_criteria": {}},+ ],+)+def test_live_collection_requires_questions(override):+ with pytest.raises(ValueError):+ collect_live_decisions([], client=FakeClient(), **(LIVE_KWARGS | override))+++def test_live_collection_skips_recorded_decisions():+ client = FakeClient()+ records = [guardrail({"noul": 0.9}), guardrail(None, error="earlier failure")]++ assert collect_live_decisions(records, client=client, **LIVE_KWARGS) == records+ assert client.calls == []diff --git a/scripts/jev_evaluation_runner.py b/scripts/jev_evaluation_runner.pynew file mode 100644index 000000000..c5ba02d9f--- /dev/null+++ b/scripts/jev_evaluation_runner.py@@ -0,0 +1,602 @@+# ruff: noqa: T201+"""Offline evaluation of Jev routing (Choice) and guardrail (Noul) decisions.++Usage:+ uv run python scripts/jev_evaluation_runner.py examples.jsonl \\+ --min-confidence 0.7 --noul-threshold 0.8 [--route billing --route tech]++The dataset is a .jsonl file (one record per line; blank lines are skipped) or+a .json array, read as UTF-8 with an optional BOM. Malformed JSON, unknown+fields, and missing or duplicate ids are reported with the line or index:++ {"id": "r1", "kind": "route", "state": "Refund for order 1234", "expected_route": "billing", "decision": {"choice": "billing", "confidence": 0.91, "probabilities": {"billing": 0.95, "tech": 0.05}}}+ {"id": "g1", "kind": "guardrail", "state": "Paris is the capital of France.", "expected": "accept", "decision": {"noul": 0.97}}++Route records have expected_route; guardrail records have expected ("accept"+or "reject"). Both can have an error (a recorded failure) and metadata (model,+usage, latency_ms). The report is printed as JSON.++A record fails if it has a recorded error, no decision, or a malformed decision+(a missing field, or a value that isn't a finite number in [0, 1]); a route+also fails if its choice isn't a known route (every expected_route in the+dataset, or the --route values). Failed records are never counted as correct or+accepted. Routes with confidence >= min_confidence are decided (correct or+incorrect), lower ones abstain; guardrails with noul >= noul_threshold are+accepted, lower ones rejected. Choice confidence and Noul probability are never+converted into each other.++- routing accuracy / coverage / abstention_rate: correct / decided / abstained+ over all route records.+- guardrail false_accept_rate / false_reject_rate: accepted / rejected over+ decided records expected to be rejected / accepted.+- usage, latency: token sums and mean latency_ms over only the records that+ report them.++A rate is null when its denominator is 0. These are decision-quality metrics,+not a replay of a runtime guardrail: a guardrail that rejects when the answer+is missing or the call fails would count those cases as rejections, while here+they are failed and left out of the false-accept and false-reject rates.++Live collection is opt-in and needs your own TypeSafe client, which you close:++ with TypeSafeClient(api_key="...") as client:+ records = collect_live_decisions(+ load_dataset("unlabeled.jsonl"),+ client=client,+ route_instructions="Which team should handle this ticket?",+ route_criteria={"billing": "Payments", "tech": "Bugs and outages"},+ guardrail_instructions="Is this statement true?",+ )++It times each call, stores client errors on the record (which then fails), and+fails an answer whose type doesn't match the question. The type is not stored,+so recorded decisions are checked by their fields only.+"""++from __future__ import annotations++import argparse+from collections import Counter+from collections.abc import Collection, Mapping, Sequence+import json+from pathlib import Path+import time+from typing import Annotated, Any, Literal, Protocol++from pydantic import (+ BaseModel,+ ConfigDict,+ Field,+ TypeAdapter,+ ValidationError,+)+++Probability = Annotated[float, Field(ge=0.0, le=1.0, allow_inf_nan=False)]+_THRESHOLD: TypeAdapter[float] = TypeAdapter(+ Probability, config=ConfigDict(strict=True)+)++ROUTE_QUESTION = "route"+GUARDRAIL_QUESTION = "guardrail"+++class ChoiceDecision(BaseModel):+ """A Jev Choice answer, mirroring the SDK's ``ChoiceAnswer`` fields."""++ model_config = ConfigDict(strict=True, extra="ignore")++ choice: str+ confidence: Probability+ probabilities: dict[str, Probability]+++class NoulDecision(BaseModel):+ """A Jev Noul answer: the probability that the question is true."""++ model_config = ConfigDict(strict=True, extra="ignore")++ noul: Probability+++class TokenUsage(BaseModel):+ """Token counts as reported by the SDK; ``None`` means not reported."""++ model_config = ConfigDict(extra="forbid")++ input_tokens: int | None = Field(default=None, ge=0)+ output_tokens: int | None = Field(default=None, ge=0)+++class DecisionMetadata(BaseModel):+ """Request metadata kept only when it was actually recorded."""++ model_config = ConfigDict(extra="forbid")++ model: str | None = None+ usage: TokenUsage | None = None+ latency_ms: float | None = Field(default=None, ge=0.0, allow_inf_nan=False)+++class _BaseRecord(BaseModel):+ model_config = ConfigDict(extra="forbid")++ id: str = Field(min_length=1)+ state: Any+ decision: dict[str, Any] | None = None+ error: str | None = None+ metadata: DecisionMetadata | None = None+++class RouteRecord(_BaseRecord):+ """A labeled routing example with an optional recorded Choice decision."""++ kind: Literal["route"]+ expected_route: str = Field(min_length=1)+++class GuardrailRecord(_BaseRecord):+ """A labeled guardrail example with an optional recorded Noul decision."""++ kind: Literal["guardrail"]+ expected: Literal["accept", "reject"]+++EvaluationRecord = Annotated[RouteRecord | GuardrailRecord, Field(discriminator="kind")]+_RECORD_ADAPTER: TypeAdapter[RouteRecord | GuardrailRecord] = TypeAdapter(+ EvaluationRecord+)++Outcome = Literal["correct", "incorrect", "abstained", "accepted", "rejected", "failed"]+++class RecordOutcome(BaseModel):+ """How a single record was classified, and why."""++ id: str+ kind: Literal["route", "guardrail"]+ outcome: Outcome+ reason: str+++class RoutingReport(BaseModel):+ """Routing counts and rates; every rate is over all routing records."""++ total: int+ decided: int+ abstained: int+ failed: int+ correct: int+ accuracy: float | None+ coverage: float | None+ abstention_rate: float | None+++class GuardrailReport(BaseModel):+ """Guardrail counts; error rates are over decided records only."""++ total: int+ decided: int+ failed: int+ accepted: int+ rejected: int+ false_accepts: int+ false_accept_denominator: int+ false_accept_rate: float | None+ false_rejects: int+ false_reject_denominator: int+ false_reject_rate: float | None+++class UsageSummary(BaseModel):+ """Token totals over the records that reported each count."""++ input_tokens: int+ input_token_records: int+ output_tokens: int+ output_token_records: int+++class LatencySummary(BaseModel):+ """Mean latency over records with a measured latency."""++ count: int+ mean_ms: float | None+++class EvaluationReport(BaseModel):+ """Full evaluation result."""++ routing: RoutingReport+ guardrail: GuardrailReport+ usage: UsageSummary+ latency: LatencySummary+ outcomes: list[RecordOutcome]+++class SystemOneClient(Protocol):+ """The part of the TypeSafe client used for live collection."""++ def system_one(+ self, state: Any, questions: Mapping[str, Any], *, model: str | None = None+ ) -> Any:+ """Answer the named questions about ``state``."""+ ...+++def load_dataset(path: str | Path) -> list[RouteRecord | GuardrailRecord]:+ """Load and validate records from a ``.jsonl`` or ``.json`` file.++ Args:+ path: A ``.jsonl`` file (one record per line; blank lines are skipped)+ or a ``.json`` file containing an array of records.++ Returns:+ The validated records, in file order.++ Raises:+ ValueError: If the file or any record is malformed.+ """+ path = Path(path)+ text = path.read_text(encoding="utf-8-sig")+ if path.suffix == ".jsonl":+ items = _parse_jsonl(text)+ elif path.suffix == ".json":+ try:+ data = json.loads(text)+ except ValueError as e:+ raise ValueError(f"Invalid JSON: {e}") from e+ if not isinstance(data, list):+ raise ValueError("A .json dataset must be an array of records")+ items = [(f"index {i}", item) for i, item in enumerate(data)]+ else:+ raise ValueError(f"Unsupported dataset extension: {path.suffix!r}")++ records: list[RouteRecord | GuardrailRecord] = []+ seen: set[str] = set()+ for location, item in items:+ try:+ record = _RECORD_ADAPTER.validate_python(item)+ except ValidationError as e:+ raise ValueError(f"Invalid record at {location}: {e}") from e+ if record.id in seen:+ raise ValueError(f"Duplicate record id {record.id!r} at {location}")+ seen.add(record.id)+ records.append(record)+ return records+++def evaluate(+ records: Sequence[RouteRecord | GuardrailRecord],+ *,+ min_confidence: float,+ noul_threshold: float,+ routes: Collection[str] | None = None,+) -> EvaluationReport:+ """Classify every record and compute routing and guardrail metrics.++ Both thresholds are inclusive: a Choice confidence equal to+ ``min_confidence`` is decided, and a Noul probability equal to+ ``noul_threshold`` is accepted.++ Args:+ records: Records to evaluate.+ min_confidence: Minimum Choice confidence for a route to be decided.+ noul_threshold: Minimum Noul probability for an output to be accepted.+ routes: Known route labels. Defaults to every ``expected_route``.++ Returns:+ Counts, rates, usage, latency, and a per-record outcome with its reason.++ Raises:+ ValueError: If a threshold is not a finite number in [0, 1].+ """+ _validate_threshold("min_confidence", min_confidence)+ _validate_threshold("noul_threshold", noul_threshold)+ route_records = [r for r in records if isinstance(r, RouteRecord)]+ known_routes = (+ set(routes) if routes is not None else {r.expected_route for r in route_records}+ )++ outcomes: list[RecordOutcome] = []+ false_accepts = false_rejects = fa_denominator = fr_denominator = 0+ for record in records:+ if isinstance(record, RouteRecord):+ outcome, reason = _classify_route(record, known_routes, min_confidence)+ else:+ outcome, reason = _classify_guardrail(record, noul_threshold)+ if outcome != "failed" and record.expected == "reject":+ fa_denominator += 1+ if outcome == "accepted":+ false_accepts += 1+ elif outcome != "failed":+ fr_denominator += 1+ if outcome == "rejected":+ false_rejects += 1+ outcomes.append(+ RecordOutcome(+ id=record.id, kind=record.kind, outcome=outcome, reason=reason+ )+ )++ route_counts = Counter(o.outcome for o in outcomes if o.kind == "route")+ guard_counts = Counter(o.outcome for o in outcomes if o.kind == "guardrail")+ route_total = len(route_records)+ route_decided = route_counts["correct"] + route_counts["incorrect"]+ guard_total = len(records) - route_total+ return EvaluationReport(+ routing=RoutingReport(+ total=route_total,+ decided=route_decided,+ abstained=route_counts["abstained"],+ failed=route_counts["failed"],+ correct=route_counts["correct"],+ accuracy=_rate(route_counts["correct"], route_total),+ coverage=_rate(route_decided, route_total),+ abstention_rate=_rate(route_counts["abstained"], route_total),+ ),+ guardrail=GuardrailReport(+ total=guard_total,+ decided=guard_total - guard_counts["failed"],+ failed=guard_counts["failed"],+ accepted=guard_counts["accepted"],+ rejected=guard_counts["rejected"],+ false_accepts=false_accepts,+ false_accept_denominator=fa_denominator,+ false_accept_rate=_rate(false_accepts, fa_denominator),+ false_rejects=false_rejects,+ false_reject_denominator=fr_denominator,+ false_reject_rate=_rate(false_rejects, fr_denominator),+ ),+ usage=_summarize_usage(records),+ latency=_summarize_latency(records),+ outcomes=outcomes,+ )+++def collect_live_decisions(+ records: Sequence[RouteRecord | GuardrailRecord],+ *,+ client: SystemOneClient,+ route_instructions: str,+ route_criteria: Mapping[str, str],+ guardrail_instructions: str,+ model: str | None = None,+) -> list[RouteRecord | GuardrailRecord]:+ """Ask Jev for each record that has no recorded decision or error.++ Opt-in: the caller supplies a client built with their own credentials,+ and remains responsible for closing it. Client errors are recorded on the+ record (which then evaluates as failed) instead of stopping the run.++ Args:+ records: Records to complete; those with a decision or error are kept as is.+ client: A TypeSafe client, or any object with a compatible ``system_one``.+ route_instructions: Instructions for the Choice question on route records.+ route_criteria: Route labels mapped to their descriptions.+ guardrail_instructions: Instructions for the Noul question on guardrail records.+ model: Optional Jev model name passed to the client.++ Returns:+ The records with decisions, latency, and usage filled in where collected.++ Raises:+ ValueError: If instructions are blank or ``route_criteria`` is empty.+ """+ if not route_instructions.strip() or not guardrail_instructions.strip():+ raise ValueError("route_instructions and guardrail_instructions are required")+ if not route_criteria:+ raise ValueError("route_criteria must not be empty")++ collected: list[RouteRecord | GuardrailRecord] = []+ for record in records:+ if record.decision is not None or record.error is not None:+ collected.append(record)+ continue+ if isinstance(record, RouteRecord):+ name = ROUTE_QUESTION+ question: dict[str, Any] = {+ "type": "choice",+ "instructions": route_instructions,+ "criteria": dict(route_criteria),+ }+ else:+ name = GUARDRAIL_QUESTION+ question = {"type": "noul", "instructions": guardrail_instructions}+ collected.append(_query(record, client, name, question, model))+ return collected+++def _query(+ record: RouteRecord | GuardrailRecord,+ client: SystemOneClient,+ name: str,+ question: dict[str, Any],+ model: str | None,+) -> RouteRecord | GuardrailRecord:+ start = time.perf_counter()+ try:+ response = client.system_one(record.state, {name: question}, model=model)+ except Exception as e:+ return record.model_copy(update={"error": f"{type(e).__name__}: {e}"})+ latency_ms = (time.perf_counter() - start) * 1000++ usage = getattr(response, "usage", None)+ response_model = getattr(response, "model", None)+ metadata = DecisionMetadata(+ model=response_model if isinstance(response_model, str) else None,+ latency_ms=latency_ms,+ )+ if usage is not None:+ try:+ metadata.usage = TokenUsage.model_validate(usage, from_attributes=True)+ except ValidationError as e:+ first = e.errors()[0]+ location = ".".join(str(part) for part in first["loc"]) or "usage"+ return record.model_copy(+ update={+ "error": f"invalid usage: {location}: {first['msg']}",+ "metadata": metadata,+ }+ )+ answers = getattr(response, "answers", None)+ answer = answers.get(name) if isinstance(answers, Mapping) else None+ if answer is None:+ return record.model_copy(+ update={"error": f"no answer returned for {name!r}", "metadata": metadata}+ )+ answer_type = getattr(answer, "type", None)+ if answer_type is not None and answer_type != question["type"]:+ return record.model_copy(+ update={+ "error": f"expected a {question['type']!r} answer for {name!r},"+ f" got {answer_type!r}",+ "metadata": metadata,+ }+ )+ if isinstance(record, RouteRecord):+ probabilities = getattr(answer, "probabilities", None)+ if probabilities is None:+ probabilities = {}+ elif isinstance(probabilities, Mapping):+ probabilities = dict(probabilities)+ decision = {+ "choice": getattr(answer, "choice", None),+ "confidence": getattr(answer, "confidence", None),+ "probabilities": probabilities,+ }+ else:+ decision = {"noul": getattr(answer, "noul", None)}+ return record.model_copy(update={"decision": decision, "metadata": metadata})+++def _parse_decision(+ record: RouteRecord | GuardrailRecord,+ model: type[ChoiceDecision] | type[NoulDecision],+) -> tuple[ChoiceDecision | NoulDecision | None, str]:+ """Return the validated decision, or ``None`` and the failure reason."""+ if record.error is not None:+ return None, f"recorded error: {record.error}"+ if record.decision is None:+ return None, "missing decision"+ try:+ return model.model_validate(record.decision), ""+ except ValidationError as e:+ first = e.errors()[0]+ location = ".".join(str(part) for part in first["loc"]) or "decision"+ return None, f"invalid {model.__name__}: {location}: {first['msg']}"+++def _classify_route(+ record: RouteRecord, known_routes: set[str], min_confidence: float+) -> tuple[Outcome, str]:+ decision, reason = _parse_decision(record, ChoiceDecision)+ if not isinstance(decision, ChoiceDecision):+ return "failed", reason+ if decision.choice not in known_routes:+ return "failed", f"choice {decision.choice!r} is not a known route"+ if decision.confidence < min_confidence:+ return (+ "abstained",+ f"confidence {decision.confidence} below min_confidence {min_confidence}",+ )+ outcome: Outcome = (+ "correct" if decision.choice == record.expected_route else "incorrect"+ )+ return (+ outcome,+ f"confidence {decision.confidence} meets min_confidence {min_confidence}",+ )+++def _classify_guardrail(+ record: GuardrailRecord, noul_threshold: float+) -> tuple[Outcome, str]:+ decision, reason = _parse_decision(record, NoulDecision)+ if not isinstance(decision, NoulDecision):+ return "failed", reason+ if decision.noul >= noul_threshold:+ return "accepted", f"noul {decision.noul} meets threshold {noul_threshold}"+ return "rejected", f"noul {decision.noul} below threshold {noul_threshold}"+++def _summarize_usage(records: Sequence[RouteRecord | GuardrailRecord]) -> UsageSummary:+ usages = [r.metadata.usage for r in records if r.metadata and r.metadata.usage]+ inputs = [u.input_tokens for u in usages if u.input_tokens is not None]+ outputs = [u.output_tokens for u in usages if u.output_tokens is not None]+ return UsageSummary(+ input_tokens=sum(inputs),+ input_token_records=len(inputs),+ output_tokens=sum(outputs),+ output_token_records=len(outputs),+ )+++def _summarize_latency(+ records: Sequence[RouteRecord | GuardrailRecord],+) -> LatencySummary:+ values = [+ r.metadata.latency_ms+ for r in records+ if r.metadata and r.metadata.latency_ms is not None+ ]+ return LatencySummary(+ count=len(values), mean_ms=sum(values) / len(values) if values else None+ )+++def _parse_jsonl(text: str) -> list[tuple[str, Any]]:+ items: list[tuple[str, Any]] = []+ for number, line in enumerate(text.split("\n"), start=1):+ if not line.strip():+ continue+ try:+ items.append((f"line {number}", json.loads(line)))+ except ValueError as e:+ raise ValueError(f"Invalid JSONL at line {number}: {e}") from e+ return items+++def _validate_threshold(name: str, value: float) -> None:+ try:+ _THRESHOLD.validate_python(value)+ except ValidationError as e:+ raise ValueError(+ f"{name} must be a finite number in [0, 1], got {value!r}"+ ) from e+++def _rate(numerator: int, denominator: int) -> float | None:+ return numerator / denominator if denominator else None+++def main(argv: Sequence[str] | None = None) -> int:+ """Evaluate a dataset file and print the report as JSON."""+ parser = argparse.ArgumentParser(+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter+ )+ parser.add_argument("dataset", type=Path, help="a .jsonl or .json file")+ parser.add_argument("--min-confidence", type=float, required=True)+ parser.add_argument("--noul-threshold", type=float, required=True)+ parser.add_argument(+ "--route",+ action="append",+ dest="routes",+ help="a known route; repeat for each (default: every expected_route)",+ )+ args = parser.parse_args(argv)+ try:+ report = evaluate(+ load_dataset(args.dataset),+ min_confidence=args.min_confidence,+ noul_threshold=args.noul_threshold,+ routes=args.routes,+ )+ except (OSError, ValueError) as e:+ parser.error(str(e))+ print(report.model_dump_json(indent=2))+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())