Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat: add offline jev routing and guardrail evaluation runner (#8230)

Brooke Ballhaus5d58569ae2a2
diff --git a/lib/crewai/tests/test_jev_evaluation_runner.py b/lib/crewai/tests/test_jev_evaluation_runner.py
new file mode 100644
index 000000000..9ca829461
--- /dev/null
+++ b/lib/crewai/tests/test_jev_evaluation_runner.py
@@ -0,0 +1,655 @@
+"""Offline checks for the Jev routing and guardrail evaluation runner."""
+
+import importlib.util
+import json
+from pathlib import Path
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+
+def _load_runner():
+ path = Path(__file__).resolve().parents[3] / "scripts" / "jev_evaluation_runner.py"
+ spec = importlib.util.spec_from_file_location("jev_evaluation_runner", path)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+runner = _load_runner()
+GuardrailRecord = runner.GuardrailRecord
+RouteRecord = runner.RouteRecord
+collect_live_decisions = runner.collect_live_decisions
+evaluate = runner.evaluate
+load_dataset = runner.load_dataset
+
+
+EXAMPLE = [
+ {
+ "id": "r1",
+ "kind": "route",
+ "state": "Refund for order 1234",
+ "expected_route": "billing",
+ "decision": {
+ "choice": "billing",
+ "confidence": 0.91,
+ "probabilities": {"billing": 0.95, "tech": 0.05},
+ },
+ "metadata": {
+ "latency_ms": 212.0,
+ "usage": {"input_tokens": 40, "output_tokens": 1},
+ },
+ },
+ {
+ "id": "r2",
+ "kind": "route",
+ "state": "App crashes on login",
+ "expected_route": "tech",
+ "decision": {
+ "choice": "tech",
+ "confidence": 0.62,
+ "probabilities": {"billing": 0.19, "tech": 0.81},
+ },
+ },
+ {
+ "id": "r3",
+ "kind": "route",
+ "state": "Change my plan",
+ "expected_route": "billing",
+ "decision": {
+ "choice": "tech",
+ "confidence": 0.88,
+ "probabilities": {"billing": 0.06, "tech": 0.94},
+ },
+ },
+ {
+ "id": "r4",
+ "kind": "route",
+ "state": "Where is my invoice?",
+ "expected_route": "billing",
+ "decision": None,
+ "error": "TypeSafeAPITimeoutError: request timed out",
+ },
+ {
+ "id": "g1",
+ "kind": "guardrail",
+ "state": "Paris is the capital of France.",
+ "expected": "accept",
+ "decision": {"noul": 0.97},
+ },
+ {
+ "id": "g2",
+ "kind": "guardrail",
+ "state": "The moon is made of cheese.",
+ "expected": "reject",
+ "decision": {"noul": 0.8},
+ },
+ {
+ "id": "g3",
+ "kind": "guardrail",
+ "state": "Water boils at 100 C at sea level.",
+ "expected": "accept",
+ "decision": {"noul": 0.79},
+ },
+ {
+ "id": "g4",
+ "kind": "guardrail",
+ "state": "2 + 2 = 5",
+ "expected": "reject",
+ "decision": {"noul": 1.4},
+ },
+]
+
+
+def write_jsonl(tmp_path, records, name="data.jsonl"):
+ path = tmp_path / name
+ path.write_text("".join(json.dumps(r) + "\n" for r in records), encoding="utf-8")
+ return path
+
+
+def route(decision, expected="billing", **extra):
+ return RouteRecord(
+ id="r",
+ kind="route",
+ state="s",
+ expected_route=expected,
+ decision=decision,
+ **extra,
+ )
+
+
+def guardrail(decision, expected="accept", **extra):
+ return GuardrailRecord(
+ id="g",
+ kind="guardrail",
+ state="s",
+ expected=expected,
+ decision=decision,
+ **extra,
+ )
+
+
+def outcome_of(record, **thresholds):
+ kwargs = {
+ "min_confidence": 0.7,
+ "noul_threshold": 0.8,
+ "routes": {"billing", "tech"},
+ } | thresholds
+ return evaluate([record], **kwargs).outcomes[0]
+
+
+# Metric denominators
+
+
+def test_example_dataset_report(tmp_path):
+ report = evaluate(
+ load_dataset(write_jsonl(tmp_path, EXAMPLE)),
+ min_confidence=0.7,
+ noul_threshold=0.8,
+ )
+
+ assert report.routing.model_dump() == {
+ "total": 4,
+ "decided": 2,
+ "abstained": 1,
+ "failed": 1,
+ "correct": 1,
+ "accuracy": 0.25,
+ "coverage": 0.5,
+ "abstention_rate": 0.25,
+ }
+ assert report.guardrail.model_dump() == {
+ "total": 4,
+ "decided": 3,
+ "failed": 1,
+ "accepted": 2,
+ "rejected": 1,
+ "false_accepts": 1,
+ "false_accept_denominator": 1,
+ "false_accept_rate": 1.0,
+ "false_rejects": 1,
+ "false_reject_denominator": 2,
+ "false_reject_rate": 0.5,
+ }
+ assert report.usage.model_dump() == {
+ "input_tokens": 40,
+ "input_token_records": 1,
+ "output_tokens": 1,
+ "output_token_records": 1,
+ }
+ assert report.latency.model_dump() == {"count": 1, "mean_ms": 212.0}
+ assert [o.outcome for o in report.outcomes] == [
+ "correct",
+ "abstained",
+ "incorrect",
+ "failed",
+ "accepted",
+ "accepted",
+ "rejected",
+ "failed",
+ ]
+
+
+def test_failed_guardrails_are_excluded_from_false_rate_denominators():
+ records = [
+ guardrail(None, expected="reject"),
+ guardrail({"noul": 0.9}, expected="accept"),
+ ]
+ report = evaluate(records, min_confidence=0.5, noul_threshold=0.8)
+
+ assert report.guardrail.failed == 1
+ assert report.guardrail.false_accept_denominator == 0
+ assert report.guardrail.false_accept_rate is None
+ assert report.guardrail.false_reject_denominator == 1
+ assert report.guardrail.false_reject_rate == 0.0
+
+
+def test_empty_dataset_reports_none_rates():
+ report = evaluate([], min_confidence=0.5, noul_threshold=0.5)
+
+ assert report.routing.accuracy is None
+ assert report.routing.coverage is None
+ assert report.guardrail.false_reject_rate is None
+ assert report.latency.mean_ms is None
+
+
+# Threshold boundaries
+
+
+ ("confidence", "expected"),
+ [(0.69, "abstained"), (0.7, "correct"), (0.71, "correct")],
+)
+def test_choice_threshold_is_inclusive(confidence, expected):
+ record = route({"choice": "billing", "confidence": confidence, "probabilities": {}})
+
+ assert outcome_of(record, min_confidence=0.7).outcome == expected
+
+
+ ("noul", "threshold", "expected"),
+ [
+ (0.79, 0.8, "rejected"),
+ (0.8, 0.8, "accepted"),
+ (0.81, 0.8, "accepted"),
+ (0.0, 0.0, "accepted"),
+ (1.0, 1.0, "accepted"),
+ ],
+)
+def test_noul_threshold_is_inclusive(noul, threshold, expected):
+ assert (
+ outcome_of(guardrail({"noul": noul}), noul_threshold=threshold).outcome
+ == expected
+ )
+
+
[email protected]("bad", [float("nan"), float("inf"), -0.1, 1.1, True, "0.5"])
+def test_invalid_thresholds_raise(bad):
+ with pytest.raises(ValueError, match="noul_threshold"):
+ evaluate([], min_confidence=0.5, noul_threshold=bad)
+ with pytest.raises(ValueError, match="min_confidence"):
+ evaluate([], min_confidence=bad, noul_threshold=0.5)
+
+
+# Missing and invalid decisions
+
+
+ ("decision", "extra", "reason"),
+ [
+ (None, {}, "missing decision"),
+ (
+ {"choice": "billing", "confidence": 0.9, "probabilities": {}},
+ {"error": "boom"},
+ "recorded error: boom",
+ ),
+ (
+ {"choice": "sales", "confidence": 0.9, "probabilities": {}},
+ {},
+ "not a known route",
+ ),
+ (
+ {"choice": "billing", "confidence": 1.2, "probabilities": {}},
+ {},
+ "invalid ChoiceDecision: confidence",
+ ),
+ (
+ {"choice": "billing", "confidence": "0.9", "probabilities": {}},
+ {},
+ "invalid ChoiceDecision: confidence",
+ ),
+ (
+ {"choice": "billing", "probabilities": {}},
+ {},
+ "invalid ChoiceDecision: confidence",
+ ),
+ (
+ {"choice": "billing", "confidence": 0.9, "probabilities": {"billing": 1.5}},
+ {},
+ "invalid ChoiceDecision: probabilities.billing",
+ ),
+ ],
+)
+def test_unusable_route_decisions_fail(decision, extra, reason):
+ result = outcome_of(route(decision, **extra))
+
+ assert result.outcome == "failed"
+ assert reason in result.reason
+
+
+ "decision",
+ [
+ None,
+ {"noul": -0.1},
+ {"noul": 1.4},
+ {"noul": True},
+ {"noul": "0.9"},
+ {"confidence": 0.9},
+ ],
+)
+def test_unusable_guardrail_decisions_fail_and_never_accept(decision):
+ assert outcome_of(guardrail(decision, expected="reject")).outcome == "failed"
+
+
+def test_recorded_guardrail_error_fails_even_with_a_passing_decision():
+ result = outcome_of(guardrail({"noul": 0.99}, error="boom"))
+
+ assert (result.outcome, result.reason) == ("failed", "recorded error: boom")
+
+
+def test_route_confidence_is_never_used_as_noul():
+ record = guardrail(
+ {"choice": "accept", "confidence": 0.99, "probabilities": {"accept": 1.0}}
+ )
+
+ assert outcome_of(record).outcome == "failed"
+
+
+# Invalid inputs
+
+
+ ("lines", "message"),
+ [
+ (['{"id": "a"', ""], "line 1"),
+ (['["not", "an", "object"]'], "Invalid record at line 1"),
+ (
+ ['{"kind": "route", "state": "s", "expected_route": "b"}'],
+ "Invalid record at line 1",
+ ),
+ (['{"id": "a", "kind": "other", "state": "s"}'], "Invalid record at line 1"),
+ (
+ ['{"id": "a", "kind": "guardrail", "state": "s", "expected": "maybe"}'],
+ "Invalid record at line 1",
+ ),
+ (
+ [json.dumps(EXAMPLE[0]), json.dumps(EXAMPLE[0])],
+ "Duplicate record id 'r1' at line 2",
+ ),
+ (
+ [
+ '{"id": "a", "kind": "route", "state": "s", "expected_route": "b", "notes": "x"}'
+ ],
+ "Invalid record at line 1",
+ ),
+ (
+ ['{"id": "a", "kind": "route", "expected_route": "b"}'],
+ "Invalid record at line 1",
+ ),
+ (
+ [
+ '{"id": "a", "kind": "route", "state": "s", "expected_route": "b", "metadata": {"latency_ms": -1}}'
+ ],
+ "Invalid record at line 1",
+ ),
+ ],
+)
+def test_malformed_jsonl_raises_with_location(tmp_path, lines, message):
+ path = tmp_path / "bad.jsonl"
+ path.write_text("\n".join(lines), encoding="utf-8")
+
+ with pytest.raises(ValueError, match=message):
+ load_dataset(path)
+
+
[email protected]("name", ["data.json", "data.jsonl"])
+def test_json_and_jsonl_with_bom_and_blank_lines_are_accepted(tmp_path, name):
+ content = (
+ json.dumps(EXAMPLE)
+ if name.endswith(".json")
+ else "\n\n".join(json.dumps(r) for r in EXAMPLE)
+ )
+ (tmp_path / name).write_text(content, encoding="utf-8-sig")
+
+ assert [r.id for r in load_dataset(tmp_path / name)] == [r["id"] for r in EXAMPLE]
+
+
+def test_jsonl_strings_may_contain_unicode_line_separators(tmp_path):
+ record = {"id": "a", "kind": "route", "state": "a\u2028b\x85c", "expected_route": "b"}
+ path = tmp_path / "data.jsonl"
+ path.write_text(json.dumps(record, ensure_ascii=False), encoding="utf-8")
+
+ assert [r.state for r in load_dataset(path)] == ["a\u2028b\x85c"]
+
+
+ ("name", "content", "message"),
+ [
+ ("data.json", "{}", "must be an array"),
+ ("data.json", "[1,", "Invalid JSON"),
+ ("data.json", '[{"id": "a"}]', "Invalid record at index 0"),
+ ("data.csv", "", "Unsupported"),
+ ],
+)
+def test_wrong_json_shape_or_extension_raises(tmp_path, name, content, message):
+ (tmp_path / name).write_text(content, encoding="utf-8")
+
+ with pytest.raises(ValueError, match=message):
+ load_dataset(tmp_path / name)
+
+
+# Command line
+
+CLI_THRESHOLDS = ["--min-confidence", "0.7", "--noul-threshold", "0.8"]
+
+
+def test_main_prints_the_report_as_json(tmp_path, capsys):
+ path = write_jsonl(tmp_path, EXAMPLE)
+
+ assert runner.main([str(path), *CLI_THRESHOLDS]) == 0
+ expected = evaluate(load_dataset(path), min_confidence=0.7, noul_threshold=0.8)
+ assert json.loads(capsys.readouterr().out) == expected.model_dump(mode="json")
+
+
+def test_main_reports_invalid_input_as_a_usage_error(tmp_path, capsys):
+ path = tmp_path / "bad.jsonl"
+ path.write_text("{", encoding="utf-8")
+
+ with pytest.raises(SystemExit) as exit_info:
+ runner.main([str(path), *CLI_THRESHOLDS])
+
+ assert exit_info.value.code == 2
+ assert "Invalid JSONL at line 1" in capsys.readouterr().err
+
+
+# Usage and latency
+
+
+def test_missing_usage_and_latency_are_not_counted_as_zero():
+ records = [
+ guardrail(
+ {"noul": 0.9}, metadata={"usage": {"input_tokens": 10}, "latency_ms": 100.0}
+ ),
+ guardrail({"noul": 0.9}, metadata={"usage": {"output_tokens": 2}}),
+ guardrail({"noul": 0.9}),
+ ]
+ report = evaluate(records, min_confidence=0.5, noul_threshold=0.5)
+
+ assert report.usage.model_dump() == {
+ "input_tokens": 10,
+ "input_token_records": 1,
+ "output_tokens": 2,
+ "output_token_records": 1,
+ }
+ assert report.latency.model_dump() == {"count": 1, "mean_ms": 100.0}
+
+
+# Live collection with a fake client
+
+
+class FakeClient:
+ def __init__(self, *results):
+ self.results = list(results)
+ self.calls = []
+ self.closed = False
+
+ def system_one(self, state, questions, *, model=None):
+ self.calls.append((state, questions, model))
+ result = self.results.pop(0)
+ if isinstance(result, Exception):
+ raise result
+ return result
+
+ def close(self):
+ self.closed = True
+
+
+LIVE_KWARGS = {
+ "route_instructions": "Pick a team",
+ "route_criteria": {"billing": "Payments", "tech": "Bugs"},
+ "guardrail_instructions": "Is it true?",
+}
+
+
+def test_live_collection_builds_questions_and_records_metadata(monkeypatch):
+ ticks = iter([1.0, 1.25, 2.0, 2.5])
+ monkeypatch.setattr(
+ runner, "time", SimpleNamespace(perf_counter=lambda: next(ticks))
+ )
+ choice = SimpleNamespace(
+ choice="tech", confidence=0.8, probabilities={"billing": 0.1, "tech": 0.9}
+ )
+ client = FakeClient(
+ SimpleNamespace(
+ model="jev-1",
+ usage=SimpleNamespace(input_tokens=5, output_tokens=1),
+ answers={"route": choice},
+ ),
+ SimpleNamespace(
+ model="jev-1", usage=None, answers={"guardrail": SimpleNamespace(noul=0.9)}
+ ),
+ )
+ records = collect_live_decisions(
+ [route(None), guardrail(None)], client=client, model="jev", **LIVE_KWARGS
+ )
+
+ assert client.calls[0] == (
+ "s",
+ {
+ "route": {
+ "type": "choice",
+ "instructions": "Pick a team",
+ "criteria": {"billing": "Payments", "tech": "Bugs"},
+ }
+ },
+ "jev",
+ )
+ assert client.calls[1] == (
+ "s",
+ {"guardrail": {"type": "noul", "instructions": "Is it true?"}},
+ "jev",
+ )
+ assert records[0].decision == {
+ "choice": "tech",
+ "confidence": 0.8,
+ "probabilities": {"billing": 0.1, "tech": 0.9},
+ }
+ assert records[0].metadata.model_dump() == {
+ "model": "jev-1",
+ "usage": {"input_tokens": 5, "output_tokens": 1},
+ "latency_ms": 250.0,
+ }
+ assert records[1].decision == {"noul": 0.9}
+ assert records[1].metadata.latency_ms == 500.0
+ assert not client.closed
+
+
+def test_live_client_errors_become_failed_records():
+ client = FakeClient(RuntimeError("timeout"), SimpleNamespace(answers={}))
+ records = collect_live_decisions(
+ [guardrail(None), guardrail(None)], client=client, **LIVE_KWARGS
+ )
+ report = evaluate(records, min_confidence=0.5, noul_threshold=0.5)
+
+ assert records[0].error == "RuntimeError: timeout"
+ assert records[0].metadata is None
+ assert records[1].error == "no answer returned for 'guardrail'"
+ assert [o.outcome for o in report.outcomes] == ["failed", "failed"]
+
+
+ ("response", "error"),
+ [
+ (SimpleNamespace(answers=["guardrail"]), "no answer returned for 'guardrail'"),
+ (
+ SimpleNamespace(
+ usage=SimpleNamespace(input_tokens=-1, output_tokens=1),
+ answers={"guardrail": SimpleNamespace(noul=0.9)},
+ ),
+ "invalid usage: input_tokens: Input should be greater than or equal to 0",
+ ),
+ ],
+)
+def test_malformed_live_responses_become_failed_records(response, error):
+ [record] = collect_live_decisions(
+ [guardrail(None)], client=FakeClient(response), **LIVE_KWARGS
+ )
+
+ assert record.error == error
+ assert record.decision is None
+ assert record.metadata.latency_ms is not None
+
+
+ ("record", "answers", "error"),
+ [
+ (
+ guardrail(None),
+ {
+ "guardrail": SimpleNamespace(
+ type="choice", choice="tech", confidence=0.9, probabilities={}
+ )
+ },
+ "expected a 'noul' answer for 'guardrail', got 'choice'",
+ ),
+ (
+ guardrail(None),
+ {"guardrail": SimpleNamespace(type="choice", choice="tech", noul=0.9)},
+ "expected a 'noul' answer for 'guardrail', got 'choice'",
+ ),
+ (
+ route(None),
+ {"route": SimpleNamespace(type="noul", noul=0.9)},
+ "expected a 'choice' answer for 'route', got 'noul'",
+ ),
+ ],
+)
+def test_live_answer_of_the_wrong_type_fails(record, answers, error):
+ client = FakeClient(SimpleNamespace(answers=answers))
+ [record] = collect_live_decisions([record], client=client, **LIVE_KWARGS)
+
+ assert record.error == error
+ assert record.decision is None
+ assert outcome_of(record).outcome == "failed"
+
+
[email protected]("probabilities", [42, 0, False, "", []])
+def test_malformed_probabilities_fail_without_stopping_collection(probabilities):
+ bad = SimpleNamespace(choice="tech", confidence=0.9, probabilities=probabilities)
+ good = SimpleNamespace(choice="tech", confidence=0.9, probabilities={"tech": 0.9})
+ client = FakeClient(
+ SimpleNamespace(answers={"route": bad}),
+ SimpleNamespace(answers={"route": good}),
+ )
+ records = collect_live_decisions(
+ [route(None, expected="tech"), route(None, expected="tech")],
+ client=client,
+ **LIVE_KWARGS,
+ )
+
+ assert [outcome_of(r).outcome for r in records] == ["failed", "correct"]
+ assert "probabilities" in outcome_of(records[0]).reason
+
+
+def test_live_answer_without_probabilities_or_usage():
+ answer = SimpleNamespace(choice="tech", confidence=0.8, probabilities=None)
+ client = FakeClient(SimpleNamespace(answers={"route": answer}))
+ [record] = collect_live_decisions([route(None)], client=client, **LIVE_KWARGS)
+
+ assert record.decision["probabilities"] == {}
+ assert record.metadata.usage is None
+ assert record.metadata.model is None
+
+
+ "override",
+ [
+ {"route_instructions": " "},
+ {"guardrail_instructions": ""},
+ {"route_criteria": {}},
+ ],
+)
+def test_live_collection_requires_questions(override):
+ with pytest.raises(ValueError):
+ collect_live_decisions([], client=FakeClient(), **(LIVE_KWARGS | override))
+
+
+def test_live_collection_skips_recorded_decisions():
+ client = FakeClient()
+ records = [guardrail({"noul": 0.9}), guardrail(None, error="earlier failure")]
+
+ assert collect_live_decisions(records, client=client, **LIVE_KWARGS) == records
+ assert client.calls == []
diff --git a/scripts/jev_evaluation_runner.py b/scripts/jev_evaluation_runner.py
new file mode 100644
index 000000000..c5ba02d9f
--- /dev/null
+++ b/scripts/jev_evaluation_runner.py
@@ -0,0 +1,602 @@
+# ruff: noqa: T201
+"""Offline evaluation of Jev routing (Choice) and guardrail (Noul) decisions.
+
+Usage:
+ uv run python scripts/jev_evaluation_runner.py examples.jsonl \\
+ --min-confidence 0.7 --noul-threshold 0.8 [--route billing --route tech]
+
+The dataset is a .jsonl file (one record per line; blank lines are skipped) or
+a .json array, read as UTF-8 with an optional BOM. Malformed JSON, unknown
+fields, and missing or duplicate ids are reported with the line or index:
+
+ {"id": "r1", "kind": "route", "state": "Refund for order 1234", "expected_route": "billing", "decision": {"choice": "billing", "confidence": 0.91, "probabilities": {"billing": 0.95, "tech": 0.05}}}
+ {"id": "g1", "kind": "guardrail", "state": "Paris is the capital of France.", "expected": "accept", "decision": {"noul": 0.97}}
+
+Route records have expected_route; guardrail records have expected ("accept"
+or "reject"). Both can have an error (a recorded failure) and metadata (model,
+usage, latency_ms). The report is printed as JSON.
+
+A record fails if it has a recorded error, no decision, or a malformed decision
+(a missing field, or a value that isn't a finite number in [0, 1]); a route
+also fails if its choice isn't a known route (every expected_route in the
+dataset, or the --route values). Failed records are never counted as correct or
+accepted. Routes with confidence >= min_confidence are decided (correct or
+incorrect), lower ones abstain; guardrails with noul >= noul_threshold are
+accepted, lower ones rejected. Choice confidence and Noul probability are never
+converted into each other.
+
+- routing accuracy / coverage / abstention_rate: correct / decided / abstained
+ over all route records.
+- guardrail false_accept_rate / false_reject_rate: accepted / rejected over
+ decided records expected to be rejected / accepted.
+- usage, latency: token sums and mean latency_ms over only the records that
+ report them.
+
+A rate is null when its denominator is 0. These are decision-quality metrics,
+not a replay of a runtime guardrail: a guardrail that rejects when the answer
+is missing or the call fails would count those cases as rejections, while here
+they are failed and left out of the false-accept and false-reject rates.
+
+Live collection is opt-in and needs your own TypeSafe client, which you close:
+
+ with TypeSafeClient(api_key="...") as client:
+ records = collect_live_decisions(
+ load_dataset("unlabeled.jsonl"),
+ client=client,
+ route_instructions="Which team should handle this ticket?",
+ route_criteria={"billing": "Payments", "tech": "Bugs and outages"},
+ guardrail_instructions="Is this statement true?",
+ )
+
+It times each call, stores client errors on the record (which then fails), and
+fails an answer whose type doesn't match the question. The type is not stored,
+so recorded decisions are checked by their fields only.
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections import Counter
+from collections.abc import Collection, Mapping, Sequence
+import json
+from pathlib import Path
+import time
+from typing import Annotated, Any, Literal, Protocol
+
+from pydantic import (
+ BaseModel,
+ ConfigDict,
+ Field,
+ TypeAdapter,
+ ValidationError,
+)
+
+
+Probability = Annotated[float, Field(ge=0.0, le=1.0, allow_inf_nan=False)]
+_THRESHOLD: TypeAdapter[float] = TypeAdapter(
+ Probability, config=ConfigDict(strict=True)
+)
+
+ROUTE_QUESTION = "route"
+GUARDRAIL_QUESTION = "guardrail"
+
+
+class ChoiceDecision(BaseModel):
+ """A Jev Choice answer, mirroring the SDK's ``ChoiceAnswer`` fields."""
+
+ model_config = ConfigDict(strict=True, extra="ignore")
+
+ choice: str
+ confidence: Probability
+ probabilities: dict[str, Probability]
+
+
+class NoulDecision(BaseModel):
+ """A Jev Noul answer: the probability that the question is true."""
+
+ model_config = ConfigDict(strict=True, extra="ignore")
+
+ noul: Probability
+
+
+class TokenUsage(BaseModel):
+ """Token counts as reported by the SDK; ``None`` means not reported."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ input_tokens: int | None = Field(default=None, ge=0)
+ output_tokens: int | None = Field(default=None, ge=0)
+
+
+class DecisionMetadata(BaseModel):
+ """Request metadata kept only when it was actually recorded."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ model: str | None = None
+ usage: TokenUsage | None = None
+ latency_ms: float | None = Field(default=None, ge=0.0, allow_inf_nan=False)
+
+
+class _BaseRecord(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ id: str = Field(min_length=1)
+ state: Any
+ decision: dict[str, Any] | None = None
+ error: str | None = None
+ metadata: DecisionMetadata | None = None
+
+
+class RouteRecord(_BaseRecord):
+ """A labeled routing example with an optional recorded Choice decision."""
+
+ kind: Literal["route"]
+ expected_route: str = Field(min_length=1)
+
+
+class GuardrailRecord(_BaseRecord):
+ """A labeled guardrail example with an optional recorded Noul decision."""
+
+ kind: Literal["guardrail"]
+ expected: Literal["accept", "reject"]
+
+
+EvaluationRecord = Annotated[RouteRecord | GuardrailRecord, Field(discriminator="kind")]
+_RECORD_ADAPTER: TypeAdapter[RouteRecord | GuardrailRecord] = TypeAdapter(
+ EvaluationRecord
+)
+
+Outcome = Literal["correct", "incorrect", "abstained", "accepted", "rejected", "failed"]
+
+
+class RecordOutcome(BaseModel):
+ """How a single record was classified, and why."""
+
+ id: str
+ kind: Literal["route", "guardrail"]
+ outcome: Outcome
+ reason: str
+
+
+class RoutingReport(BaseModel):
+ """Routing counts and rates; every rate is over all routing records."""
+
+ total: int
+ decided: int
+ abstained: int
+ failed: int
+ correct: int
+ accuracy: float | None
+ coverage: float | None
+ abstention_rate: float | None
+
+
+class GuardrailReport(BaseModel):
+ """Guardrail counts; error rates are over decided records only."""
+
+ total: int
+ decided: int
+ failed: int
+ accepted: int
+ rejected: int
+ false_accepts: int
+ false_accept_denominator: int
+ false_accept_rate: float | None
+ false_rejects: int
+ false_reject_denominator: int
+ false_reject_rate: float | None
+
+
+class UsageSummary(BaseModel):
+ """Token totals over the records that reported each count."""
+
+ input_tokens: int
+ input_token_records: int
+ output_tokens: int
+ output_token_records: int
+
+
+class LatencySummary(BaseModel):
+ """Mean latency over records with a measured latency."""
+
+ count: int
+ mean_ms: float | None
+
+
+class EvaluationReport(BaseModel):
+ """Full evaluation result."""
+
+ routing: RoutingReport
+ guardrail: GuardrailReport
+ usage: UsageSummary
+ latency: LatencySummary
+ outcomes: list[RecordOutcome]
+
+
+class SystemOneClient(Protocol):
+ """The part of the TypeSafe client used for live collection."""
+
+ def system_one(
+ self, state: Any, questions: Mapping[str, Any], *, model: str | None = None
+ ) -> Any:
+ """Answer the named questions about ``state``."""
+ ...
+
+
+def load_dataset(path: str | Path) -> list[RouteRecord | GuardrailRecord]:
+ """Load and validate records from a ``.jsonl`` or ``.json`` file.
+
+ Args:
+ path: A ``.jsonl`` file (one record per line; blank lines are skipped)
+ or a ``.json`` file containing an array of records.
+
+ Returns:
+ The validated records, in file order.
+
+ Raises:
+ ValueError: If the file or any record is malformed.
+ """
+ path = Path(path)
+ text = path.read_text(encoding="utf-8-sig")
+ if path.suffix == ".jsonl":
+ items = _parse_jsonl(text)
+ elif path.suffix == ".json":
+ try:
+ data = json.loads(text)
+ except ValueError as e:
+ raise ValueError(f"Invalid JSON: {e}") from e
+ if not isinstance(data, list):
+ raise ValueError("A .json dataset must be an array of records")
+ items = [(f"index {i}", item) for i, item in enumerate(data)]
+ else:
+ raise ValueError(f"Unsupported dataset extension: {path.suffix!r}")
+
+ records: list[RouteRecord | GuardrailRecord] = []
+ seen: set[str] = set()
+ for location, item in items:
+ try:
+ record = _RECORD_ADAPTER.validate_python(item)
+ except ValidationError as e:
+ raise ValueError(f"Invalid record at {location}: {e}") from e
+ if record.id in seen:
+ raise ValueError(f"Duplicate record id {record.id!r} at {location}")
+ seen.add(record.id)
+ records.append(record)
+ return records
+
+
+def evaluate(
+ records: Sequence[RouteRecord | GuardrailRecord],
+ *,
+ min_confidence: float,
+ noul_threshold: float,
+ routes: Collection[str] | None = None,
+) -> EvaluationReport:
+ """Classify every record and compute routing and guardrail metrics.
+
+ Both thresholds are inclusive: a Choice confidence equal to
+ ``min_confidence`` is decided, and a Noul probability equal to
+ ``noul_threshold`` is accepted.
+
+ Args:
+ records: Records to evaluate.
+ min_confidence: Minimum Choice confidence for a route to be decided.
+ noul_threshold: Minimum Noul probability for an output to be accepted.
+ routes: Known route labels. Defaults to every ``expected_route``.
+
+ Returns:
+ Counts, rates, usage, latency, and a per-record outcome with its reason.
+
+ Raises:
+ ValueError: If a threshold is not a finite number in [0, 1].
+ """
+ _validate_threshold("min_confidence", min_confidence)
+ _validate_threshold("noul_threshold", noul_threshold)
+ route_records = [r for r in records if isinstance(r, RouteRecord)]
+ known_routes = (
+ set(routes) if routes is not None else {r.expected_route for r in route_records}
+ )
+
+ outcomes: list[RecordOutcome] = []
+ false_accepts = false_rejects = fa_denominator = fr_denominator = 0
+ for record in records:
+ if isinstance(record, RouteRecord):
+ outcome, reason = _classify_route(record, known_routes, min_confidence)
+ else:
+ outcome, reason = _classify_guardrail(record, noul_threshold)
+ if outcome != "failed" and record.expected == "reject":
+ fa_denominator += 1
+ if outcome == "accepted":
+ false_accepts += 1
+ elif outcome != "failed":
+ fr_denominator += 1
+ if outcome == "rejected":
+ false_rejects += 1
+ outcomes.append(
+ RecordOutcome(
+ id=record.id, kind=record.kind, outcome=outcome, reason=reason
+ )
+ )
+
+ route_counts = Counter(o.outcome for o in outcomes if o.kind == "route")
+ guard_counts = Counter(o.outcome for o in outcomes if o.kind == "guardrail")
+ route_total = len(route_records)
+ route_decided = route_counts["correct"] + route_counts["incorrect"]
+ guard_total = len(records) - route_total
+ return EvaluationReport(
+ routing=RoutingReport(
+ total=route_total,
+ decided=route_decided,
+ abstained=route_counts["abstained"],
+ failed=route_counts["failed"],
+ correct=route_counts["correct"],
+ accuracy=_rate(route_counts["correct"], route_total),
+ coverage=_rate(route_decided, route_total),
+ abstention_rate=_rate(route_counts["abstained"], route_total),
+ ),
+ guardrail=GuardrailReport(
+ total=guard_total,
+ decided=guard_total - guard_counts["failed"],
+ failed=guard_counts["failed"],
+ accepted=guard_counts["accepted"],
+ rejected=guard_counts["rejected"],
+ false_accepts=false_accepts,
+ false_accept_denominator=fa_denominator,
+ false_accept_rate=_rate(false_accepts, fa_denominator),
+ false_rejects=false_rejects,
+ false_reject_denominator=fr_denominator,
+ false_reject_rate=_rate(false_rejects, fr_denominator),
+ ),
+ usage=_summarize_usage(records),
+ latency=_summarize_latency(records),
+ outcomes=outcomes,
+ )
+
+
+def collect_live_decisions(
+ records: Sequence[RouteRecord | GuardrailRecord],
+ *,
+ client: SystemOneClient,
+ route_instructions: str,
+ route_criteria: Mapping[str, str],
+ guardrail_instructions: str,
+ model: str | None = None,
+) -> list[RouteRecord | GuardrailRecord]:
+ """Ask Jev for each record that has no recorded decision or error.
+
+ Opt-in: the caller supplies a client built with their own credentials,
+ and remains responsible for closing it. Client errors are recorded on the
+ record (which then evaluates as failed) instead of stopping the run.
+
+ Args:
+ records: Records to complete; those with a decision or error are kept as is.
+ client: A TypeSafe client, or any object with a compatible ``system_one``.
+ route_instructions: Instructions for the Choice question on route records.
+ route_criteria: Route labels mapped to their descriptions.
+ guardrail_instructions: Instructions for the Noul question on guardrail records.
+ model: Optional Jev model name passed to the client.
+
+ Returns:
+ The records with decisions, latency, and usage filled in where collected.
+
+ Raises:
+ ValueError: If instructions are blank or ``route_criteria`` is empty.
+ """
+ if not route_instructions.strip() or not guardrail_instructions.strip():
+ raise ValueError("route_instructions and guardrail_instructions are required")
+ if not route_criteria:
+ raise ValueError("route_criteria must not be empty")
+
+ collected: list[RouteRecord | GuardrailRecord] = []
+ for record in records:
+ if record.decision is not None or record.error is not None:
+ collected.append(record)
+ continue
+ if isinstance(record, RouteRecord):
+ name = ROUTE_QUESTION
+ question: dict[str, Any] = {
+ "type": "choice",
+ "instructions": route_instructions,
+ "criteria": dict(route_criteria),
+ }
+ else:
+ name = GUARDRAIL_QUESTION
+ question = {"type": "noul", "instructions": guardrail_instructions}
+ collected.append(_query(record, client, name, question, model))
+ return collected
+
+
+def _query(
+ record: RouteRecord | GuardrailRecord,
+ client: SystemOneClient,
+ name: str,
+ question: dict[str, Any],
+ model: str | None,
+) -> RouteRecord | GuardrailRecord:
+ start = time.perf_counter()
+ try:
+ response = client.system_one(record.state, {name: question}, model=model)
+ except Exception as e:
+ return record.model_copy(update={"error": f"{type(e).__name__}: {e}"})
+ latency_ms = (time.perf_counter() - start) * 1000
+
+ usage = getattr(response, "usage", None)
+ response_model = getattr(response, "model", None)
+ metadata = DecisionMetadata(
+ model=response_model if isinstance(response_model, str) else None,
+ latency_ms=latency_ms,
+ )
+ if usage is not None:
+ try:
+ metadata.usage = TokenUsage.model_validate(usage, from_attributes=True)
+ except ValidationError as e:
+ first = e.errors()[0]
+ location = ".".join(str(part) for part in first["loc"]) or "usage"
+ return record.model_copy(
+ update={
+ "error": f"invalid usage: {location}: {first['msg']}",
+ "metadata": metadata,
+ }
+ )
+ answers = getattr(response, "answers", None)
+ answer = answers.get(name) if isinstance(answers, Mapping) else None
+ if answer is None:
+ return record.model_copy(
+ update={"error": f"no answer returned for {name!r}", "metadata": metadata}
+ )
+ answer_type = getattr(answer, "type", None)
+ if answer_type is not None and answer_type != question["type"]:
+ return record.model_copy(
+ update={
+ "error": f"expected a {question['type']!r} answer for {name!r},"
+ f" got {answer_type!r}",
+ "metadata": metadata,
+ }
+ )
+ if isinstance(record, RouteRecord):
+ probabilities = getattr(answer, "probabilities", None)
+ if probabilities is None:
+ probabilities = {}
+ elif isinstance(probabilities, Mapping):
+ probabilities = dict(probabilities)
+ decision = {
+ "choice": getattr(answer, "choice", None),
+ "confidence": getattr(answer, "confidence", None),
+ "probabilities": probabilities,
+ }
+ else:
+ decision = {"noul": getattr(answer, "noul", None)}
+ return record.model_copy(update={"decision": decision, "metadata": metadata})
+
+
+def _parse_decision(
+ record: RouteRecord | GuardrailRecord,
+ model: type[ChoiceDecision] | type[NoulDecision],
+) -> tuple[ChoiceDecision | NoulDecision | None, str]:
+ """Return the validated decision, or ``None`` and the failure reason."""
+ if record.error is not None:
+ return None, f"recorded error: {record.error}"
+ if record.decision is None:
+ return None, "missing decision"
+ try:
+ return model.model_validate(record.decision), ""
+ except ValidationError as e:
+ first = e.errors()[0]
+ location = ".".join(str(part) for part in first["loc"]) or "decision"
+ return None, f"invalid {model.__name__}: {location}: {first['msg']}"
+
+
+def _classify_route(
+ record: RouteRecord, known_routes: set[str], min_confidence: float
+) -> tuple[Outcome, str]:
+ decision, reason = _parse_decision(record, ChoiceDecision)
+ if not isinstance(decision, ChoiceDecision):
+ return "failed", reason
+ if decision.choice not in known_routes:
+ return "failed", f"choice {decision.choice!r} is not a known route"
+ if decision.confidence < min_confidence:
+ return (
+ "abstained",
+ f"confidence {decision.confidence} below min_confidence {min_confidence}",
+ )
+ outcome: Outcome = (
+ "correct" if decision.choice == record.expected_route else "incorrect"
+ )
+ return (
+ outcome,
+ f"confidence {decision.confidence} meets min_confidence {min_confidence}",
+ )
+
+
+def _classify_guardrail(
+ record: GuardrailRecord, noul_threshold: float
+) -> tuple[Outcome, str]:
+ decision, reason = _parse_decision(record, NoulDecision)
+ if not isinstance(decision, NoulDecision):
+ return "failed", reason
+ if decision.noul >= noul_threshold:
+ return "accepted", f"noul {decision.noul} meets threshold {noul_threshold}"
+ return "rejected", f"noul {decision.noul} below threshold {noul_threshold}"
+
+
+def _summarize_usage(records: Sequence[RouteRecord | GuardrailRecord]) -> UsageSummary:
+ usages = [r.metadata.usage for r in records if r.metadata and r.metadata.usage]
+ inputs = [u.input_tokens for u in usages if u.input_tokens is not None]
+ outputs = [u.output_tokens for u in usages if u.output_tokens is not None]
+ return UsageSummary(
+ input_tokens=sum(inputs),
+ input_token_records=len(inputs),
+ output_tokens=sum(outputs),
+ output_token_records=len(outputs),
+ )
+
+
+def _summarize_latency(
+ records: Sequence[RouteRecord | GuardrailRecord],
+) -> LatencySummary:
+ values = [
+ r.metadata.latency_ms
+ for r in records
+ if r.metadata and r.metadata.latency_ms is not None
+ ]
+ return LatencySummary(
+ count=len(values), mean_ms=sum(values) / len(values) if values else None
+ )
+
+
+def _parse_jsonl(text: str) -> list[tuple[str, Any]]:
+ items: list[tuple[str, Any]] = []
+ for number, line in enumerate(text.split("\n"), start=1):
+ if not line.strip():
+ continue
+ try:
+ items.append((f"line {number}", json.loads(line)))
+ except ValueError as e:
+ raise ValueError(f"Invalid JSONL at line {number}: {e}") from e
+ return items
+
+
+def _validate_threshold(name: str, value: float) -> None:
+ try:
+ _THRESHOLD.validate_python(value)
+ except ValidationError as e:
+ raise ValueError(
+ f"{name} must be a finite number in [0, 1], got {value!r}"
+ ) from e
+
+
+def _rate(numerator: int, denominator: int) -> float | None:
+ return numerator / denominator if denominator else None
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ """Evaluate a dataset file and print the report as JSON."""
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+ )
+ parser.add_argument("dataset", type=Path, help="a .jsonl or .json file")
+ parser.add_argument("--min-confidence", type=float, required=True)
+ parser.add_argument("--noul-threshold", type=float, required=True)
+ parser.add_argument(
+ "--route",
+ action="append",
+ dest="routes",
+ help="a known route; repeat for each (default: every expected_route)",
+ )
+ args = parser.parse_args(argv)
+ try:
+ report = evaluate(
+ load_dataset(args.dataset),
+ min_confidence=args.min_confidence,
+ noul_threshold=args.noul_threshold,
+ routes=args.routes,
+ )
+ except (OSError, ValueError) as e:
+ parser.error(str(e))
+ print(report.model_dump_json(indent=2))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())