Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(evals): attribute tournament elimination errors
xp59676702b531
diff --git a/docs/evals.md b/docs/evals.mdindex d63bb027..4fade7a9 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -184,6 +184,47 @@ This does not reproduce the BANKING77 results in [issue #102](https://github.comthose numbers depend on its dataset, checkpoint and bi-encoder. This API makes thesame kind of diagnosis repeatable on a caller's own labelled set.+## Attributing tournament errors++For a labelled many-option choice set, `laya.evals_tournament.evaluate_tournament`+runs the existing `predict_tournament` path and keeps its finalist metadata in the+regular evaluation report:++```python+import laya+from laya.evals import Dataset+from laya.evals_tournament import evaluate_tournament++agent = laya.load()+dataset_path = "intents.jsonl"+report = evaluate_tournament(+ agent, Dataset.from_jsonl(dataset_path), group_size=16,+ checkpoint_id="my-checkpoint@revision", dataset_path=dataset_path,+)+print(report.overall)+print(report.cases[0]["tournament_status"])+```++Each choice case records its finalists and number of elimination rounds, and is+classified as `correct`, `elimination_miss` (the gold label did not reach the+final call), or `final_choice_miss` (it survived, but the final call chose another+label). `tournament_survival_rate` is the fraction of all choices whose gold label+reached the final call; choices needing no elimination are included. Conditional+`tournament_accuracy_on_survivors` is omitted when none survived. Ordinary+`choice_accuracy` remains end-to-end accuracy, and all three metrics appear in the+existing language, model, question and tag slices. Request latency includes every+tournament round; the report does not time rounds separately.++Use the checkpoint and group size you intend to deploy. The caller supplies the+checkpoint identifier, ideally an immutable revision; the report cannot inspect+the weights. Passing `dataset_path` records a SHA256 of the JSONL bytes. The+`laya-tournament-evals-report/1` schema and group size prevent a baseline gate+from comparing different strategies as though they were the same measurement.+Compare group sizes side by side on development data, then evaluate the chosen+size once on held-out data. The runtime's final probabilities and confidence are+over finalists, not over the original labels. The metadata identifies whether+elimination lost the gold label, but not which round lost it.+## Evaluating an ONNX export`run --onnx PATH` scores an exported ONNX model through `ONNXAgent` instead of the torchdiff --git a/laya/evals.py b/laya/evals.pyindex 1506d35e..2d35f682 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -726,6 +726,10 @@ class EvalReport:continuereasons.append("%s (%s): baseline is %s, this run is %s"% (key, _IDENTITY_LABELS[key], there, here))+ if self.config.get("schema") == other.get("schema") == "laya-tournament-evals-report/1":+ here, there = self.config.get("tournament_group_size"), other.get("tournament_group_size")+ if here is None or there is None or here != there:+ reasons.append("tournament group size: baseline is %s, this run is %s" % (there, here))return (not reasons), reasonsdef compare(self, baseline: Dict[str, Any], tolerances: Optional[Dict[str, float]] = None,@@ -1143,6 +1147,9 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluashortlist = (result or {}).get("shortlist")if isinstance(shortlist, dict) and qid in shortlist:case["shortlist"] = shortlist[qid]+ tournament = (result or {}).get("tournament")+ if isinstance(tournament, dict) and qid in tournament:+ case["tournament"] = tournament[qid]cases.append(case)index += len(chunk)diff --git a/laya/evals_tournament.py b/laya/evals_tournament.pynew file mode 100644index 00000000..b619d5ad--- /dev/null+++ b/laya/evals_tournament.py@@ -0,0 +1,142 @@+"""Opt-in attribution of tournament elimination and final-choice errors."""+from __future__ import annotations++from typing import Any, Dict, Optional, Sequence++from .evals import Dataset, EvalError, EvalReport, _group_cases, evaluate, file_fingerprint+from .shortlist import DEFAULT_TOURNAMENT_GROUP, predict_tournament++TOURNAMENT_REPORT_SCHEMA = "laya-tournament-evals-report/1"+++def _labels(question: Dict[str, Any], where: str) -> list[str]:+ criteria = question.get("criteria")+ if isinstance(criteria, dict):+ labels = list(criteria)+ elif isinstance(criteria, list):+ labels = criteria+ else:+ raise EvalError("%s: choice criteria must be a dict or list" % where)+ if not labels or any(not isinstance(label, str) for label in labels):+ raise EvalError("%s: choice criteria must contain string labels" % where)+ if len(set(labels)) != len(labels):+ raise EvalError("%s: choice criteria contain duplicate labels" % where)+ return labels+++def _validate_dataset(dataset: Dataset) -> None:+ if not isinstance(dataset, Dataset) or not dataset.examples:+ raise EvalError("tournament evaluation needs a non-empty Dataset")+ for index, example in enumerate(dataset.examples):+ if not example.expected:+ raise EvalError("row %d: expected must contain a choice label" % index)+ for qid, expected in example.expected.items():+ where = "row %d question %r" % (index, qid)+ question = example.questions.get(qid)+ if not isinstance(question, dict) or question.get("type") != "choice":+ raise EvalError("%s: tournament evaluation requires a choice question" % where)+ labels = _labels(question, where)+ if not isinstance(expected, str) or expected not in labels:+ raise EvalError("%s: expected label %r is not in criteria" % (where, expected))+++def _shape(n: int, group_size: int) -> tuple[int, int]:+ rounds = 0+ while n > group_size:+ n = -(-n // group_size)+ rounds += 1+ return rounds, n+++class _TournamentRunner:+ def __init__(self, agent: Any, group_size: int):+ self.agent, self.group_size = agent, group_size++ def predict(self, state: Any, questions: Dict[str, Any], model: Optional[str] = None) -> Dict[str, Any]:+ kwargs = {"model": model} if model is not None else {}+ result = predict_tournament(self.agent, state, questions, self.group_size, **kwargs)+ meta = result.get("tournament")+ choice_ids = {qid for qid, question in questions.items()+ if isinstance(question, dict) and question.get("type") == "choice"}+ if not isinstance(meta, dict) or set(meta) != choice_ids:+ raise EvalError("predict_tournament returned incomplete tournament metadata")+ answers = result.get("answers")+ for qid in choice_ids:+ original = _labels(questions[qid], "question %r" % qid)+ item = meta[qid]+ rounds, finalists = _shape(len(original), self.group_size)+ kept = item.get("labels") if isinstance(item, dict) else None+ if (not isinstance(kept, list) or len(kept) != finalists+ or any(not isinstance(label, str) or label not in original for label in kept)+ or len(set(kept)) != len(kept)+ or kept != [label for label in original if label in kept]+ or type(item.get("n")) is not int or item["n"] != len(original)+ or type(item.get("rounds")) is not int+ or item["rounds"] != rounds):+ raise EvalError("question %r: inconsistent tournament metadata" % qid)+ answer = answers.get(qid) if isinstance(answers, dict) else None+ if (not isinstance(answer, dict) or answer.get("type") != "choice"+ or answer.get("choice") not in kept):+ raise EvalError("question %r: choice answer is outside the finalists" % qid)+ return result+++def _metrics(cases: Sequence[Dict[str, Any]]) -> Dict[str, float]:+ if not cases:+ return {}+ survived = sum(case["tournament_status"] != "elimination_miss" for case in cases)+ correct = sum(case["tournament_status"] == "correct" for case in cases)+ metrics = {"tournament_survival_rate": survived / len(cases)}+ if survived:+ metrics["tournament_accuracy_on_survivors"] = correct / survived+ return metrics+++def evaluate_tournament(+ agent: Any,+ dataset: Dataset,+ *,+ group_size: int = DEFAULT_TOURNAMENT_GROUP,+ checkpoint_id: str,+ dataset_path: Optional[str] = None,+) -> EvalReport:+ """Score labelled choices and attribute misses to elimination or the final choice.++ ``checkpoint_id`` is a caller-supplied identifier, ideally an immutable revision.+ ``dataset_path`` adds the JSONL file's SHA256. The existing choice accuracy is+ end-to-end; tournament confidence and probabilities describe finalists only.+ ``tournament_survival_rate`` includes choices that needed no elimination round.+ """+ if isinstance(group_size, bool) or not isinstance(group_size, int) or group_size < 2:+ raise EvalError("group_size must be an integer of at least 2")+ if not isinstance(checkpoint_id, str) or not checkpoint_id.strip():+ raise EvalError("checkpoint_id must be a non-empty string")+ _validate_dataset(dataset)+ config: Dict[str, Any] = {+ "tournament_group_size": group_size,+ "tournament": {"group_size": group_size, "checkpoint_id": checkpoint_id},+ }+ if dataset_path is not None:+ config["dataset"] = dataset_path+ config["dataset_sha256"] = file_fingerprint(dataset_path)+ report = evaluate(_TournamentRunner(agent, group_size), dataset, config=config)+ for case in report.cases:+ item = case.get("tournament")+ if not isinstance(item, dict) or not isinstance(item.get("labels"), list):+ raise EvalError("question %r: missing tournament metadata" % case["qid"])+ if case["expected"] not in item["labels"]:+ case["tournament_status"] = "elimination_miss"+ elif case["correct"]:+ case["tournament_status"] = "correct"+ else:+ case["tournament_status"] = "final_choice_miss"+ report.overall.update(_metrics(report.cases))+ for dimension in ("language", "model", "qid", "tag"):+ for value, group in _group_cases(report.cases, dimension).items():+ report.slices[dimension][value].update(_metrics(group))+ report.config["schema"] = TOURNAMENT_REPORT_SCHEMA+ report.config["tournament"]["counts"] = {+ status: sum(case["tournament_status"] == status for case in report.cases)+ for status in ("correct", "elimination_miss", "final_choice_miss")+ }+ return reportdiff --git a/scripts/test_suites.py b/scripts/test_suites.pyindex 922ade5f..949baa9a 100644--- a/scripts/test_suites.py+++ b/scripts/test_suites.py@@ -106,6 +106,7 @@ SCRIPT_SUITES = [PYTEST_SUITES = ["tests/test_evals_agreement.py",+ "tests/test_evals_tournament.py","tests/test_serve.py","tests/test_mcp_remote.py","tests/test_router_batch.py",diff --git a/tests/test_evals_tournament.py b/tests/test_evals_tournament.pynew file mode 100644index 00000000..9eb32062--- /dev/null+++ b/tests/test_evals_tournament.py@@ -0,0 +1,148 @@+"""Weight-free checks for tournament stage attribution and report identity."""+import json++import pytest++from laya.evals import Dataset, EvalError, Example, evaluate+from laya.evals_tournament import TOURNAMENT_REPORT_SCHEMA, evaluate_tournament+++WIDE = {"intent": {"type": "choice", "instructions": "Which?",+ "criteria": ["a", "b", "c", "d", "e", "f"]}}+NARROW = {"intent": {"type": "choice", "instructions": "Which?", "criteria": ["a", "b"]}}+++class ChoiceAgent:+ def __init__(self):+ self.received = []++ def system_one(self, state, questions):+ self.received.append((state, {qid: list(q["criteria"]) for qid, q in questions.items()}))+ answers = {}+ for qid, question in questions.items():+ labels = list(question["criteria"])+ if state == "early" and labels == ["a", "b"]:+ choice = "a"+ elif state == "late" and labels == ["c", "e"]:+ choice = "e"+ elif (state == "final" and labels == ["a", "c"]) or (state == "pass" and labels == ["a", "b"]):+ choice = labels[-1]+ else:+ choice = labels[0]+ answers[qid] = {"type": "choice", "choice": choice, "answer_confidence": 0.7}+ return {"model": "stub", "answers": answers}+++def row(state, gold, *, narrow=False, tag="test"):+ return Example(state, NARROW if narrow else WIDE, {"intent": gold}, tags=(tag,))+++def run(dataset, group_size=2):+ return evaluate_tournament(ChoiceAgent(), dataset, group_size=group_size,+ checkpoint_id="checkpoint@123")+++def test_stage_attribution_and_slices():+ dataset = Dataset([row("ok", "a", tag="kept"), row("final", "a", tag="kept"),+ row("early", "b", tag="lost"), row("late", "c", tag="lost"),+ row("pass", "a", narrow=True, tag="kept")])+ report = run(dataset)+ assert [case["tournament_status"] for case in report.cases] == [+ "correct", "final_choice_miss", "elimination_miss", "elimination_miss",+ "final_choice_miss"]+ assert [case["tournament"]["rounds"] for case in report.cases] == [2, 2, 2, 2, 0]+ assert report.overall["tournament_survival_rate"] == pytest.approx(3 / 5)+ assert report.overall["tournament_accuracy_on_survivors"] == pytest.approx(1 / 3)+ assert report.overall["choice_accuracy"] == pytest.approx(1 / 5)+ assert report.slices["tag"]["lost"]["tournament_survival_rate"] == 0.0+ assert "tournament_accuracy_on_survivors" not in report.slices["tag"]["lost"]+ assert report.config["schema"] == TOURNAMENT_REPORT_SCHEMA+ assert report.config["tournament"]["counts"] == {+ "correct": 1, "elimination_miss": 2, "final_choice_miss": 2}+ assert json.loads(json.dumps(report.to_json()))["cases"][3]["tournament_status"] == "elimination_miss"+++def test_passthrough_and_group_size_are_reported_and_compared():+ dataset = Dataset([row("ok", "a", narrow=True)])+ report = run(dataset)+ assert report.cases[0]["tournament"]["labels"] == ["a", "b"]+ assert report.cases[0]["tournament"]["rounds"] == 0+ assert report.overall["tournament_survival_rate"] == 1.0+ same = run(dataset)+ assert report.comparable_to(same.to_json())[0]+ wider = run(dataset, group_size=3)+ ok, reasons = report.comparable_to(wider.to_json())+ assert not ok+ assert any("tournament group size" in reason for reason in reasons)+ missing = same.to_json()+ missing["config"].pop("tournament_group_size")+ assert not report.comparable_to(missing)[0]+++def test_invalid_inputs_fail_before_inference():+ with pytest.raises(EvalError, match="not in criteria"):+ run(Dataset([row("ok", "missing")]))+ with pytest.raises(EvalError, match="choice question"):+ run(Dataset([Example("ok", {"intent": {"type": "score"}}, {"intent": 1})]))+ with pytest.raises(EvalError, match="at least 2"):+ run(Dataset([row("ok", "a")]), group_size=True)+ with pytest.raises(EvalError, match="checkpoint_id"):+ evaluate_tournament(ChoiceAgent(), Dataset([row("ok", "a")]), checkpoint_id=" ")+++def test_malformed_tournament_metadata_fails(monkeypatch):+ def malformed(*_args, **_kwargs):+ return {"answers": {"intent": {"type": "choice", "choice": "a"}},+ "tournament": {"intent": {"labels": ["a", "a"], "n": 6, "rounds": 2}}}++ monkeypatch.setattr("laya.evals_tournament.predict_tournament", malformed)+ with pytest.raises(EvalError, match="inconsistent tournament metadata"):+ run(Dataset([row("ok", "a")]))+++def test_explicit_model_reaches_every_round():+ class ModelAgent(ChoiceAgent):+ def predict(self, state, questions, model=None):+ assert model == "english"+ return self.system_one(state, questions)++ agent = ModelAgent()+ example = Example("ok", WIDE, {"intent": "a"}, model="english")+ report = evaluate_tournament(agent, Dataset([example]), group_size=2,+ checkpoint_id="checkpoint@123")+ assert len(agent.received) == 3 # two elimination rounds and the final call+ assert report.slices["model"]["english"]["choice_accuracy"] == 1.0+++def test_multiple_questions_and_described_criteria():+ questions = {+ "route": {"type": "choice", "criteria": {label: label.upper() for label in "abcdef"}},+ "fallback": NARROW["intent"],+ }+ report = run(Dataset([Example("ok", questions, {"route": "a", "fallback": "a"})]))+ assert [case["qid"] for case in report.cases] == ["route", "fallback"]+ assert [case["tournament"]["rounds"] for case in report.cases] == [2, 0]+ assert report.slices["qid"]["route"]["tournament_survival_rate"] == 1.0+ assert report.slices["qid"]["fallback"]["choice_accuracy"] == 1.0+++def test_dataset_hash_and_plain_evaluate(tmp_path):+ path = tmp_path / "intents.jsonl"+ path.write_text(json.dumps({"state": "ok", "questions": WIDE,+ "expected": {"intent": "a"}}) + "\n", encoding="utf-8")+ dataset = Dataset.from_jsonl(str(path))+ report = evaluate_tournament(ChoiceAgent(), dataset, group_size=2,+ checkpoint_id="checkpoint@123", dataset_path=str(path))+ from laya.evals import file_fingerprint++ assert report.config["dataset_sha256"] == file_fingerprint(str(path))+ assert report.config["questions_sha256"]++ class PlainRunner:+ def predict(self, state, questions, model=None):+ return {"answers": {"intent": {"type": "choice", "choice": "a"}}}++ plain = evaluate(PlainRunner(), Dataset([row("ok", "a", narrow=True)]))+ assert plain.config["schema"] == "laya-evals-report/1"+ assert "tournament_survival_rate" not in plain.overall+ assert "tournament" not in plain.cases[0]diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex f1914b7e..fe53d252 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -1073,6 +1073,12 @@ check_param("evaluate_shortlist", evaluate_shortlist, "dataset_path", None)for param in ("checkpoint_id", "embedder_id"):check_param("evaluate_shortlist", evaluate_shortlist, param, inspect.Parameter.empty)+from laya.evals_tournament import evaluate_tournament # noqa: E402++check_param("evaluate_tournament", evaluate_tournament, "group_size", 16)+check_param("evaluate_tournament", evaluate_tournament, "dataset_path", None)+check_param("evaluate_tournament", evaluate_tournament, "checkpoint_id", inspect.Parameter.empty)+# docs/hooks/examples.md ## Composition teaches one scope ordering contract; the pre-fix wording# ("Installed hooks first, then convenience callables") named only two tiers and put installed at