Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): attribute tournament elimination errors

xp59676702b531
diff --git a/docs/evals.md b/docs/evals.md
index d63bb027..4fade7a9 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -184,6 +184,47 @@ This does not reproduce the BANKING77 results in [issue #102](https://github.com
those numbers depend on its dataset, checkpoint and bi-encoder. This API makes the
same kind of diagnosis repeatable on a caller's own labelled set.
+## Attributing tournament errors
+
+For a labelled many-option choice set, `laya.evals_tournament.evaluate_tournament`
+runs the existing `predict_tournament` path and keeps its finalist metadata in the
+regular evaluation report:
+
+```python
+import laya
+from laya.evals import Dataset
+from laya.evals_tournament import evaluate_tournament
+
+agent = laya.load()
+dataset_path = "intents.jsonl"
+report = evaluate_tournament(
+ agent, Dataset.from_jsonl(dataset_path), group_size=16,
+ checkpoint_id="my-checkpoint@revision", dataset_path=dataset_path,
+)
+print(report.overall)
+print(report.cases[0]["tournament_status"])
+```
+
+Each choice case records its finalists and number of elimination rounds, and is
+classified as `correct`, `elimination_miss` (the gold label did not reach the
+final call), or `final_choice_miss` (it survived, but the final call chose another
+label). `tournament_survival_rate` is the fraction of all choices whose gold label
+reached the final call; choices needing no elimination are included. Conditional
+`tournament_accuracy_on_survivors` is omitted when none survived. Ordinary
+`choice_accuracy` remains end-to-end accuracy, and all three metrics appear in the
+existing language, model, question and tag slices. Request latency includes every
+tournament round; the report does not time rounds separately.
+
+Use the checkpoint and group size you intend to deploy. The caller supplies the
+checkpoint identifier, ideally an immutable revision; the report cannot inspect
+the weights. Passing `dataset_path` records a SHA256 of the JSONL bytes. The
+`laya-tournament-evals-report/1` schema and group size prevent a baseline gate
+from comparing different strategies as though they were the same measurement.
+Compare group sizes side by side on development data, then evaluate the chosen
+size once on held-out data. The runtime's final probabilities and confidence are
+over finalists, not over the original labels. The metadata identifies whether
+elimination lost the gold label, but not which round lost it.
+
## Evaluating an ONNX export
`run --onnx PATH` scores an exported ONNX model through `ONNXAgent` instead of the torch
diff --git a/laya/evals.py b/laya/evals.py
index 1506d35e..2d35f682 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -726,6 +726,10 @@ class EvalReport:
continue
reasons.append("%s (%s): baseline is %s, this run is %s"
% (key, _IDENTITY_LABELS[key], there, here))
+ if self.config.get("schema") == other.get("schema") == "laya-tournament-evals-report/1":
+ here, there = self.config.get("tournament_group_size"), other.get("tournament_group_size")
+ if here is None or there is None or here != there:
+ reasons.append("tournament group size: baseline is %s, this run is %s" % (there, here))
return (not reasons), reasons
def compare(self, baseline: Dict[str, Any], tolerances: Optional[Dict[str, float]] = None,
@@ -1143,6 +1147,9 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
shortlist = (result or {}).get("shortlist")
if isinstance(shortlist, dict) and qid in shortlist:
case["shortlist"] = shortlist[qid]
+ tournament = (result or {}).get("tournament")
+ if isinstance(tournament, dict) and qid in tournament:
+ case["tournament"] = tournament[qid]
cases.append(case)
index += len(chunk)
diff --git a/laya/evals_tournament.py b/laya/evals_tournament.py
new file mode 100644
index 00000000..b619d5ad
--- /dev/null
+++ b/laya/evals_tournament.py
@@ -0,0 +1,142 @@
+"""Opt-in attribution of tournament elimination and final-choice errors."""
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Sequence
+
+from .evals import Dataset, EvalError, EvalReport, _group_cases, evaluate, file_fingerprint
+from .shortlist import DEFAULT_TOURNAMENT_GROUP, predict_tournament
+
+TOURNAMENT_REPORT_SCHEMA = "laya-tournament-evals-report/1"
+
+
+def _labels(question: Dict[str, Any], where: str) -> list[str]:
+ criteria = question.get("criteria")
+ if isinstance(criteria, dict):
+ labels = list(criteria)
+ elif isinstance(criteria, list):
+ labels = criteria
+ else:
+ raise EvalError("%s: choice criteria must be a dict or list" % where)
+ if not labels or any(not isinstance(label, str) for label in labels):
+ raise EvalError("%s: choice criteria must contain string labels" % where)
+ if len(set(labels)) != len(labels):
+ raise EvalError("%s: choice criteria contain duplicate labels" % where)
+ return labels
+
+
+def _validate_dataset(dataset: Dataset) -> None:
+ if not isinstance(dataset, Dataset) or not dataset.examples:
+ raise EvalError("tournament evaluation needs a non-empty Dataset")
+ for index, example in enumerate(dataset.examples):
+ if not example.expected:
+ raise EvalError("row %d: expected must contain a choice label" % index)
+ for qid, expected in example.expected.items():
+ where = "row %d question %r" % (index, qid)
+ question = example.questions.get(qid)
+ if not isinstance(question, dict) or question.get("type") != "choice":
+ raise EvalError("%s: tournament evaluation requires a choice question" % where)
+ labels = _labels(question, where)
+ if not isinstance(expected, str) or expected not in labels:
+ raise EvalError("%s: expected label %r is not in criteria" % (where, expected))
+
+
+def _shape(n: int, group_size: int) -> tuple[int, int]:
+ rounds = 0
+ while n > group_size:
+ n = -(-n // group_size)
+ rounds += 1
+ return rounds, n
+
+
+class _TournamentRunner:
+ def __init__(self, agent: Any, group_size: int):
+ self.agent, self.group_size = agent, group_size
+
+ def predict(self, state: Any, questions: Dict[str, Any], model: Optional[str] = None) -> Dict[str, Any]:
+ kwargs = {"model": model} if model is not None else {}
+ result = predict_tournament(self.agent, state, questions, self.group_size, **kwargs)
+ meta = result.get("tournament")
+ choice_ids = {qid for qid, question in questions.items()
+ if isinstance(question, dict) and question.get("type") == "choice"}
+ if not isinstance(meta, dict) or set(meta) != choice_ids:
+ raise EvalError("predict_tournament returned incomplete tournament metadata")
+ answers = result.get("answers")
+ for qid in choice_ids:
+ original = _labels(questions[qid], "question %r" % qid)
+ item = meta[qid]
+ rounds, finalists = _shape(len(original), self.group_size)
+ kept = item.get("labels") if isinstance(item, dict) else None
+ if (not isinstance(kept, list) or len(kept) != finalists
+ or any(not isinstance(label, str) or label not in original for label in kept)
+ or len(set(kept)) != len(kept)
+ or kept != [label for label in original if label in kept]
+ or type(item.get("n")) is not int or item["n"] != len(original)
+ or type(item.get("rounds")) is not int
+ or item["rounds"] != rounds):
+ raise EvalError("question %r: inconsistent tournament metadata" % qid)
+ answer = answers.get(qid) if isinstance(answers, dict) else None
+ if (not isinstance(answer, dict) or answer.get("type") != "choice"
+ or answer.get("choice") not in kept):
+ raise EvalError("question %r: choice answer is outside the finalists" % qid)
+ return result
+
+
+def _metrics(cases: Sequence[Dict[str, Any]]) -> Dict[str, float]:
+ if not cases:
+ return {}
+ survived = sum(case["tournament_status"] != "elimination_miss" for case in cases)
+ correct = sum(case["tournament_status"] == "correct" for case in cases)
+ metrics = {"tournament_survival_rate": survived / len(cases)}
+ if survived:
+ metrics["tournament_accuracy_on_survivors"] = correct / survived
+ return metrics
+
+
+def evaluate_tournament(
+ agent: Any,
+ dataset: Dataset,
+ *,
+ group_size: int = DEFAULT_TOURNAMENT_GROUP,
+ checkpoint_id: str,
+ dataset_path: Optional[str] = None,
+) -> EvalReport:
+ """Score labelled choices and attribute misses to elimination or the final choice.
+
+ ``checkpoint_id`` is a caller-supplied identifier, ideally an immutable revision.
+ ``dataset_path`` adds the JSONL file's SHA256. The existing choice accuracy is
+ end-to-end; tournament confidence and probabilities describe finalists only.
+ ``tournament_survival_rate`` includes choices that needed no elimination round.
+ """
+ if isinstance(group_size, bool) or not isinstance(group_size, int) or group_size < 2:
+ raise EvalError("group_size must be an integer of at least 2")
+ if not isinstance(checkpoint_id, str) or not checkpoint_id.strip():
+ raise EvalError("checkpoint_id must be a non-empty string")
+ _validate_dataset(dataset)
+ config: Dict[str, Any] = {
+ "tournament_group_size": group_size,
+ "tournament": {"group_size": group_size, "checkpoint_id": checkpoint_id},
+ }
+ if dataset_path is not None:
+ config["dataset"] = dataset_path
+ config["dataset_sha256"] = file_fingerprint(dataset_path)
+ report = evaluate(_TournamentRunner(agent, group_size), dataset, config=config)
+ for case in report.cases:
+ item = case.get("tournament")
+ if not isinstance(item, dict) or not isinstance(item.get("labels"), list):
+ raise EvalError("question %r: missing tournament metadata" % case["qid"])
+ if case["expected"] not in item["labels"]:
+ case["tournament_status"] = "elimination_miss"
+ elif case["correct"]:
+ case["tournament_status"] = "correct"
+ else:
+ case["tournament_status"] = "final_choice_miss"
+ report.overall.update(_metrics(report.cases))
+ for dimension in ("language", "model", "qid", "tag"):
+ for value, group in _group_cases(report.cases, dimension).items():
+ report.slices[dimension][value].update(_metrics(group))
+ report.config["schema"] = TOURNAMENT_REPORT_SCHEMA
+ report.config["tournament"]["counts"] = {
+ status: sum(case["tournament_status"] == status for case in report.cases)
+ for status in ("correct", "elimination_miss", "final_choice_miss")
+ }
+ return report
diff --git a/scripts/test_suites.py b/scripts/test_suites.py
index 922ade5f..949baa9a 100644
--- a/scripts/test_suites.py
+++ b/scripts/test_suites.py
@@ -106,6 +106,7 @@ SCRIPT_SUITES = [
PYTEST_SUITES = [
"tests/test_evals_agreement.py",
+ "tests/test_evals_tournament.py",
"tests/test_serve.py",
"tests/test_mcp_remote.py",
"tests/test_router_batch.py",
diff --git a/tests/test_evals_tournament.py b/tests/test_evals_tournament.py
new file mode 100644
index 00000000..9eb32062
--- /dev/null
+++ b/tests/test_evals_tournament.py
@@ -0,0 +1,148 @@
+"""Weight-free checks for tournament stage attribution and report identity."""
+import json
+
+import pytest
+
+from laya.evals import Dataset, EvalError, Example, evaluate
+from laya.evals_tournament import TOURNAMENT_REPORT_SCHEMA, evaluate_tournament
+
+
+WIDE = {"intent": {"type": "choice", "instructions": "Which?",
+ "criteria": ["a", "b", "c", "d", "e", "f"]}}
+NARROW = {"intent": {"type": "choice", "instructions": "Which?", "criteria": ["a", "b"]}}
+
+
+class ChoiceAgent:
+ def __init__(self):
+ self.received = []
+
+ def system_one(self, state, questions):
+ self.received.append((state, {qid: list(q["criteria"]) for qid, q in questions.items()}))
+ answers = {}
+ for qid, question in questions.items():
+ labels = list(question["criteria"])
+ if state == "early" and labels == ["a", "b"]:
+ choice = "a"
+ elif state == "late" and labels == ["c", "e"]:
+ choice = "e"
+ elif (state == "final" and labels == ["a", "c"]) or (state == "pass" and labels == ["a", "b"]):
+ choice = labels[-1]
+ else:
+ choice = labels[0]
+ answers[qid] = {"type": "choice", "choice": choice, "answer_confidence": 0.7}
+ return {"model": "stub", "answers": answers}
+
+
+def row(state, gold, *, narrow=False, tag="test"):
+ return Example(state, NARROW if narrow else WIDE, {"intent": gold}, tags=(tag,))
+
+
+def run(dataset, group_size=2):
+ return evaluate_tournament(ChoiceAgent(), dataset, group_size=group_size,
+ checkpoint_id="checkpoint@123")
+
+
+def test_stage_attribution_and_slices():
+ dataset = Dataset([row("ok", "a", tag="kept"), row("final", "a", tag="kept"),
+ row("early", "b", tag="lost"), row("late", "c", tag="lost"),
+ row("pass", "a", narrow=True, tag="kept")])
+ report = run(dataset)
+ assert [case["tournament_status"] for case in report.cases] == [
+ "correct", "final_choice_miss", "elimination_miss", "elimination_miss",
+ "final_choice_miss"]
+ assert [case["tournament"]["rounds"] for case in report.cases] == [2, 2, 2, 2, 0]
+ assert report.overall["tournament_survival_rate"] == pytest.approx(3 / 5)
+ assert report.overall["tournament_accuracy_on_survivors"] == pytest.approx(1 / 3)
+ assert report.overall["choice_accuracy"] == pytest.approx(1 / 5)
+ assert report.slices["tag"]["lost"]["tournament_survival_rate"] == 0.0
+ assert "tournament_accuracy_on_survivors" not in report.slices["tag"]["lost"]
+ assert report.config["schema"] == TOURNAMENT_REPORT_SCHEMA
+ assert report.config["tournament"]["counts"] == {
+ "correct": 1, "elimination_miss": 2, "final_choice_miss": 2}
+ assert json.loads(json.dumps(report.to_json()))["cases"][3]["tournament_status"] == "elimination_miss"
+
+
+def test_passthrough_and_group_size_are_reported_and_compared():
+ dataset = Dataset([row("ok", "a", narrow=True)])
+ report = run(dataset)
+ assert report.cases[0]["tournament"]["labels"] == ["a", "b"]
+ assert report.cases[0]["tournament"]["rounds"] == 0
+ assert report.overall["tournament_survival_rate"] == 1.0
+ same = run(dataset)
+ assert report.comparable_to(same.to_json())[0]
+ wider = run(dataset, group_size=3)
+ ok, reasons = report.comparable_to(wider.to_json())
+ assert not ok
+ assert any("tournament group size" in reason for reason in reasons)
+ missing = same.to_json()
+ missing["config"].pop("tournament_group_size")
+ assert not report.comparable_to(missing)[0]
+
+
+def test_invalid_inputs_fail_before_inference():
+ with pytest.raises(EvalError, match="not in criteria"):
+ run(Dataset([row("ok", "missing")]))
+ with pytest.raises(EvalError, match="choice question"):
+ run(Dataset([Example("ok", {"intent": {"type": "score"}}, {"intent": 1})]))
+ with pytest.raises(EvalError, match="at least 2"):
+ run(Dataset([row("ok", "a")]), group_size=True)
+ with pytest.raises(EvalError, match="checkpoint_id"):
+ evaluate_tournament(ChoiceAgent(), Dataset([row("ok", "a")]), checkpoint_id=" ")
+
+
+def test_malformed_tournament_metadata_fails(monkeypatch):
+ def malformed(*_args, **_kwargs):
+ return {"answers": {"intent": {"type": "choice", "choice": "a"}},
+ "tournament": {"intent": {"labels": ["a", "a"], "n": 6, "rounds": 2}}}
+
+ monkeypatch.setattr("laya.evals_tournament.predict_tournament", malformed)
+ with pytest.raises(EvalError, match="inconsistent tournament metadata"):
+ run(Dataset([row("ok", "a")]))
+
+
+def test_explicit_model_reaches_every_round():
+ class ModelAgent(ChoiceAgent):
+ def predict(self, state, questions, model=None):
+ assert model == "english"
+ return self.system_one(state, questions)
+
+ agent = ModelAgent()
+ example = Example("ok", WIDE, {"intent": "a"}, model="english")
+ report = evaluate_tournament(agent, Dataset([example]), group_size=2,
+ checkpoint_id="checkpoint@123")
+ assert len(agent.received) == 3 # two elimination rounds and the final call
+ assert report.slices["model"]["english"]["choice_accuracy"] == 1.0
+
+
+def test_multiple_questions_and_described_criteria():
+ questions = {
+ "route": {"type": "choice", "criteria": {label: label.upper() for label in "abcdef"}},
+ "fallback": NARROW["intent"],
+ }
+ report = run(Dataset([Example("ok", questions, {"route": "a", "fallback": "a"})]))
+ assert [case["qid"] for case in report.cases] == ["route", "fallback"]
+ assert [case["tournament"]["rounds"] for case in report.cases] == [2, 0]
+ assert report.slices["qid"]["route"]["tournament_survival_rate"] == 1.0
+ assert report.slices["qid"]["fallback"]["choice_accuracy"] == 1.0
+
+
+def test_dataset_hash_and_plain_evaluate(tmp_path):
+ path = tmp_path / "intents.jsonl"
+ path.write_text(json.dumps({"state": "ok", "questions": WIDE,
+ "expected": {"intent": "a"}}) + "\n", encoding="utf-8")
+ dataset = Dataset.from_jsonl(str(path))
+ report = evaluate_tournament(ChoiceAgent(), dataset, group_size=2,
+ checkpoint_id="checkpoint@123", dataset_path=str(path))
+ from laya.evals import file_fingerprint
+
+ assert report.config["dataset_sha256"] == file_fingerprint(str(path))
+ assert report.config["questions_sha256"]
+
+ class PlainRunner:
+ def predict(self, state, questions, model=None):
+ return {"answers": {"intent": {"type": "choice", "choice": "a"}}}
+
+ plain = evaluate(PlainRunner(), Dataset([row("ok", "a", narrow=True)]))
+ assert plain.config["schema"] == "laya-evals-report/1"
+ assert "tournament_survival_rate" not in plain.overall
+ assert "tournament" not in plain.cases[0]
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index f1914b7e..fe53d252 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -1073,6 +1073,12 @@ check_param("evaluate_shortlist", evaluate_shortlist, "dataset_path", None)
for param in ("checkpoint_id", "embedder_id"):
check_param("evaluate_shortlist", evaluate_shortlist, param, inspect.Parameter.empty)
+from laya.evals_tournament import evaluate_tournament # noqa: E402
+
+check_param("evaluate_tournament", evaluate_tournament, "group_size", 16)
+check_param("evaluate_tournament", evaluate_tournament, "dataset_path", None)
+check_param("evaluate_tournament", evaluate_tournament, "checkpoint_id", inspect.Parameter.empty)
+
# docs/hooks/examples.md ## Composition teaches one scope ordering contract; the pre-fix wording
# ("Installed hooks first, then convenience callables") named only two tiers and put installed at