Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): label a routed row by the checkpoint that answered
Aashish20bf0758dc29
diff --git a/docs/evals.md b/docs/evals.mdindex 3660ec3a..073f24e2 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -40,7 +40,7 @@ One JSON object per line (JSONL). Blank lines and lines starting with `#` are ig| `expected` | yes | ground truth keyed by question id: a label for `choice`, a number for `score`, `true`/`false` for `noul` || `tags` | no | strings to slice by || `language` | no | a code to slice by |-| `model` | no | force a checkpoint for this row; `--model` overrides it |+| `model` | no | force a checkpoint for this row; `--model` overrides it. A row that forces nothing is labelled with whatever checkpoint the `Router` answered with |`research/evals/dataset.template.jsonl` has a commented example.@@ -65,7 +65,8 @@ Add `ScoreWithin(0.25)` to the evaluator list for a tolerance metric; the defaul`compare` and `run` report overall numbers and, for `--slice language|model|qid|tag`, the samemetrics per slice value, so a regression in one language or one question is visible without-reading the aggregate.+reading the aggregate. The `model` slice holds the checkpoint that answered each row: the+`Router`'s own choice per request, or the runner's `model` for a runner that does not route.## Baseline and CI gatediff --git a/laya/evals.py b/laya/evals.pyindex cac581d6..21ee5301 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -189,6 +189,23 @@ def ece(confidences: Sequence[float], corrects: Sequence[bool], bins: int = 15)np.asarray(corrects, dtype=bool), bins=bins))+def _answered_model(result: Any) -> Optional[Any]:+ """Which checkpoint produced `result`, read the way each runner records it.++ A `Router` puts its choice under `routing["model"]` and leaves no checkpoint name at the top+ level: `result["model"]` is the payload's family tag, `"laya-rl-agent"`, on every Agent and+ ONNXAgent result too. So a routed run has to read `routing`, or its `by model` slice reports+ one family for every checkpoint that answered.+ """+ result = result or {}+ routing = result.get("routing")+ if isinstance(routing, dict):+ chosen = routing.get("model")+ if isinstance(chosen, str) and chosen:+ return chosen+ return result.get("model")++# --------------------------------------------------------------------------- evaluation@dataclassclass EvalReport:@@ -334,7 +351,7 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluacases.append({"qid": qid,"language": example.language,- "model": example.model or (result or {}).get("model"),+ "model": example.model or _answered_model(result),"tags": list(example.tags),"expected": expected,"answer": answer,diff --git a/tests/test_evals.py b/tests/test_evals.pyindex 0e9ff2d7..8d26d931 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -143,6 +143,65 @@ def test_evaluate_overall_and_slices():assert "choice_accuracy" in report.to_markdown()+def test_model_slice_follows_the_routed_checkpoint():+ """A Router records the checkpoint it chose under `routing`, not at the top level.++ `result["model"]` is the payload's family tag -- `laya-rl-agent` -- on every Laya runner, so+ reading only that collapses `by model` into one bucket for a mixed-language run.+ """+ class RoutedRunner(StubRunner):+ def _result(self, state, model):+ # What this runner answers with. The pinned row is deliberately routed somewhere+ # else, so the case label can only come from the caller's pin.+ chosen = "english" if state == "s_en" else "multilingual"+ return {"model": "laya-rl-agent", "routing": {"model": chosen},+ "answers": self.by_state[state]}++ def predict(self, state, questions, model=None):+ return self._result(state, model)++ def predict_batch(self, states, questions, model=None, batch_size=None):+ return [self._result(s, model) for s in states]++ dataset = Dataset([+ Example("s_en", Q, {"intent": "a"}),+ Example("s_de", Q, {"intent": "a"}),+ Example("pinned", Q, {"intent": "a"}, model="english"),+ ])+ runner = RoutedRunner({s: {"intent": choice_answer("a")} for s in ("s_en", "s_de", "pinned")})+ wanted = ["english", "multilingual", "english"]+ for kwargs in ({}, {"batch_size": 8}):+ report = evaluate(runner, dataset, evaluators=[ChoiceAccuracy()], **kwargs)+ assert [c["model"] for c in report.cases] == wanted, kwargs+ assert sorted(report.slices["model"]) == ["english", "multilingual"], kwargs+++def test_model_slice_falls_back_when_a_runner_reports_no_route():+ # Absent, empty, or not a usable checkpoint name -- none of these may label a case.+ UNUSABLE = [None, {}, {"model": None}, {"model": ""}, {"model": 5}, "english", ["english"]]++ class OddRunner(StubRunner):+ def __init__(self, by_state, routing):+ super().__init__(by_state)+ self.routing = routing++ def predict(self, state, questions, model=None):+ result = {"model": "laya-rl-agent", "answers": self.by_state[state]}+ if self.routing is not None:+ result["routing"] = self.routing+ return result++ answers = {"s": {"intent": choice_answer("a")}}+ dataset = Dataset([Example("s", Q, {"intent": "a"})])+ for routing in UNUSABLE:+ report = evaluate(OddRunner(answers, routing), dataset, evaluators=[ChoiceAccuracy()])+ assert sorted(report.slices["model"]) == ["laya-rl-agent"], routing+ # An Agent runner that never routes is labelled by its own payload, as before.+ report = evaluate(StubRunner({"s1": {"intent": choice_answer("a")}}),+ Dataset([Example("s1", Q, {"intent": "a"})]), evaluators=[ChoiceAccuracy()])+ assert sorted(report.slices["model"]) == ["stub"]++def test_evaluate_batches_same_questions():class BatchRunner(StubRunner):def __init__(self, by_state):