Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): label a routed row by the checkpoint that answered

Aashish20bf0758dc29
diff --git a/docs/evals.md b/docs/evals.md
index 3660ec3a..073f24e2 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -40,7 +40,7 @@ One JSON object per line (JSONL). Blank lines and lines starting with `#` are ig
| `expected` | yes | ground truth keyed by question id: a label for `choice`, a number for `score`, `true`/`false` for `noul` |
| `tags` | no | strings to slice by |
| `language` | no | a code to slice by |
-| `model` | no | force a checkpoint for this row; `--model` overrides it |
+| `model` | no | force a checkpoint for this row; `--model` overrides it. A row that forces nothing is labelled with whatever checkpoint the `Router` answered with |
`research/evals/dataset.template.jsonl` has a commented example.
@@ -65,7 +65,8 @@ Add `ScoreWithin(0.25)` to the evaluator list for a tolerance metric; the defaul
`compare` and `run` report overall numbers and, for `--slice language|model|qid|tag`, the same
metrics per slice value, so a regression in one language or one question is visible without
-reading the aggregate.
+reading the aggregate. The `model` slice holds the checkpoint that answered each row: the
+`Router`'s own choice per request, or the runner's `model` for a runner that does not route.
## Baseline and CI gate
diff --git a/laya/evals.py b/laya/evals.py
index cac581d6..21ee5301 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -189,6 +189,23 @@ def ece(confidences: Sequence[float], corrects: Sequence[bool], bins: int = 15)
np.asarray(corrects, dtype=bool), bins=bins))
+def _answered_model(result: Any) -> Optional[Any]:
+ """Which checkpoint produced `result`, read the way each runner records it.
+
+ A `Router` puts its choice under `routing["model"]` and leaves no checkpoint name at the top
+ level: `result["model"]` is the payload's family tag, `"laya-rl-agent"`, on every Agent and
+ ONNXAgent result too. So a routed run has to read `routing`, or its `by model` slice reports
+ one family for every checkpoint that answered.
+ """
+ result = result or {}
+ routing = result.get("routing")
+ if isinstance(routing, dict):
+ chosen = routing.get("model")
+ if isinstance(chosen, str) and chosen:
+ return chosen
+ return result.get("model")
+
+
# --------------------------------------------------------------------------- evaluation
@dataclass
class EvalReport:
@@ -334,7 +351,7 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
cases.append({
"qid": qid,
"language": example.language,
- "model": example.model or (result or {}).get("model"),
+ "model": example.model or _answered_model(result),
"tags": list(example.tags),
"expected": expected,
"answer": answer,
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 0e9ff2d7..8d26d931 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -143,6 +143,65 @@ def test_evaluate_overall_and_slices():
assert "choice_accuracy" in report.to_markdown()
+def test_model_slice_follows_the_routed_checkpoint():
+ """A Router records the checkpoint it chose under `routing`, not at the top level.
+
+ `result["model"]` is the payload's family tag -- `laya-rl-agent` -- on every Laya runner, so
+ reading only that collapses `by model` into one bucket for a mixed-language run.
+ """
+ class RoutedRunner(StubRunner):
+ def _result(self, state, model):
+ # What this runner answers with. The pinned row is deliberately routed somewhere
+ # else, so the case label can only come from the caller's pin.
+ chosen = "english" if state == "s_en" else "multilingual"
+ return {"model": "laya-rl-agent", "routing": {"model": chosen},
+ "answers": self.by_state[state]}
+
+ def predict(self, state, questions, model=None):
+ return self._result(state, model)
+
+ def predict_batch(self, states, questions, model=None, batch_size=None):
+ return [self._result(s, model) for s in states]
+
+ dataset = Dataset([
+ Example("s_en", Q, {"intent": "a"}),
+ Example("s_de", Q, {"intent": "a"}),
+ Example("pinned", Q, {"intent": "a"}, model="english"),
+ ])
+ runner = RoutedRunner({s: {"intent": choice_answer("a")} for s in ("s_en", "s_de", "pinned")})
+ wanted = ["english", "multilingual", "english"]
+ for kwargs in ({}, {"batch_size": 8}):
+ report = evaluate(runner, dataset, evaluators=[ChoiceAccuracy()], **kwargs)
+ assert [c["model"] for c in report.cases] == wanted, kwargs
+ assert sorted(report.slices["model"]) == ["english", "multilingual"], kwargs
+
+
+def test_model_slice_falls_back_when_a_runner_reports_no_route():
+ # Absent, empty, or not a usable checkpoint name -- none of these may label a case.
+ UNUSABLE = [None, {}, {"model": None}, {"model": ""}, {"model": 5}, "english", ["english"]]
+
+ class OddRunner(StubRunner):
+ def __init__(self, by_state, routing):
+ super().__init__(by_state)
+ self.routing = routing
+
+ def predict(self, state, questions, model=None):
+ result = {"model": "laya-rl-agent", "answers": self.by_state[state]}
+ if self.routing is not None:
+ result["routing"] = self.routing
+ return result
+
+ answers = {"s": {"intent": choice_answer("a")}}
+ dataset = Dataset([Example("s", Q, {"intent": "a"})])
+ for routing in UNUSABLE:
+ report = evaluate(OddRunner(answers, routing), dataset, evaluators=[ChoiceAccuracy()])
+ assert sorted(report.slices["model"]) == ["laya-rl-agent"], routing
+ # An Agent runner that never routes is labelled by its own payload, as before.
+ report = evaluate(StubRunner({"s1": {"intent": choice_answer("a")}}),
+ Dataset([Example("s1", Q, {"intent": "a"})]), evaluators=[ChoiceAccuracy()])
+ assert sorted(report.slices["model"]) == ["stub"]
+
+
def test_evaluate_batches_same_questions():
class BatchRunner(StubRunner):
def __init__(self, by_state):