Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): drop model= for runners whose predict predates it (#1107)

akushonkamen747e62eea175
diff --git a/laya/evals.py b/laya/evals.py
index b066859c..94c163d0 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -898,6 +898,29 @@ def _takes_min_confidence(runner: Any, fn_name: str = "predict_batch") -> bool:
return any(p.name == "min_confidence" or p.kind is p.VAR_KEYWORD for p in params)
+def _takes_model(runner: Any, fn_name: str = "predict") -> bool:
+ """Whether `runner`'s entry point accepts a per-example checkpoint on the call.
+
+ The same signature check `_takes_min_confidence` makes, applied to the `model` kwarg the
+ single-row path forwards from `Example.model` (#1107). The batch half of this exact mistake
+ was `_batch_form`'s founding bug -- every chunk raised ``TypeError: ... unexpected keyword
+ argument 'model'``, which `on_error="skip"` turned into a zero-case report -- and
+ `laya.load()`'s default runner still hit it one row at a time, because `Agent.predict` is
+ `system_one`, which takes neither `model` nor any `**kwargs` forwarder. Unlike the threshold,
+ an unsent `model` is no lie: a runner without the parameter answers from its own checkpoint.
+ So the harness drops the kwarg when no example names one, and the pre-flight refusal in
+ `evaluate` fires only when an example does.
+ """
+ fn = getattr(runner, fn_name, None)
+ if fn is None:
+ return False
+ try:
+ params = inspect.signature(fn).parameters.values()
+ except (TypeError, ValueError):
+ return False
+ return any(p.name == "model" or p.kind is p.VAR_KEYWORD for p in params)
+
+
def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluator]] = None,
batch_size: Optional[int] = None, on_error: str = "fail",
config: Optional[Dict[str, Any]] = None, sort_by_length: bool = False,
@@ -909,7 +932,11 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
``predict_batch(states, questions, model=..., batch_size=...)``, or the per-request-dict form
`Router.predict_batch` takes -- ``predict_batch([{"state": ..., "questions": ...,
"model": ...}, ...], batch_size=...)``. A runner that offers neither is scored one ``predict``
- at a time, which is slower but not wrong.
+ at a time, which is slower but not wrong. A runner whose ``predict`` takes no ``model`` --
+ ``laya.load()``'s default `Agent` is one, its ``predict`` being an alias of `system_one`
+ (#1107) -- is scored on its own checkpoint: the kwarg is dropped unless an example names an
+ explicit ``model``, and a dataset that names one on such a runner is refused in pre-flight
+ rather than published as a silent zero-case report.
`sort_by_length` is forwarded to a chunked runner so similarly sized examples share a forward
pass and pad to a shorter maximum. It reaches the checkpoint's own batching only when the
@@ -972,6 +999,21 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
"abstention threshold. Either use a Router/ONNXAgent that gates on "
"answer_confidence (#361), or drop the threshold -- the report would "
"otherwise name a `min_confidence` it never applied." % (min_confidence, target))
+ # Whether `predict` can be handed `Example.model` at all (#1107). `Agent.predict` is
+ # `system_one` -- no `model`, no `**kwargs` -- so the single-row path used to raise `TypeError`
+ # on every row, and `on_error="skip"` published a zero-case report. The batch half was already
+ # gated by `_batch_form`; this is the single-row guard, computed once and reused at the call.
+ takes_model = _takes_model(runner)
+ if not takes_model and any(e.model is not None for e in dataset.examples):
+ # Dropping the kwarg here would score each row on whatever checkpoint the runner answers
+ # with while the report's `model` slice labels it with the named one -- the same class of
+ # lie the `min_confidence` guard exists to prevent. The kwarg is dropped only when no
+ # example names a checkpoint, in which case scoring on the runner's own is what was asked.
+ raise EvalError(
+ "evaluate() refused: the dataset names an explicit per-example `model`, but this "
+ "runner's predict does not accept one. Either use a Router/ONNXAgent whose predict "
+ "forwards the checkpoint, or drop the per-example `model` -- the report would "
+ "otherwise name a checkpoint it never sent (#1107).")
evaluators = list(evaluators) if evaluators is not None else default_evaluators()
cases: List[Dict[str, Any]] = []
waits: List[float] = [] # what each request actually waited: its chunk's whole call
@@ -1034,9 +1076,15 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
else:
# A chunk of one still sees the gate: the pre-flight refusal proved `predict`
# accepts it whenever `mc` is set, so nothing here has to re-check the signature.
+ # The `model` kwarg is the same honesty (#1107): sent only when `predict` accepts
+ # it, and the pre-flight already refused any run where an example needed it sent.
single_kwargs = {"min_confidence": mc} if mc is not None else {}
- results = [runner.predict(chunk[0].state, chunk[0].questions,
- model=chunk[0].model, **single_kwargs)]
+ if takes_model:
+ results = [runner.predict(chunk[0].state, chunk[0].questions,
+ model=chunk[0].model, **single_kwargs)]
+ else:
+ results = [runner.predict(chunk[0].state, chunk[0].questions,
+ **single_kwargs)]
except Exception as exc: # noqa: BLE001 -- honoured by on_error
if on_error == "fail":
raise
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 12ed6555..e96ad739 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -1249,6 +1249,86 @@ def test_cli_min_confidence_zero_is_accepted(monkeypatch, tmp_path):
assert seen["min_confidence"] == 0.0
+# ------------------------------------------------- per-example model (#1107 guard)
+#
+# `Example.model` names an explicit checkpoint, and the single-row path used to forward it
+# unconditionally -- `model=chunk[0].model`, even when the value was `None`. `Agent.predict` is
+# `system_one`, which takes neither `model` nor `**kwargs`, so every row raised `TypeError` and
+# `on_error="skip"` published a zero-case, empty-overall, exit-0 report: the batch half of this
+# exact bug was `_batch_form`'s founding case, the single-row path was left outside it.
+
+
+class AgentShapedRunner:
+ """What `laya.load()` returns: `predict` is `system_one` -- no `model`, no `**kwargs`."""
+
+ def __init__(self, by_state):
+ self.by_state = by_state
+
+ def predict(self, state, questions, lang=None, min_confidence=None):
+ return {"answers": self.by_state[state]}
+
+ def predict_batch(self, states, questions, batch_size=None, lang=None):
+ return [self.predict(s, questions) for s in states]
+
+
+def test_an_agent_shaped_runner_scores_instead_of_a_zero_case_report():
+ """Every row of the issue's report: a runner whose `predict` predates `model` used to fail
+ row by row and -- under the default skip -- publish 0 cases with an empty `overall`."""
+ report = evaluate(AgentShapedRunner(ANSWERS3), _three(), evaluators=[ChoiceAccuracy()],
+ batch_size=4, on_error="skip")
+ assert len(report.cases) == 3, "the rows score; they do not vanish into on_error"
+ assert report.overall["choice_accuracy"] == pytest.approx(1.0)
+ assert not report.config.get("errored"), "a scoring run is not an erroring one"
+
+
+def test_the_model_kwarg_is_dropped_for_a_runner_without_one():
+ """With `on_error="fail"` the dropped kwarg is observable: had the harness still sent
+ `model=None`, every call here would raise `TypeError` instead of scoring."""
+ report = evaluate(AgentShapedRunner(ANSWERS3), _three(), evaluators=[ChoiceAccuracy()],
+ batch_size=4)
+ assert len(report.cases) == 3
+ assert report.overall["choice_accuracy"] == pytest.approx(1.0)
+
+
+def test_a_runner_that_takes_model_still_receives_it():
+ """The guard drops the kwarg only for a runner without the parameter: a `Router`-shaped
+ `predict(state, questions, model=...)` keeps seeing the call the docstring promises."""
+ seen = {}
+
+ class Recording(StubRunner):
+ def predict(self, state, questions, model=None):
+ seen[state] = model
+ return StubRunner.predict(self, state, questions, model)
+
+ report = evaluate(Recording(ANSWERS3), _three(), evaluators=[ChoiceAccuracy()], batch_size=2)
+ assert set(seen.values()) == {None}, "no example names a checkpoint, so `None` is the value"
+ assert len(report.cases) == 3
+
+
+def test_an_explicit_model_on_a_runner_without_the_parameter_is_refused():
+ """Dropping the kwarg when an example names a checkpoint would score the row on whatever the
+ runner answers with while the `model` slice labels it with the named one -- a lying report.
+ The pre-flight refuses instead, before any weights load, even under `on_error="skip"`."""
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}, model="english"),
+ Example("s2", Q, {"intent": "a"})])
+ with pytest.raises(EvalError) as exc:
+ evaluate(AgentShapedRunner(ANSWERS3), dataset, evaluators=[ChoiceAccuracy()],
+ batch_size=4, on_error="skip")
+ assert "model" in str(exc.value)
+ assert "does not accept" in str(exc.value)
+
+
+def test_an_explicit_model_still_reaches_a_runner_that_takes_it():
+ """The refusal is scoped to runners without the parameter; a named checkpoint keeps flowing
+ through both entry points of a `Router`-shaped runner."""
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}, model="english"),
+ Example("s2", Q, {"intent": "a"}, model="english")])
+ report = evaluate(StubRunner({"s1": {"intent": choice_answer("a")},
+ "s2": {"intent": choice_answer("a")}}),
+ dataset, evaluators=[ChoiceAccuracy()], batch_size=2)
+ assert report.overall["choice_accuracy"] == 1.0
+
+
# --------------------------------------------------------------- timing (#585)
FORWARD_MS = 100.0
SHARED = 0.6 # a batch of n costs SHARED * n * FORWARD_MS, as one call