Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): adopt the #1112 review — routing test, unified fallback, refusal detail

akushonkamenc05312a5fe05
diff --git a/laya/evals.py b/laya/evals.py
index 94c163d0..d4431a4c 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -901,7 +901,7 @@ def _takes_min_confidence(runner: Any, fn_name: str = "predict_batch") -> bool:
def _takes_model(runner: Any, fn_name: str = "predict") -> bool:
"""Whether `runner`'s entry point accepts a per-example checkpoint on the call.
- The same signature check `_takes_min_confidence` makes, applied to the `model` kwarg the
+ The same signature check `_batch_form` makes, applied to the `model` kwarg the
single-row path forwards from `Example.model` (#1107). The batch half of this exact mistake
was `_batch_form`'s founding bug -- every chunk raised ``TypeError: ... unexpected keyword
argument 'model'``, which `on_error="skip"` turned into a zero-case report -- and
@@ -910,6 +910,16 @@ def _takes_model(runner: Any, fn_name: str = "predict") -> bool:
an unsent `model` is no lie: a runner without the parameter answers from its own checkpoint.
So the harness drops the kwarg when no example names one, and the pre-flight refusal in
`evaluate` fires only when an example does.
+
+ Not introspectable (a C-level or hand-rolled ``__call__``): keep the call this harness makes,
+ the fallback `_batch_form` makes for the same kwarg on ``predict_batch`` -- deliberately not
+ `_takes_min_confidence`'s ``False``, which had the two guards disagreeing: on a runner whose
+ entry points are both C-level, `model` was sent on the batch path and dropped on the single
+ one, and a dataset naming an explicit checkpoint was refused even though the batch entry point
+ would have carried it (#1112 review). With no example naming a model the value is ``None``
+ either way, so the only run the ``True`` fallback changes is the one where sending it is the
+ point; a runner whose attribute is missing is the `fn is None` case, and a C-level runner that
+ refuses the kwarg fails loudly on the call -- the risk `_batch_form` already accepts.
"""
fn = getattr(runner, fn_name, None)
if fn is None:
@@ -917,7 +927,9 @@ def _takes_model(runner: Any, fn_name: str = "predict") -> bool:
try:
params = inspect.signature(fn).parameters.values()
except (TypeError, ValueError):
- return False
+ # Same fallback as `_batch_form` (its comment records why): keep the call this harness
+ # made before the shape was checked, rather than half-dropping the kwarg.
+ return True
return any(p.name == "model" or p.kind is p.VAR_KEYWORD for p in params)
@@ -1004,16 +1016,20 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
# on every row, and `on_error="skip"` published a zero-case report. The batch half was already
# gated by `_batch_form`; this is the single-row guard, computed once and reused at the call.
takes_model = _takes_model(runner)
- if not takes_model and any(e.model is not None for e in dataset.examples):
+ named = next(((i, e) for i, e in enumerate(dataset.examples) if e.model is not None), None)
+ if not takes_model and named is not None:
# Dropping the kwarg here would score each row on whatever checkpoint the runner answers
# with while the report's `model` slice labels it with the named one -- the same class of
# lie the `min_confidence` guard exists to prevent. The kwarg is dropped only when no
# example names a checkpoint, in which case scoring on the runner's own is what was asked.
+ # The refusal names the first offender: on a large dataset, "the dataset names a `model`"
+ # alone leaves the user grepping for the row (#1112 review).
+ first, offender = named
raise EvalError(
- "evaluate() refused: the dataset names an explicit per-example `model`, but this "
- "runner's predict does not accept one. Either use a Router/ONNXAgent whose predict "
- "forwards the checkpoint, or drop the per-example `model` -- the report would "
- "otherwise name a checkpoint it never sent (#1107).")
+ "evaluate() refused: example %d names model %r, but this runner's predict does not "
+ "accept a `model` argument. Either use a Router/ONNXAgent whose predict forwards the "
+ "checkpoint, or drop the per-example `model` -- the report would otherwise name a "
+ "checkpoint it never sent (#1107)." % (first, offender.model))
evaluators = list(evaluators) if evaluators is not None else default_evaluators()
cases: List[Dict[str, Any]] = []
waits: List[float] = [] # what each request actually waited: its chunk's whole call
@@ -1076,8 +1092,10 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
else:
# A chunk of one still sees the gate: the pre-flight refusal proved `predict`
# accepts it whenever `mc` is set, so nothing here has to re-check the signature.
- # The `model` kwarg is the same honesty (#1107): sent only when `predict` accepts
- # it, and the pre-flight already refused any run where an example needed it sent.
+ # The `model` kwarg is the same honesty (#1107): sent only when `predict` takes
+ # it -- kept as-is when the signature cannot be read, `_batch_form`'s fallback on
+ # the same kwarg -- and the pre-flight already refused any run where an example
+ # needed it sent and the guard said no.
single_kwargs = {"min_confidence": mc} if mc is not None else {}
if takes_model:
results = [runner.predict(chunk[0].state, chunk[0].questions,
diff --git a/tests/test_evals.py b/tests/test_evals.py
index e96ad739..a79729df 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -1271,6 +1271,23 @@ class AgentShapedRunner:
return [self.predict(s, questions) for s in states]
+class NotIntrospectableRunner:
+ """Entry points whose signature cannot be read -- a C-level or hand-rolled ``__call__``
+ (#1112 review): `_batch_form` keeps the batch call there, so `_takes_model` must too."""
+
+ class _Callable:
+ __signature__ = None
+
+ def __call__(self, *args, **kwargs):
+ return None
+
+ def __get__(self, obj, cls):
+ return self
+
+ predict = _Callable()
+ predict_batch = _Callable()
+
+
def test_an_agent_shaped_runner_scores_instead_of_a_zero_case_report():
"""Every row of the issue's report: a runner whose `predict` predates `model` used to fail
row by row and -- under the default skip -- publish 0 cases with an empty `overall`."""
@@ -1305,10 +1322,37 @@ def test_a_runner_that_takes_model_still_receives_it():
assert len(report.cases) == 3
+class BatchOnlyRouting(StubRunner):
+ """Routes in the batch entry point but not in the single one -- the shape `_batch_form`
+ approves (#1112 review)."""
+
+ def predict(self, state, questions):
+ return {"model": "single", "answers": self.by_state[state]}
+
+ def predict_batch(self, states, questions, model=None, batch_size=None):
+ return [{"model": model or "batched", "answers": self.by_state[s]} for s in states]
+
+
+def test_a_runner_that_routes_only_in_batch_survives_a_chunk_of_one():
+ """Chunking splits on (model, questions), so a differing row lands on the single `predict`.
+ `takes_model` reads `predict` alone, so this runner keeps batching and survives: the shape
+ that made the bug read as intermittent rather than as a contract mismatch."""
+ mixed = Dataset([Example("s1", Q, {"intent": "a"}),
+ Example("s3", QNOUL, {"flag": True}),
+ Example("s2", Q, {"intent": "a"})])
+ answers = {"s1": {"intent": choice_answer("a")}, "s2": {"intent": choice_answer("a")},
+ "s3": {"flag": noul_answer(0.9)}}
+ report = evaluate(BatchOnlyRouting(answers), mixed,
+ evaluators=[ChoiceAccuracy(), NoulAccuracy()], batch_size=8)
+ assert not report.config.get("errored")
+ assert report.overall["choice_accuracy"] == 1.0
+
+
def test_an_explicit_model_on_a_runner_without_the_parameter_is_refused():
"""Dropping the kwarg when an example names a checkpoint would score the row on whatever the
runner answers with while the `model` slice labels it with the named one -- a lying report.
- The pre-flight refuses instead, before any weights load, even under `on_error="skip"`."""
+ The pre-flight refuses instead, before any weights load, even under `on_error="skip"`, and
+ names the offending row and checkpoint so a large dataset is not a grep hunt."""
dataset = Dataset([Example("s1", Q, {"intent": "a"}, model="english"),
Example("s2", Q, {"intent": "a"})])
with pytest.raises(EvalError) as exc:
@@ -1316,6 +1360,27 @@ def test_an_explicit_model_on_a_runner_without_the_parameter_is_refused():
batch_size=4, on_error="skip")
assert "model" in str(exc.value)
assert "does not accept" in str(exc.value)
+ assert "example 0" in str(exc.value)
+ assert "'english'" in str(exc.value)
+
+
+def test_an_unreadable_signature_answers_both_guards_the_same_way():
+ """A C-level entry point cannot be read, and both guards then keep the call the harness makes
+ (#1112 review): with `_takes_model` on `_takes_min_confidence`'s ``False`` fallback, `model`
+ was sent on the batch path and dropped on the single one, and a named checkpoint was refused
+ even though the batch entry point would have carried it."""
+ assert evals._batch_form(NotIntrospectableRunner()) == evals._BATCH_STATES
+ assert evals._takes_model(NotIntrospectableRunner()) is True
+
+
+def test_a_named_model_runs_on_a_runner_with_unreadable_entry_points():
+ """The refusal is about a guard saying no, not about introspection failing: on the same
+ C-level runner the run goes through, batch path and all."""
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}, model="english")])
+ report = evaluate(NotIntrospectableRunner(), dataset, evaluators=[ChoiceAccuracy()],
+ batch_size=4, on_error="skip")
+ assert report.config["timing"]["batch_form"] == "states"
+ assert report.config["timing"]["chunks"] == 1
def test_an_explicit_model_still_reaches_a_runner_that_takes_it():