Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): drop model= for runners whose predict predates it (#1107)
akushonkamen747e62eea175
diff --git a/laya/evals.py b/laya/evals.pyindex b066859c..94c163d0 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -898,6 +898,29 @@ def _takes_min_confidence(runner: Any, fn_name: str = "predict_batch") -> bool:return any(p.name == "min_confidence" or p.kind is p.VAR_KEYWORD for p in params)+def _takes_model(runner: Any, fn_name: str = "predict") -> bool:+ """Whether `runner`'s entry point accepts a per-example checkpoint on the call.++ The same signature check `_takes_min_confidence` makes, applied to the `model` kwarg the+ single-row path forwards from `Example.model` (#1107). The batch half of this exact mistake+ was `_batch_form`'s founding bug -- every chunk raised ``TypeError: ... unexpected keyword+ argument 'model'``, which `on_error="skip"` turned into a zero-case report -- and+ `laya.load()`'s default runner still hit it one row at a time, because `Agent.predict` is+ `system_one`, which takes neither `model` nor any `**kwargs` forwarder. Unlike the threshold,+ an unsent `model` is no lie: a runner without the parameter answers from its own checkpoint.+ So the harness drops the kwarg when no example names one, and the pre-flight refusal in+ `evaluate` fires only when an example does.+ """+ fn = getattr(runner, fn_name, None)+ if fn is None:+ return False+ try:+ params = inspect.signature(fn).parameters.values()+ except (TypeError, ValueError):+ return False+ return any(p.name == "model" or p.kind is p.VAR_KEYWORD for p in params)++def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluator]] = None,batch_size: Optional[int] = None, on_error: str = "fail",config: Optional[Dict[str, Any]] = None, sort_by_length: bool = False,@@ -909,7 +932,11 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua``predict_batch(states, questions, model=..., batch_size=...)``, or the per-request-dict form`Router.predict_batch` takes -- ``predict_batch([{"state": ..., "questions": ...,"model": ...}, ...], batch_size=...)``. A runner that offers neither is scored one ``predict``- at a time, which is slower but not wrong.+ at a time, which is slower but not wrong. A runner whose ``predict`` takes no ``model`` --+ ``laya.load()``'s default `Agent` is one, its ``predict`` being an alias of `system_one`+ (#1107) -- is scored on its own checkpoint: the kwarg is dropped unless an example names an+ explicit ``model``, and a dataset that names one on such a runner is refused in pre-flight+ rather than published as a silent zero-case report.`sort_by_length` is forwarded to a chunked runner so similarly sized examples share a forwardpass and pad to a shorter maximum. It reaches the checkpoint's own batching only when the@@ -972,6 +999,21 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua"abstention threshold. Either use a Router/ONNXAgent that gates on ""answer_confidence (#361), or drop the threshold -- the report would ""otherwise name a `min_confidence` it never applied." % (min_confidence, target))+ # Whether `predict` can be handed `Example.model` at all (#1107). `Agent.predict` is+ # `system_one` -- no `model`, no `**kwargs` -- so the single-row path used to raise `TypeError`+ # on every row, and `on_error="skip"` published a zero-case report. The batch half was already+ # gated by `_batch_form`; this is the single-row guard, computed once and reused at the call.+ takes_model = _takes_model(runner)+ if not takes_model and any(e.model is not None for e in dataset.examples):+ # Dropping the kwarg here would score each row on whatever checkpoint the runner answers+ # with while the report's `model` slice labels it with the named one -- the same class of+ # lie the `min_confidence` guard exists to prevent. The kwarg is dropped only when no+ # example names a checkpoint, in which case scoring on the runner's own is what was asked.+ raise EvalError(+ "evaluate() refused: the dataset names an explicit per-example `model`, but this "+ "runner's predict does not accept one. Either use a Router/ONNXAgent whose predict "+ "forwards the checkpoint, or drop the per-example `model` -- the report would "+ "otherwise name a checkpoint it never sent (#1107).")evaluators = list(evaluators) if evaluators is not None else default_evaluators()cases: List[Dict[str, Any]] = []waits: List[float] = [] # what each request actually waited: its chunk's whole call@@ -1034,9 +1076,15 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluaelse:# A chunk of one still sees the gate: the pre-flight refusal proved `predict`# accepts it whenever `mc` is set, so nothing here has to re-check the signature.+ # The `model` kwarg is the same honesty (#1107): sent only when `predict` accepts+ # it, and the pre-flight already refused any run where an example needed it sent.single_kwargs = {"min_confidence": mc} if mc is not None else {}- results = [runner.predict(chunk[0].state, chunk[0].questions,- model=chunk[0].model, **single_kwargs)]+ if takes_model:+ results = [runner.predict(chunk[0].state, chunk[0].questions,+ model=chunk[0].model, **single_kwargs)]+ else:+ results = [runner.predict(chunk[0].state, chunk[0].questions,+ **single_kwargs)]except Exception as exc: # noqa: BLE001 -- honoured by on_errorif on_error == "fail":raisediff --git a/tests/test_evals.py b/tests/test_evals.pyindex 12ed6555..e96ad739 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -1249,6 +1249,86 @@ def test_cli_min_confidence_zero_is_accepted(monkeypatch, tmp_path):assert seen["min_confidence"] == 0.0+# ------------------------------------------------- per-example model (#1107 guard)+#+# `Example.model` names an explicit checkpoint, and the single-row path used to forward it+# unconditionally -- `model=chunk[0].model`, even when the value was `None`. `Agent.predict` is+# `system_one`, which takes neither `model` nor `**kwargs`, so every row raised `TypeError` and+# `on_error="skip"` published a zero-case, empty-overall, exit-0 report: the batch half of this+# exact bug was `_batch_form`'s founding case, the single-row path was left outside it.+++class AgentShapedRunner:+ """What `laya.load()` returns: `predict` is `system_one` -- no `model`, no `**kwargs`."""++ def __init__(self, by_state):+ self.by_state = by_state++ def predict(self, state, questions, lang=None, min_confidence=None):+ return {"answers": self.by_state[state]}++ def predict_batch(self, states, questions, batch_size=None, lang=None):+ return [self.predict(s, questions) for s in states]+++def test_an_agent_shaped_runner_scores_instead_of_a_zero_case_report():+ """Every row of the issue's report: a runner whose `predict` predates `model` used to fail+ row by row and -- under the default skip -- publish 0 cases with an empty `overall`."""+ report = evaluate(AgentShapedRunner(ANSWERS3), _three(), evaluators=[ChoiceAccuracy()],+ batch_size=4, on_error="skip")+ assert len(report.cases) == 3, "the rows score; they do not vanish into on_error"+ assert report.overall["choice_accuracy"] == pytest.approx(1.0)+ assert not report.config.get("errored"), "a scoring run is not an erroring one"+++def test_the_model_kwarg_is_dropped_for_a_runner_without_one():+ """With `on_error="fail"` the dropped kwarg is observable: had the harness still sent+ `model=None`, every call here would raise `TypeError` instead of scoring."""+ report = evaluate(AgentShapedRunner(ANSWERS3), _three(), evaluators=[ChoiceAccuracy()],+ batch_size=4)+ assert len(report.cases) == 3+ assert report.overall["choice_accuracy"] == pytest.approx(1.0)+++def test_a_runner_that_takes_model_still_receives_it():+ """The guard drops the kwarg only for a runner without the parameter: a `Router`-shaped+ `predict(state, questions, model=...)` keeps seeing the call the docstring promises."""+ seen = {}++ class Recording(StubRunner):+ def predict(self, state, questions, model=None):+ seen[state] = model+ return StubRunner.predict(self, state, questions, model)++ report = evaluate(Recording(ANSWERS3), _three(), evaluators=[ChoiceAccuracy()], batch_size=2)+ assert set(seen.values()) == {None}, "no example names a checkpoint, so `None` is the value"+ assert len(report.cases) == 3+++def test_an_explicit_model_on_a_runner_without_the_parameter_is_refused():+ """Dropping the kwarg when an example names a checkpoint would score the row on whatever the+ runner answers with while the `model` slice labels it with the named one -- a lying report.+ The pre-flight refuses instead, before any weights load, even under `on_error="skip"`."""+ dataset = Dataset([Example("s1", Q, {"intent": "a"}, model="english"),+ Example("s2", Q, {"intent": "a"})])+ with pytest.raises(EvalError) as exc:+ evaluate(AgentShapedRunner(ANSWERS3), dataset, evaluators=[ChoiceAccuracy()],+ batch_size=4, on_error="skip")+ assert "model" in str(exc.value)+ assert "does not accept" in str(exc.value)+++def test_an_explicit_model_still_reaches_a_runner_that_takes_it():+ """The refusal is scoped to runners without the parameter; a named checkpoint keeps flowing+ through both entry points of a `Router`-shaped runner."""+ dataset = Dataset([Example("s1", Q, {"intent": "a"}, model="english"),+ Example("s2", Q, {"intent": "a"}, model="english")])+ report = evaluate(StubRunner({"s1": {"intent": choice_answer("a")},+ "s2": {"intent": choice_answer("a")}}),+ dataset, evaluators=[ChoiceAccuracy()], batch_size=2)+ assert report.overall["choice_accuracy"] == 1.0++# --------------------------------------------------------------- timing (#585)FORWARD_MS = 100.0SHARED = 0.6 # a batch of n costs SHARED * n * FORWARD_MS, as one call