Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): adopt the #1112 review — routing test, unified fallback, refusal detail
akushonkamenc05312a5fe05
diff --git a/laya/evals.py b/laya/evals.pyindex 94c163d0..d4431a4c 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -901,7 +901,7 @@ def _takes_min_confidence(runner: Any, fn_name: str = "predict_batch") -> bool:def _takes_model(runner: Any, fn_name: str = "predict") -> bool:"""Whether `runner`'s entry point accepts a per-example checkpoint on the call.- The same signature check `_takes_min_confidence` makes, applied to the `model` kwarg the+ The same signature check `_batch_form` makes, applied to the `model` kwarg thesingle-row path forwards from `Example.model` (#1107). The batch half of this exact mistakewas `_batch_form`'s founding bug -- every chunk raised ``TypeError: ... unexpected keywordargument 'model'``, which `on_error="skip"` turned into a zero-case report -- and@@ -910,6 +910,16 @@ def _takes_model(runner: Any, fn_name: str = "predict") -> bool:an unsent `model` is no lie: a runner without the parameter answers from its own checkpoint.So the harness drops the kwarg when no example names one, and the pre-flight refusal in`evaluate` fires only when an example does.++ Not introspectable (a C-level or hand-rolled ``__call__``): keep the call this harness makes,+ the fallback `_batch_form` makes for the same kwarg on ``predict_batch`` -- deliberately not+ `_takes_min_confidence`'s ``False``, which had the two guards disagreeing: on a runner whose+ entry points are both C-level, `model` was sent on the batch path and dropped on the single+ one, and a dataset naming an explicit checkpoint was refused even though the batch entry point+ would have carried it (#1112 review). With no example naming a model the value is ``None``+ either way, so the only run the ``True`` fallback changes is the one where sending it is the+ point; a runner whose attribute is missing is the `fn is None` case, and a C-level runner that+ refuses the kwarg fails loudly on the call -- the risk `_batch_form` already accepts."""fn = getattr(runner, fn_name, None)if fn is None:@@ -917,7 +927,9 @@ def _takes_model(runner: Any, fn_name: str = "predict") -> bool:try:params = inspect.signature(fn).parameters.values()except (TypeError, ValueError):- return False+ # Same fallback as `_batch_form` (its comment records why): keep the call this harness+ # made before the shape was checked, rather than half-dropping the kwarg.+ return Truereturn any(p.name == "model" or p.kind is p.VAR_KEYWORD for p in params)@@ -1004,16 +1016,20 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua# on every row, and `on_error="skip"` published a zero-case report. The batch half was already# gated by `_batch_form`; this is the single-row guard, computed once and reused at the call.takes_model = _takes_model(runner)- if not takes_model and any(e.model is not None for e in dataset.examples):+ named = next(((i, e) for i, e in enumerate(dataset.examples) if e.model is not None), None)+ if not takes_model and named is not None:# Dropping the kwarg here would score each row on whatever checkpoint the runner answers# with while the report's `model` slice labels it with the named one -- the same class of# lie the `min_confidence` guard exists to prevent. The kwarg is dropped only when no# example names a checkpoint, in which case scoring on the runner's own is what was asked.+ # The refusal names the first offender: on a large dataset, "the dataset names a `model`"+ # alone leaves the user grepping for the row (#1112 review).+ first, offender = namedraise EvalError(- "evaluate() refused: the dataset names an explicit per-example `model`, but this "- "runner's predict does not accept one. Either use a Router/ONNXAgent whose predict "- "forwards the checkpoint, or drop the per-example `model` -- the report would "- "otherwise name a checkpoint it never sent (#1107).")+ "evaluate() refused: example %d names model %r, but this runner's predict does not "+ "accept a `model` argument. Either use a Router/ONNXAgent whose predict forwards the "+ "checkpoint, or drop the per-example `model` -- the report would otherwise name a "+ "checkpoint it never sent (#1107)." % (first, offender.model))evaluators = list(evaluators) if evaluators is not None else default_evaluators()cases: List[Dict[str, Any]] = []waits: List[float] = [] # what each request actually waited: its chunk's whole call@@ -1076,8 +1092,10 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluaelse:# A chunk of one still sees the gate: the pre-flight refusal proved `predict`# accepts it whenever `mc` is set, so nothing here has to re-check the signature.- # The `model` kwarg is the same honesty (#1107): sent only when `predict` accepts- # it, and the pre-flight already refused any run where an example needed it sent.+ # The `model` kwarg is the same honesty (#1107): sent only when `predict` takes+ # it -- kept as-is when the signature cannot be read, `_batch_form`'s fallback on+ # the same kwarg -- and the pre-flight already refused any run where an example+ # needed it sent and the guard said no.single_kwargs = {"min_confidence": mc} if mc is not None else {}if takes_model:results = [runner.predict(chunk[0].state, chunk[0].questions,diff --git a/tests/test_evals.py b/tests/test_evals.pyindex e96ad739..a79729df 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -1271,6 +1271,23 @@ class AgentShapedRunner:return [self.predict(s, questions) for s in states]+class NotIntrospectableRunner:+ """Entry points whose signature cannot be read -- a C-level or hand-rolled ``__call__``+ (#1112 review): `_batch_form` keeps the batch call there, so `_takes_model` must too."""++ class _Callable:+ __signature__ = None++ def __call__(self, *args, **kwargs):+ return None++ def __get__(self, obj, cls):+ return self++ predict = _Callable()+ predict_batch = _Callable()++def test_an_agent_shaped_runner_scores_instead_of_a_zero_case_report():"""Every row of the issue's report: a runner whose `predict` predates `model` used to failrow by row and -- under the default skip -- publish 0 cases with an empty `overall`."""@@ -1305,10 +1322,37 @@ def test_a_runner_that_takes_model_still_receives_it():assert len(report.cases) == 3+class BatchOnlyRouting(StubRunner):+ """Routes in the batch entry point but not in the single one -- the shape `_batch_form`+ approves (#1112 review)."""++ def predict(self, state, questions):+ return {"model": "single", "answers": self.by_state[state]}++ def predict_batch(self, states, questions, model=None, batch_size=None):+ return [{"model": model or "batched", "answers": self.by_state[s]} for s in states]+++def test_a_runner_that_routes_only_in_batch_survives_a_chunk_of_one():+ """Chunking splits on (model, questions), so a differing row lands on the single `predict`.+ `takes_model` reads `predict` alone, so this runner keeps batching and survives: the shape+ that made the bug read as intermittent rather than as a contract mismatch."""+ mixed = Dataset([Example("s1", Q, {"intent": "a"}),+ Example("s3", QNOUL, {"flag": True}),+ Example("s2", Q, {"intent": "a"})])+ answers = {"s1": {"intent": choice_answer("a")}, "s2": {"intent": choice_answer("a")},+ "s3": {"flag": noul_answer(0.9)}}+ report = evaluate(BatchOnlyRouting(answers), mixed,+ evaluators=[ChoiceAccuracy(), NoulAccuracy()], batch_size=8)+ assert not report.config.get("errored")+ assert report.overall["choice_accuracy"] == 1.0++def test_an_explicit_model_on_a_runner_without_the_parameter_is_refused():"""Dropping the kwarg when an example names a checkpoint would score the row on whatever therunner answers with while the `model` slice labels it with the named one -- a lying report.- The pre-flight refuses instead, before any weights load, even under `on_error="skip"`."""+ The pre-flight refuses instead, before any weights load, even under `on_error="skip"`, and+ names the offending row and checkpoint so a large dataset is not a grep hunt."""dataset = Dataset([Example("s1", Q, {"intent": "a"}, model="english"),Example("s2", Q, {"intent": "a"})])with pytest.raises(EvalError) as exc:@@ -1316,6 +1360,27 @@ def test_an_explicit_model_on_a_runner_without_the_parameter_is_refused():batch_size=4, on_error="skip")assert "model" in str(exc.value)assert "does not accept" in str(exc.value)+ assert "example 0" in str(exc.value)+ assert "'english'" in str(exc.value)+++def test_an_unreadable_signature_answers_both_guards_the_same_way():+ """A C-level entry point cannot be read, and both guards then keep the call the harness makes+ (#1112 review): with `_takes_model` on `_takes_min_confidence`'s ``False`` fallback, `model`+ was sent on the batch path and dropped on the single one, and a named checkpoint was refused+ even though the batch entry point would have carried it."""+ assert evals._batch_form(NotIntrospectableRunner()) == evals._BATCH_STATES+ assert evals._takes_model(NotIntrospectableRunner()) is True+++def test_a_named_model_runs_on_a_runner_with_unreadable_entry_points():+ """The refusal is about a guard saying no, not about introspection failing: on the same+ C-level runner the run goes through, batch path and all."""+ dataset = Dataset([Example("s1", Q, {"intent": "a"}, model="english")])+ report = evaluate(NotIntrospectableRunner(), dataset, evaluators=[ChoiceAccuracy()],+ batch_size=4, on_error="skip")+ assert report.config["timing"]["batch_form"] == "states"+ assert report.config["timing"]["chunks"] == 1def test_an_explicit_model_still_reaches_a_runner_that_takes_it():