Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): batch the runner shape evaluate() already documents
Aashish79f8c817affe
diff --git a/laya/evals.py b/laya/evals.pyindex cac581d6..111f59f4 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -2,8 +2,10 @@Pure Python plus numpy, and free of torch at import time, so the metric math and the datasetparsing can be unit tested with no weights. `evaluate` only needs a runner with a-``predict(state, questions, model=...)`` method (and optionally a ``predict_batch``), so a-fixture runner stands in for a checkpoint in tests.+``predict(state, questions, model=...)`` method, plus -- for ``batch_size`` above 1 -- a+``predict_batch`` in either of the two shapes the repo ships: the positional one, or the+per-request-dict one `Router.predict_batch` documents. A fixture runner stands in for a+checkpoint in tests.The report is deterministic for a fixed runner: the same dataset produces the same numbers, and``EvalReport.compare`` turns a baseline into a pass/fail with the per-metric deltas, which is what@@ -11,6 +13,7 @@ the CI gate consumes."""from __future__ import annotations+import inspectimport jsonimport statisticsimport time@@ -277,11 +280,53 @@ def _aggregate(cases: Sequence[Dict[str, Any]], evaluators: Sequence[Evaluator])return out+# The two batch call shapes the repo ships. `RouterRunner.predict_batch(states, questions,+# model=..., batch_size=...)` is the positional one; `Router.predict_batch(requests, ...)` takes+# one dict per request, the form `route_batch` and the `laya-evals` CLI document.+_BATCH_STATES = "states"+_BATCH_REQUESTS = "requests"+++def _batch_form(runner: Any) -> Optional[str]:+ """Return which batch call `runner` accepts, or ``None`` to score it one ``predict`` at a time.++ Deliberately not ``hasattr(runner, "predict_batch")``: having a batch entry point and being+ callable the way this harness calls it are different claims, and the attribute test returned+ true for ``Router`` -- the one real runner whose ``predict(state, questions, model=...)``+ matches the contract `evaluate` states. Every chunk of more than one example then raised+ ``TypeError: ... unexpected keyword argument 'model'``, which under ``on_error="skip"`` became+ a report with zero cases and an empty ``overall``.+ """+ fn = getattr(runner, "predict_batch", None)+ if fn is None:+ return None+ try:+ params = list(inspect.signature(fn).parameters.values())+ except (TypeError, ValueError):+ # Not introspectable (a C-level or hand-rolled ``__call__``): keep the call this harness+ # made before the shape was checked, rather than silently dropping to single predicts.+ return _BATCH_STATES+ if any(p.name == "model" or p.kind is p.VAR_KEYWORD for p in params):+ return _BATCH_STATES+ positional = [p for p in params if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)]+ # A bound method, so `positional[0]` is the batch argument itself.+ if positional and positional[0].name == "requests":+ return _BATCH_REQUESTS+ return None++def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluator]] = None,batch_size: Optional[int] = None, on_error: str = "fail",config: Optional[Dict[str, Any]] = None) -> EvalReport:"""Run `runner` over `dataset`, aggregating per-answer metrics overall and per slice.+ `runner` needs a ``predict(state, questions, model=...)`` method, and for `batch_size` above 1+ a ``predict_batch`` in either shape the repo ships: the positional+ ``predict_batch(states, questions, model=..., batch_size=...)``, or the per-request-dict form+ `Router.predict_batch` takes -- ``predict_batch([{"state": ..., "questions": ...,+ "model": ...}, ...], batch_size=...)``. A runner that offers neither is scored one ``predict``+ at a time, which is slower but not wrong.+`on_error` is ``"fail"`` (re-raise a runner error) or ``"skip"`` (record it and continue),the latter for evaluating a flaky fleet without aborting the whole run."""@@ -292,14 +337,15 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalualatencies: List[float] = []errors: List[Dict[str, Any]] = []examples = dataset.examples- can_batch = batch_size is not None and batch_size > 1 and hasattr(runner, "predict_batch")+ # Only worth grouping if the runner can be handed the group in one call at all.+ batch_form = _batch_form(runner) if batch_size is not None and batch_size > 1 else Noneindex = 0while index < len(examples):# Batches only when the runner can share a forward pass: consecutive examples with the# same checkpoint and identical questions. Otherwise every example is one predict.chunk = [examples[index]]- if can_batch:+ if batch_form:signature = (examples[index].model,json.dumps(examples[index].questions, sort_keys=False, default=str))while (index + len(chunk) < len(examples) and len(chunk) < batch_size@@ -309,8 +355,15 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluastarted = time.perf_counter()try:if len(chunk) > 1:- results = runner.predict_batch([e.state for e in chunk], chunk[0].questions,- model=chunk[0].model, batch_size=batch_size)+ if batch_form == _BATCH_REQUESTS:+ # The chunk already shares one checkpoint and one question schema, so the+ # per-request dicts carry exactly what the positional call would pass.+ results = runner.predict_batch(+ [{"state": e.state, "questions": e.questions, "model": e.model}+ for e in chunk], batch_size=batch_size)+ else:+ results = runner.predict_batch([e.state for e in chunk], chunk[0].questions,+ model=chunk[0].model, batch_size=batch_size)else:results = [runner.predict(chunk[0].state, chunk[0].questions, model=chunk[0].model)]except Exception as exc: # noqa: BLE001 -- honoured by on_errordiff --git a/tests/test_evals.py b/tests/test_evals.pyindex 0e9ff2d7..c6ad813a 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -160,6 +160,92 @@ def test_evaluate_batches_same_questions():assert report.overall["choice_accuracy"] == 1.0+class RequestsRunner(StubRunner):+ """The `Router` shape: a list of per-request dicts, and no ``model=`` on the call."""++ def __init__(self, by_state):+ super().__init__(by_state)+ self.batches = []+ self.batch_sizes = []++ def predict_batch(self, requests, batch_size=None):+ self.batches.append(list(requests))+ self.batch_sizes.append(batch_size)+ return [{"model": r.get("model") or "m", "answers": self.by_state[r["state"]]}+ for r in requests]+++def _labels(report):+ """The decided labels with their correctness: parity at decision level, not as floats."""+ return [(c["answer"]["choice"], c["correct"]) for c in report.cases]+++def test_evaluate_drives_a_requests_shaped_batch():+ dataset = Dataset([Example("s1", Q, {"intent": "a"}, model="english"),+ Example("s2", Q, {"intent": "a"}, model="english")])+ runner = RequestsRunner({"s1": {"intent": choice_answer("a")},+ "s2": {"intent": choice_answer("a")}})+ report = evaluate(runner, dataset, evaluators=[ChoiceAccuracy()], batch_size=8)+ assert len(runner.batches) == 1, "a request-dict batch is a forward pass the harness can run"+ expected = [{"state": "s1", "questions": Q, "model": "english"},+ {"state": "s2", "questions": Q, "model": "english"}]+ assert runner.batches[0] == expected, "each request carries its own state, questions, checkpoint"+ assert runner.batch_sizes == [8], "the requested batch size reaches the runner"+ assert report.overall["choice_accuracy"] == 1.0+++def test_requests_shaped_batch_agrees_with_single_predicts():+ """`batch_size` changes how many forward passes a run makes, never what it scores."""+ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"}),+ Example("s3", Q, {"intent": "b"})])+ answers = {"s1": {"intent": choice_answer("a")}, "s2": {"intent": choice_answer("b")},+ "s3": {"intent": choice_answer("b")}}+ batched = evaluate(RequestsRunner(answers), dataset, evaluators=[ChoiceAccuracy()], batch_size=8)+ single = evaluate(RequestsRunner(answers), dataset, evaluators=[ChoiceAccuracy()])+ assert _labels(batched) == _labels(single) == [("a", True), ("b", False), ("b", True)]+ assert len(batched.cases) == len(single.cases) == 3+++def test_evaluate_scores_a_batch_shape_it_cannot_call():+ """A `predict_batch` in neither documented shape must not fail the run row by row."""+ class UntypedBatch(StubRunner):+ def predict_batch(self, states, questions):+ raise AssertionError("the harness may not call this: no model=, no requests")++ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"})])+ runner = UntypedBatch({"s1": {"intent": choice_answer("a")}, "s2": {"intent": choice_answer("a")}})+ report = evaluate(runner, dataset, evaluators=[ChoiceAccuracy()], batch_size=8, on_error="skip")+ assert report.overall["choice_accuracy"] == 1.0, "scored one predict at a time"+ assert not report.config.get("errored"), "a batch entry point in an unknown shape is not an error"+++def test_evaluate_scores_a_runner_with_no_batch_entry_point():+ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"})])+ runner = StubRunner({"s1": {"intent": choice_answer("a")}, "s2": {"intent": choice_answer("a")}})+ report = evaluate(runner, dataset, evaluators=[ChoiceAccuracy()], batch_size=8)+ assert report.overall["choice_accuracy"] == 1.0+++def test_evaluate_batches_a_pass_through_wrapper_positionally():+ """A wrapper that forwards `*args, **kwargs` takes the positional call, whatever it names."""+ class PassThrough(StubRunner):+ def __init__(self, by_state):+ super().__init__(by_state)+ self.calls = []++ def predict_batch(self, *args, **kwargs):+ self.calls.append((args, kwargs))+ return [{"model": "m", "answers": self.by_state[s]} for s in args[0]]++ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"})])+ runner = PassThrough({"s1": {"intent": choice_answer("a")}, "s2": {"intent": choice_answer("a")}})+ report = evaluate(runner, dataset, evaluators=[ChoiceAccuracy()], batch_size=8)+ assert len(runner.calls) == 1+ assert runner.calls[0][0] == (["s1", "s2"], Q)+ assert runner.calls[0][1] == {"model": None, "batch_size": 8}+ assert report.overall["choice_accuracy"] == 1.0++def test_evaluate_skips_errors_when_asked():class Boom(StubRunner):def predict(self, state, questions, model=None):