Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): refuse a batch reply with fewer results than rows
PerryLink0f9bae052a1a
diff --git a/laya/evals.py b/laya/evals.pyindex acb4dab7..2537edfa 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -1301,6 +1301,15 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluafor offset in range(len(chunk)))index += len(chunk)continue+ # One result per row is the batch contract; `zip` below would silently pair the rest of+ # the chunk with nothing, so a runner that drops a row would score a subset while+ # `rows_grouped` still counted every row -- a report that under-scores itself, and a+ # baseline gate that can pass on the subset.+ results = list(results)+ if len(results) != len(chunk):+ raise EvalError(+ "runner.predict_batch returned %d results for a chunk of %d rows; every row "+ "needs its own result" % (len(results), len(chunk)))elapsed = (time.perf_counter() - started) * 1000.0# A chunk is one call, so each of its rows waited all of `elapsed`; the split figure is a# throughput share, not a latency, and putting the two in one list made a batched rundiff --git a/tests/test_evals.py b/tests/test_evals.pyindex f99bf615..a2449394 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -3935,3 +3935,18 @@ def test_chunking_folds_criteria_key_order_on_the_request_form():evaluators=[ChoiceAccuracy()], batch_size=2)assert listed_report.config["timing"]["rows_grouped"] == 0, \"a reordered list criteria stays positional even for the request form"++++def test_a_short_batch_result_is_an_error_not_a_silent_drop():+ """A runner that returns fewer results than rows must not under-score the report."""+ class ShortBatch:+ def predict(self, state, questions, model=None):+ return {"answers": {"intent": choice_answer("a")}}++ def predict_batch(self, states, questions, model=None, batch_size=None):+ return [self.predict(state, questions) for state in states][:-1]++ dataset = Dataset([Example("s%d" % i, Q, {"intent": "a"}) for i in range(4)])+ with pytest.raises(EvalError, match="4 rows"):+ evaluate(ShortBatch(), dataset, batch_size=4)