Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): refuse a batch reply with fewer results than rows

PerryLink0f9bae052a1a
diff --git a/laya/evals.py b/laya/evals.py
index acb4dab7..2537edfa 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -1301,6 +1301,15 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
for offset in range(len(chunk)))
index += len(chunk)
continue
+ # One result per row is the batch contract; `zip` below would silently pair the rest of
+ # the chunk with nothing, so a runner that drops a row would score a subset while
+ # `rows_grouped` still counted every row -- a report that under-scores itself, and a
+ # baseline gate that can pass on the subset.
+ results = list(results)
+ if len(results) != len(chunk):
+ raise EvalError(
+ "runner.predict_batch returned %d results for a chunk of %d rows; every row "
+ "needs its own result" % (len(results), len(chunk)))
elapsed = (time.perf_counter() - started) * 1000.0
# A chunk is one call, so each of its rows waited all of `elapsed`; the split figure is a
# throughput share, not a latency, and putting the two in one list made a batched run
diff --git a/tests/test_evals.py b/tests/test_evals.py
index f99bf615..a2449394 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -3935,3 +3935,18 @@ def test_chunking_folds_criteria_key_order_on_the_request_form():
evaluators=[ChoiceAccuracy()], batch_size=2)
assert listed_report.config["timing"]["rows_grouped"] == 0, \
"a reordered list criteria stays positional even for the request form"
+
+
+
+def test_a_short_batch_result_is_an_error_not_a_silent_drop():
+ """A runner that returns fewer results than rows must not under-score the report."""
+ class ShortBatch:
+ def predict(self, state, questions, model=None):
+ return {"answers": {"intent": choice_answer("a")}}
+
+ def predict_batch(self, states, questions, model=None, batch_size=None):
+ return [self.predict(state, questions) for state in states][:-1]
+
+ dataset = Dataset([Example("s%d" % i, Q, {"intent": "a"}) for i in range(4)])
+ with pytest.raises(EvalError, match="4 rows"):
+ evaluate(ShortBatch(), dataset, batch_size=4)