Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): batch the runner shape evaluate() already documents

Aashish79f8c817affe
diff --git a/laya/evals.py b/laya/evals.py
index cac581d6..111f59f4 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -2,8 +2,10 @@
Pure Python plus numpy, and free of torch at import time, so the metric math and the dataset
parsing can be unit tested with no weights. `evaluate` only needs a runner with a
-``predict(state, questions, model=...)`` method (and optionally a ``predict_batch``), so a
-fixture runner stands in for a checkpoint in tests.
+``predict(state, questions, model=...)`` method, plus -- for ``batch_size`` above 1 -- a
+``predict_batch`` in either of the two shapes the repo ships: the positional one, or the
+per-request-dict one `Router.predict_batch` documents. A fixture runner stands in for a
+checkpoint in tests.
The report is deterministic for a fixed runner: the same dataset produces the same numbers, and
``EvalReport.compare`` turns a baseline into a pass/fail with the per-metric deltas, which is what
@@ -11,6 +13,7 @@ the CI gate consumes.
"""
from __future__ import annotations
+import inspect
import json
import statistics
import time
@@ -277,11 +280,53 @@ def _aggregate(cases: Sequence[Dict[str, Any]], evaluators: Sequence[Evaluator])
return out
+# The two batch call shapes the repo ships. `RouterRunner.predict_batch(states, questions,
+# model=..., batch_size=...)` is the positional one; `Router.predict_batch(requests, ...)` takes
+# one dict per request, the form `route_batch` and the `laya-evals` CLI document.
+_BATCH_STATES = "states"
+_BATCH_REQUESTS = "requests"
+
+
+def _batch_form(runner: Any) -> Optional[str]:
+ """Return which batch call `runner` accepts, or ``None`` to score it one ``predict`` at a time.
+
+ Deliberately not ``hasattr(runner, "predict_batch")``: having a batch entry point and being
+ callable the way this harness calls it are different claims, and the attribute test returned
+ true for ``Router`` -- the one real runner whose ``predict(state, questions, model=...)``
+ matches the contract `evaluate` states. Every chunk of more than one example then raised
+ ``TypeError: ... unexpected keyword argument 'model'``, which under ``on_error="skip"`` became
+ a report with zero cases and an empty ``overall``.
+ """
+ fn = getattr(runner, "predict_batch", None)
+ if fn is None:
+ return None
+ try:
+ params = list(inspect.signature(fn).parameters.values())
+ except (TypeError, ValueError):
+ # Not introspectable (a C-level or hand-rolled ``__call__``): keep the call this harness
+ # made before the shape was checked, rather than silently dropping to single predicts.
+ return _BATCH_STATES
+ if any(p.name == "model" or p.kind is p.VAR_KEYWORD for p in params):
+ return _BATCH_STATES
+ positional = [p for p in params if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)]
+ # A bound method, so `positional[0]` is the batch argument itself.
+ if positional and positional[0].name == "requests":
+ return _BATCH_REQUESTS
+ return None
+
+
def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluator]] = None,
batch_size: Optional[int] = None, on_error: str = "fail",
config: Optional[Dict[str, Any]] = None) -> EvalReport:
"""Run `runner` over `dataset`, aggregating per-answer metrics overall and per slice.
+ `runner` needs a ``predict(state, questions, model=...)`` method, and for `batch_size` above 1
+ a ``predict_batch`` in either shape the repo ships: the positional
+ ``predict_batch(states, questions, model=..., batch_size=...)``, or the per-request-dict form
+ `Router.predict_batch` takes -- ``predict_batch([{"state": ..., "questions": ...,
+ "model": ...}, ...], batch_size=...)``. A runner that offers neither is scored one ``predict``
+ at a time, which is slower but not wrong.
+
`on_error` is ``"fail"`` (re-raise a runner error) or ``"skip"`` (record it and continue),
the latter for evaluating a flaky fleet without aborting the whole run.
"""
@@ -292,14 +337,15 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
latencies: List[float] = []
errors: List[Dict[str, Any]] = []
examples = dataset.examples
- can_batch = batch_size is not None and batch_size > 1 and hasattr(runner, "predict_batch")
+ # Only worth grouping if the runner can be handed the group in one call at all.
+ batch_form = _batch_form(runner) if batch_size is not None and batch_size > 1 else None
index = 0
while index < len(examples):
# Batches only when the runner can share a forward pass: consecutive examples with the
# same checkpoint and identical questions. Otherwise every example is one predict.
chunk = [examples[index]]
- if can_batch:
+ if batch_form:
signature = (examples[index].model,
json.dumps(examples[index].questions, sort_keys=False, default=str))
while (index + len(chunk) < len(examples) and len(chunk) < batch_size
@@ -309,8 +355,15 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
started = time.perf_counter()
try:
if len(chunk) > 1:
- results = runner.predict_batch([e.state for e in chunk], chunk[0].questions,
- model=chunk[0].model, batch_size=batch_size)
+ if batch_form == _BATCH_REQUESTS:
+ # The chunk already shares one checkpoint and one question schema, so the
+ # per-request dicts carry exactly what the positional call would pass.
+ results = runner.predict_batch(
+ [{"state": e.state, "questions": e.questions, "model": e.model}
+ for e in chunk], batch_size=batch_size)
+ else:
+ results = runner.predict_batch([e.state for e in chunk], chunk[0].questions,
+ model=chunk[0].model, batch_size=batch_size)
else:
results = [runner.predict(chunk[0].state, chunk[0].questions, model=chunk[0].model)]
except Exception as exc: # noqa: BLE001 -- honoured by on_error
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 0e9ff2d7..c6ad813a 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -160,6 +160,92 @@ def test_evaluate_batches_same_questions():
assert report.overall["choice_accuracy"] == 1.0
+class RequestsRunner(StubRunner):
+ """The `Router` shape: a list of per-request dicts, and no ``model=`` on the call."""
+
+ def __init__(self, by_state):
+ super().__init__(by_state)
+ self.batches = []
+ self.batch_sizes = []
+
+ def predict_batch(self, requests, batch_size=None):
+ self.batches.append(list(requests))
+ self.batch_sizes.append(batch_size)
+ return [{"model": r.get("model") or "m", "answers": self.by_state[r["state"]]}
+ for r in requests]
+
+
+def _labels(report):
+ """The decided labels with their correctness: parity at decision level, not as floats."""
+ return [(c["answer"]["choice"], c["correct"]) for c in report.cases]
+
+
+def test_evaluate_drives_a_requests_shaped_batch():
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}, model="english"),
+ Example("s2", Q, {"intent": "a"}, model="english")])
+ runner = RequestsRunner({"s1": {"intent": choice_answer("a")},
+ "s2": {"intent": choice_answer("a")}})
+ report = evaluate(runner, dataset, evaluators=[ChoiceAccuracy()], batch_size=8)
+ assert len(runner.batches) == 1, "a request-dict batch is a forward pass the harness can run"
+ expected = [{"state": "s1", "questions": Q, "model": "english"},
+ {"state": "s2", "questions": Q, "model": "english"}]
+ assert runner.batches[0] == expected, "each request carries its own state, questions, checkpoint"
+ assert runner.batch_sizes == [8], "the requested batch size reaches the runner"
+ assert report.overall["choice_accuracy"] == 1.0
+
+
+def test_requests_shaped_batch_agrees_with_single_predicts():
+ """`batch_size` changes how many forward passes a run makes, never what it scores."""
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"}),
+ Example("s3", Q, {"intent": "b"})])
+ answers = {"s1": {"intent": choice_answer("a")}, "s2": {"intent": choice_answer("b")},
+ "s3": {"intent": choice_answer("b")}}
+ batched = evaluate(RequestsRunner(answers), dataset, evaluators=[ChoiceAccuracy()], batch_size=8)
+ single = evaluate(RequestsRunner(answers), dataset, evaluators=[ChoiceAccuracy()])
+ assert _labels(batched) == _labels(single) == [("a", True), ("b", False), ("b", True)]
+ assert len(batched.cases) == len(single.cases) == 3
+
+
+def test_evaluate_scores_a_batch_shape_it_cannot_call():
+ """A `predict_batch` in neither documented shape must not fail the run row by row."""
+ class UntypedBatch(StubRunner):
+ def predict_batch(self, states, questions):
+ raise AssertionError("the harness may not call this: no model=, no requests")
+
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"})])
+ runner = UntypedBatch({"s1": {"intent": choice_answer("a")}, "s2": {"intent": choice_answer("a")}})
+ report = evaluate(runner, dataset, evaluators=[ChoiceAccuracy()], batch_size=8, on_error="skip")
+ assert report.overall["choice_accuracy"] == 1.0, "scored one predict at a time"
+ assert not report.config.get("errored"), "a batch entry point in an unknown shape is not an error"
+
+
+def test_evaluate_scores_a_runner_with_no_batch_entry_point():
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"})])
+ runner = StubRunner({"s1": {"intent": choice_answer("a")}, "s2": {"intent": choice_answer("a")}})
+ report = evaluate(runner, dataset, evaluators=[ChoiceAccuracy()], batch_size=8)
+ assert report.overall["choice_accuracy"] == 1.0
+
+
+def test_evaluate_batches_a_pass_through_wrapper_positionally():
+ """A wrapper that forwards `*args, **kwargs` takes the positional call, whatever it names."""
+ class PassThrough(StubRunner):
+ def __init__(self, by_state):
+ super().__init__(by_state)
+ self.calls = []
+
+ def predict_batch(self, *args, **kwargs):
+ self.calls.append((args, kwargs))
+ return [{"model": "m", "answers": self.by_state[s]} for s in args[0]]
+
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"})])
+ runner = PassThrough({"s1": {"intent": choice_answer("a")}, "s2": {"intent": choice_answer("a")}})
+ report = evaluate(runner, dataset, evaluators=[ChoiceAccuracy()], batch_size=8)
+ assert len(runner.calls) == 1
+ assert runner.calls[0][0] == (["s1", "s2"], Q)
+ assert runner.calls[0][1] == {"model": None, "batch_size": 8}
+ assert report.overall["choice_accuracy"] == 1.0
+
+
def test_evaluate_skips_errors_when_asked():
class Boom(StubRunner):
def predict(self, state, questions, model=None):