Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): record the chunk shape a raised call issued

Aashish11710880e832
diff --git a/docs/evals.md b/docs/evals.md
index 4ec1c573..0a5126b5 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -74,7 +74,11 @@ requests were served fast, not whether the run was cheap. With no `--batch-size`
`compare` ignores any `*_ms` metric unless a tolerance names it, so these never fail a baseline on
timing noise. What the harness actually did -- the batch size asked for, the runner shape it
resolved to, how many rows shared a call, and the largest chunk -- is recorded in the report's
-`config.timing`, because the flag alone does not say whether anything was batched.
+`config.timing`, because the flag alone does not say whether anything was batched. Those counters
+record the calls issued, not the calls that returned: with `on_error=skip`, a chunk whose call
+raised still counts in `rows_grouped` and `max_chunk`, next to its entries in `config.errored`. The
+two `*_ms` metrics count only the calls that returned, so a failed call never contributes a latency
+it did not measure.
## Slices
diff --git a/laya/evals.py b/laya/evals.py
index 8807fbc2..06d0bc0e 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -369,6 +369,19 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
and (examples[index + len(chunk)].model,
json.dumps(examples[index + len(chunk)].questions, sort_keys=False, default=str)) == signature):
chunk.append(examples[index + len(chunk)])
+ # Why these sit above the call and `waits`/`shares` below: the shape counters answer "what
+ # did the harness issue", which is settled the moment the chunk is grouped, while the
+ # metric lists answer "what did a request wait", which needs a call that returned. Counting
+ # only completed calls made a run that issued one shared forward and lost it to an error
+ # under `on_error="skip"` report the same `rows_grouped` and `max_chunk` as a runner with
+ # no batching at all (#592 review).
+ chunks += 1
+ if len(chunk) > 1:
+ rows_grouped += len(chunk)
+ else:
+ rows_alone += 1
+ if len(chunk) > max_chunk:
+ max_chunk = len(chunk)
started = time.perf_counter()
try:
if len(chunk) > 1:
@@ -396,13 +409,6 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
# publish a per-decision cost under the name of the per-request one (#585).
waits.extend([elapsed] * len(chunk))
shares.extend([elapsed / len(chunk)] * len(chunk))
- chunks += 1
- if len(chunk) > 1:
- rows_grouped += len(chunk)
- else:
- rows_alone += 1
- if len(chunk) > max_chunk:
- max_chunk = len(chunk)
for offset, (example, result) in enumerate(zip(chunk, results)):
answers = (result or {}).get("answers") or {}
for qid, expected in example.expected.items():
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 5e699527..9b49e546 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -386,6 +386,50 @@ def test_a_requested_batch_size_is_not_a_batched_run(monkeypatch):
assert report.config["timing"]["rows_alone"] == 2
+def test_a_batched_call_that_raises_is_still_recorded_as_batched(monkeypatch):
+ """`config["timing"]` counts the calls the harness issued, not only the calls that returned.
+
+ A raised chunk used to contribute to none of the shape counters, which made a run that issued
+ one shared forward and lost it indistinguishable from a run that never shared a call at all --
+ the one mode where `docs/evals.md` says the block has to be right.
+ """
+ import laya.evals as evals_module
+
+ class RaisingBatch(TimedRunner):
+ def predict_batch(self, states, questions, model=None, batch_size=None):
+ self.chunks.append(len(states))
+ raise RuntimeError("the shared forward failed")
+
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"}),
+ Example("s3", Q, {"intent": "a"}), Example("s4", QWIDE, {"intent": "a"})])
+ answers = {state: {"intent": choice_answer("a")} for state in ("s1", "s2", "s3", "s4")}
+ runner = RaisingBatch(answers)
+ monkeypatch.setattr(evals_module, "time", runner.clock)
+ report = evals_module.evaluate(runner, dataset, evaluators=[ChoiceAccuracy()],
+ batch_size=8, on_error="skip")
+
+ # Why the runner records it itself: the counters under test are the harness's own account, so
+ # an independent witness that a three-row forward really went out is what makes the assertion
+ # mean something.
+ assert runner.chunks == [3], "a shared forward was issued for the three matching rows"
+ timing = report.config["timing"]
+ assert (timing["chunks"], timing["rows_grouped"], timing["rows_alone"]) == (2, 3, 1)
+ assert timing["max_chunk"] == 3
+ assert len(report.config["errored"]) == 3
+
+ # Why the metric lists stay below the `continue`: a call that returned nothing has no request
+ # latency to publish, so counting the attempt must not invent one. Only s4's own `predict`
+ # reaches the percentiles here.
+ assert report.overall["latency_p50_ms"] == pytest.approx(FORWARD_MS)
+ assert [case["correct"] for case in report.cases] == [True]
+
+ # The ambiguity this closes: a runner with no `predict_batch` gave the identical three
+ # counters, so the artifact could not tell "nothing was batched" from "the batch raised".
+ plain = evaluate(StubRunner(answers), dataset, evaluators=[ChoiceAccuracy()], batch_size=8)
+ for key in ("chunks", "rows_grouped", "max_chunk"):
+ assert timing[key] != plain.config["timing"][key], key
+
+
def test_compare_leaves_the_cost_metrics_alone():
report = EvalReport(overall={"choice_accuracy": 0.8, "latency_p50_ms": 12.0,
"cost_per_decision_p50_ms": 4.0})