Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): record the chunk shape a raised call issued
Aashish11710880e832
diff --git a/docs/evals.md b/docs/evals.mdindex 4ec1c573..0a5126b5 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -74,7 +74,11 @@ requests were served fast, not whether the run was cheap. With no `--batch-size``compare` ignores any `*_ms` metric unless a tolerance names it, so these never fail a baseline ontiming noise. What the harness actually did -- the batch size asked for, the runner shape itresolved to, how many rows shared a call, and the largest chunk -- is recorded in the report's-`config.timing`, because the flag alone does not say whether anything was batched.+`config.timing`, because the flag alone does not say whether anything was batched. Those counters+record the calls issued, not the calls that returned: with `on_error=skip`, a chunk whose call+raised still counts in `rows_grouped` and `max_chunk`, next to its entries in `config.errored`. The+two `*_ms` metrics count only the calls that returned, so a failed call never contributes a latency+it did not measure.## Slicesdiff --git a/laya/evals.py b/laya/evals.pyindex 8807fbc2..06d0bc0e 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -369,6 +369,19 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluaand (examples[index + len(chunk)].model,json.dumps(examples[index + len(chunk)].questions, sort_keys=False, default=str)) == signature):chunk.append(examples[index + len(chunk)])+ # Why these sit above the call and `waits`/`shares` below: the shape counters answer "what+ # did the harness issue", which is settled the moment the chunk is grouped, while the+ # metric lists answer "what did a request wait", which needs a call that returned. Counting+ # only completed calls made a run that issued one shared forward and lost it to an error+ # under `on_error="skip"` report the same `rows_grouped` and `max_chunk` as a runner with+ # no batching at all (#592 review).+ chunks += 1+ if len(chunk) > 1:+ rows_grouped += len(chunk)+ else:+ rows_alone += 1+ if len(chunk) > max_chunk:+ max_chunk = len(chunk)started = time.perf_counter()try:if len(chunk) > 1:@@ -396,13 +409,6 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua# publish a per-decision cost under the name of the per-request one (#585).waits.extend([elapsed] * len(chunk))shares.extend([elapsed / len(chunk)] * len(chunk))- chunks += 1- if len(chunk) > 1:- rows_grouped += len(chunk)- else:- rows_alone += 1- if len(chunk) > max_chunk:- max_chunk = len(chunk)for offset, (example, result) in enumerate(zip(chunk, results)):answers = (result or {}).get("answers") or {}for qid, expected in example.expected.items():diff --git a/tests/test_evals.py b/tests/test_evals.pyindex 5e699527..9b49e546 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -386,6 +386,50 @@ def test_a_requested_batch_size_is_not_a_batched_run(monkeypatch):assert report.config["timing"]["rows_alone"] == 2+def test_a_batched_call_that_raises_is_still_recorded_as_batched(monkeypatch):+ """`config["timing"]` counts the calls the harness issued, not only the calls that returned.++ A raised chunk used to contribute to none of the shape counters, which made a run that issued+ one shared forward and lost it indistinguishable from a run that never shared a call at all --+ the one mode where `docs/evals.md` says the block has to be right.+ """+ import laya.evals as evals_module++ class RaisingBatch(TimedRunner):+ def predict_batch(self, states, questions, model=None, batch_size=None):+ self.chunks.append(len(states))+ raise RuntimeError("the shared forward failed")++ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"}),+ Example("s3", Q, {"intent": "a"}), Example("s4", QWIDE, {"intent": "a"})])+ answers = {state: {"intent": choice_answer("a")} for state in ("s1", "s2", "s3", "s4")}+ runner = RaisingBatch(answers)+ monkeypatch.setattr(evals_module, "time", runner.clock)+ report = evals_module.evaluate(runner, dataset, evaluators=[ChoiceAccuracy()],+ batch_size=8, on_error="skip")++ # Why the runner records it itself: the counters under test are the harness's own account, so+ # an independent witness that a three-row forward really went out is what makes the assertion+ # mean something.+ assert runner.chunks == [3], "a shared forward was issued for the three matching rows"+ timing = report.config["timing"]+ assert (timing["chunks"], timing["rows_grouped"], timing["rows_alone"]) == (2, 3, 1)+ assert timing["max_chunk"] == 3+ assert len(report.config["errored"]) == 3++ # Why the metric lists stay below the `continue`: a call that returned nothing has no request+ # latency to publish, so counting the attempt must not invent one. Only s4's own `predict`+ # reaches the percentiles here.+ assert report.overall["latency_p50_ms"] == pytest.approx(FORWARD_MS)+ assert [case["correct"] for case in report.cases] == [True]++ # The ambiguity this closes: a runner with no `predict_batch` gave the identical three+ # counters, so the artifact could not tell "nothing was batched" from "the batch raised".+ plain = evaluate(StubRunner(answers), dataset, evaluators=[ChoiceAccuracy()], batch_size=8)+ for key in ("chunks", "rows_grouped", "max_chunk"):+ assert timing[key] != plain.config["timing"][key], key++def test_compare_leaves_the_cost_metrics_alone():report = EvalReport(overall={"choice_accuracy": 0.8, "latency_p50_ms": 12.0,"cost_per_decision_p50_ms": 4.0})