Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): report what a request waited, and name the throughput share

Aashishdfba47a4ef3d
diff --git a/docs/evals.md b/docs/evals.md
index 3660ec3a..4ec1c573 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -56,11 +56,26 @@ Each metric is computed per answer where it applies and aggregated over the data
| `score_within_<tol>` | `score` | fraction within an absolute tolerance |
| `ece` | any answer with a confidence | expected calibration error, 15 bins, computed on `answer["answer_confidence"]`, the calibrated probability Laya reports on every answer type |
| `mean_confidence` | any answer with a confidence | mean reported `answer["answer_confidence"]` |
-| `latency_p50_ms`, `latency_p95_ms` | per request | wall time, informational |
+| `latency_p50_ms`, `latency_p95_ms` | per request | wall time each request waited, informational -- see [batching](#batching-and-timing) |
+| `cost_per_decision_p50_ms`, `cost_per_decision_p95_ms` | per decision | a call's wall time divided by the rows it carried, informational |
Add `ScoreWithin(0.25)` to the evaluator list for a tolerance metric; the default set is
`choice_accuracy`, `noul_accuracy`, `score_mae`, `mean_confidence`, plus `ece`.
+## Batching and timing
+
+`--batch-size N` scores up to N consecutive rows that share a checkpoint and a question schema in
+one call. Both timing metrics come from the same measurements and answer different questions:
+every row of a batch returns when the batch does, so its `latency` is the whole call, while its
+`cost_per_decision` is `1/N` of it. Batching therefore *raises* `latency_*` and *lowers*
+`cost_per_decision_*` on an unchanged set of decisions, and `--max latency_p50_ms=...` asks whether
+requests were served fast, not whether the run was cheap. With no `--batch-size` the two agree.
+
+`compare` ignores any `*_ms` metric unless a tolerance names it, so these never fail a baseline on
+timing noise. What the harness actually did -- the batch size asked for, the runner shape it
+resolved to, how many rows shared a call, and the largest chunk -- is recorded in the report's
+`config.timing`, because the flag alone does not say whether anything was batched.
+
## Slices
`compare` and `run` report overall numbers and, for `--slice language|model|qid|tag`, the same
@@ -88,7 +103,8 @@ Two CI surfaces use this:
PR.
The harness is deterministic for a fixed checkpoint revision, so a report is reproducible.
-`run` records the dataset, model and device in the report's `config` block.
+`run` records the dataset, model and device, plus the [timing](#batching-and-timing) facts of the
+run, in the report's `config` block.
## Adding the real labelled set
diff --git a/laya/evals.py b/laya/evals.py
index 111f59f4..8807fbc2 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -264,6 +264,13 @@ def _group_cases(cases: Sequence[Dict[str, Any]], key: str) -> Dict[str, List[Di
return groups
+def _percentiles(values: Sequence[float]) -> Tuple[float, float]:
+ """(median, 95th) using the nearest-rank rule the report has always used for latency."""
+ ordered = sorted(values)
+ return (float(statistics.median(ordered)),
+ float(ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))]))
+
+
def _aggregate(cases: Sequence[Dict[str, Any]], evaluators: Sequence[Evaluator]) -> Dict[str, float]:
out: Dict[str, float] = {}
for evaluator in evaluators:
@@ -329,12 +336,22 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
`on_error` is ``"fail"`` (re-raise a runner error) or ``"skip"`` (record it and continue),
the latter for evaluating a flaky fleet without aborting the whole run.
+
+ A batched run has two honest timing answers, so the report gives both. ``latency_p50_ms`` /
+ ``latency_p95_ms`` are per request: every row of a chunk returns from the same call, so each
+ one waited that whole call. ``cost_per_decision_p50_ms`` / ``cost_per_decision_p95_ms``
+ divide a call by its own chunk size, which is the throughput figure. ``--batch-size`` therefore
+ lowers the second and raises the first. What the harness really did -- the runner shape it
+ resolved to and how many rows shared a call -- lands in ``report.config["timing"]``, because the
+ requested flag alone does not say whether anything was batched.
"""
if on_error not in ("fail", "skip"):
raise EvalError("on_error must be 'fail' or 'skip', got %r" % on_error)
evaluators = list(evaluators) if evaluators is not None else default_evaluators()
cases: List[Dict[str, Any]] = []
- latencies: List[float] = []
+ waits: List[float] = [] # what each request actually waited: its chunk's whole call
+ shares: List[float] = [] # that call split across its chunk: throughput per decision
+ chunks = rows_grouped = rows_alone = max_chunk = 0
errors: List[Dict[str, Any]] = []
examples = dataset.examples
# Only worth grouping if the runner can be handed the group in one call at all.
@@ -374,7 +391,18 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
index += len(chunk)
continue
elapsed = (time.perf_counter() - started) * 1000.0
- latencies.extend([elapsed / len(chunk)] * len(chunk))
+ # A chunk is one call, so each of its rows waited all of `elapsed`; the split figure is a
+ # throughput share, not a latency, and putting the two in one list made a batched run
+ # publish a per-decision cost under the name of the per-request one (#585).
+ waits.extend([elapsed] * len(chunk))
+ shares.extend([elapsed / len(chunk)] * len(chunk))
+ chunks += 1
+ if len(chunk) > 1:
+ rows_grouped += len(chunk)
+ else:
+ rows_alone += 1
+ if len(chunk) > max_chunk:
+ max_chunk = len(chunk)
for offset, (example, result) in enumerate(zip(chunk, results)):
answers = (result or {}).get("answers") or {}
for qid, expected in example.expected.items():
@@ -400,10 +428,20 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
report = EvalReport(config=dict(config or {}), cases=cases)
report.overall = _aggregate(cases, evaluators)
- if latencies:
- ordered = sorted(latencies)
- report.overall["latency_p50_ms"] = float(statistics.median(ordered))
- report.overall["latency_p95_ms"] = float(ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))])
+ if waits:
+ report.overall["latency_p50_ms"], report.overall["latency_p95_ms"] = _percentiles(waits)
+ (report.overall["cost_per_decision_p50_ms"],
+ report.overall["cost_per_decision_p95_ms"]) = _percentiles(shares)
+ report.config = dict(report.config, timing={
+ "latency_metric": "per request: the wall time of the call that returned it, unsplit",
+ "cost_metric": "per decision: that call divided by its own chunk size",
+ "batch_size": batch_size,
+ "batch_form": batch_form,
+ "chunks": chunks,
+ "rows_grouped": rows_grouped,
+ "rows_alone": rows_alone,
+ "max_chunk": max_chunk,
+ })
for dimension in ("language", "model", "qid", "tag"):
groups = _group_cases(cases, dimension)
if groups:
diff --git a/tests/test_evals.py b/tests/test_evals.py
index c6ad813a..5e699527 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -246,6 +246,181 @@ def test_evaluate_batches_a_pass_through_wrapper_positionally():
assert report.overall["choice_accuracy"] == 1.0
+# --------------------------------------------------------------- timing (#585)
+FORWARD_MS = 100.0
+SHARED = 0.6 # a batch of n costs SHARED * n * FORWARD_MS, as one call
+QWIDE = {"intent": {"type": "choice", "instructions": "?",
+ "criteria": {"a": "x", "b": "y", "c": "z"}}}
+
+
+class Clock:
+ """A timer that advances only when the runner predicts, so every figure below is exact."""
+
+ def __init__(self):
+ self.now = 0.0
+
+ def perf_counter(self):
+ return self.now / 1000.0
+
+
+class TimedRunner(StubRunner):
+ """Answers from the state alone, so grouping can never change a decision -- only the clock."""
+
+ def __init__(self, by_state):
+ super().__init__(by_state)
+ self.clock = Clock()
+ self.chunks = []
+ self.singles = []
+
+ def predict(self, state, questions, model=None):
+ self.clock.now += FORWARD_MS
+ self.singles.append(state)
+ return StubRunner.predict(self, state, questions, model)
+
+ def predict_batch(self, states, questions, model=None, batch_size=None):
+ self.clock.now += len(states) * FORWARD_MS * SHARED
+ self.chunks.append(len(states))
+ return [StubRunner.predict(self, s, questions, model) for s in states]
+
+
+def _timed_pair(monkeypatch, batch_size):
+ """Score three shareable rows and one that cannot join them, on a fake clock.
+
+ Returns (report, runner): the runner's own `chunks` is the witness that the grouping the test
+ asserts is the grouping the harness really issued.
+ """
+ import laya.evals as evals_module
+
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"}),
+ Example("s3", Q, {"intent": "a"}), Example("s4", QWIDE, {"intent": "a"})])
+ answers = {state: {"intent": choice_answer("a")} for state in ("s1", "s2", "s3", "s4")}
+ runner = TimedRunner(answers)
+ monkeypatch.setattr(evals_module, "time", runner.clock)
+ return evals_module.evaluate(runner, dataset, evaluators=[ChoiceAccuracy()],
+ batch_size=batch_size), runner
+
+
+def test_batched_latency_is_what_a_request_waited(monkeypatch):
+ solo, solo_runner = _timed_pair(monkeypatch, None)
+ batched, runner = _timed_pair(monkeypatch, 8)
+
+ assert _labels(batched) == _labels(solo), "identical decisions; only the timing moved"
+ assert (solo_runner.chunks, solo_runner.singles) == ([], ["s1", "s2", "s3", "s4"])
+ assert (runner.chunks, runner.singles) == ([3], ["s4"]), \
+ "one shared call of three, and the row whose questions match nothing left alone"
+
+ assert solo.overall["latency_p50_ms"] == pytest.approx(FORWARD_MS)
+ # The chunk of three returns all three requests together at +180 ms, so that is their latency.
+ assert batched.overall["latency_p50_ms"] == pytest.approx(3 * FORWARD_MS * SHARED)
+ assert batched.overall["latency_p50_ms"] > solo.overall["latency_p50_ms"], \
+ "batching trades request latency for throughput; the report has to say so"
+
+
+def test_the_throughput_share_keeps_its_own_metric(monkeypatch):
+ solo, _ = _timed_pair(monkeypatch, None)
+ batched, _ = _timed_pair(monkeypatch, 8)
+
+ # Unbatched, the two quantities are the same number, so every report without the flag is
+ # unchanged by this fix.
+ assert solo.overall["cost_per_decision_p50_ms"] == pytest.approx(solo.overall["latency_p50_ms"])
+ assert solo.overall["cost_per_decision_p95_ms"] == pytest.approx(solo.overall["latency_p95_ms"])
+ # Batched, the share is the figure the old `latency_p50_ms` published: 180 ms over three rows.
+ assert batched.overall["cost_per_decision_p50_ms"] == pytest.approx(FORWARD_MS * SHARED)
+ assert batched.overall["cost_per_decision_p95_ms"] == pytest.approx(FORWARD_MS)
+
+
+def test_a_latency_gate_cannot_pass_a_run_where_nothing_finished_in_time(monkeypatch):
+ from laya import evals_cli
+
+ solo, _ = _timed_pair(monkeypatch, None)
+ batched, _ = _timed_pair(monkeypatch, 8)
+ # 80 ms is below every wait in either run (100 ms alone, 180 ms shared). The old report passed
+ # the batched run at 60 ms, which was 1/3 of a call no request could see the end of.
+ for name, report in (("unbatched", solo), ("--batch-size 8", batched)):
+ assert evals_cli._check_thresholds(report.overall, {}, {"latency_p50_ms": 80.0}), \
+ "%s abstains: no request in it was served inside the limit" % name
+ for name, report in (("unbatched", solo), ("--batch-size 8", batched)):
+ assert not evals_cli._check_thresholds(report.overall, {}, {"latency_p50_ms": 200.0}), \
+ "%s passes a bound every request beat" % name
+ # The throughput win is still gateable, under the name that measures it.
+ assert not evals_cli._check_thresholds(batched.overall, {}, {"cost_per_decision_p50_ms": 80.0})
+ assert evals_cli._check_thresholds(solo.overall, {}, {"cost_per_decision_p50_ms": 80.0})
+
+
+def test_timing_facts_record_what_the_harness_did(monkeypatch):
+ solo, _ = _timed_pair(monkeypatch, None)
+ batched, _ = _timed_pair(monkeypatch, 8)
+
+ assert solo.config["timing"]["batch_size"] is None
+ assert solo.config["timing"]["batch_form"] is None
+ assert solo.config["timing"]["rows_grouped"] == 0
+ assert solo.config["timing"]["rows_alone"] == 4
+ assert solo.config["timing"]["max_chunk"] == 1
+ assert batched.config["timing"]["batch_size"] == 8
+ assert batched.config["timing"]["batch_form"] == "states"
+ assert batched.config["timing"]["chunks"] == 2
+ assert batched.config["timing"]["rows_grouped"] == 3
+ assert batched.config["timing"]["rows_alone"] == 1
+ assert batched.config["timing"]["max_chunk"] == 3
+ assert batched.config["timing"]["latency_metric"] != batched.config["timing"]["cost_metric"]
+
+ # `--json` is the artifact a reviewer reads, so the two runs must differ there, not only in
+ # the flag they were asked with.
+ assert json.dumps(batched.to_json()) != json.dumps(solo.to_json())
+
+
+def test_a_requested_batch_size_is_not_a_batched_run(monkeypatch):
+ """The report records the grouping it achieved, so a runner that cannot batch cannot hide it."""
+ import laya.evals as evals_module
+
+ class Untimed(StubRunner):
+ pass
+
+ dataset = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"})])
+ monkeypatch.setattr(evals_module, "time", Clock())
+ report = evals_module.evaluate(Untimed({"s1": {"intent": choice_answer("a")},
+ "s2": {"intent": choice_answer("a")}}),
+ dataset, evaluators=[ChoiceAccuracy()], batch_size=8)
+ assert report.config["timing"]["batch_form"] is None
+ assert report.config["timing"]["rows_grouped"] == 0, "asked to batch, and the report says it did not"
+ assert report.config["timing"]["rows_alone"] == 2
+
+
+def test_compare_leaves_the_cost_metrics_alone():
+ report = EvalReport(overall={"choice_accuracy": 0.8, "latency_p50_ms": 12.0,
+ "cost_per_decision_p50_ms": 4.0})
+ baseline = {"overall": {"choice_accuracy": 0.8, "latency_p50_ms": 5.0,
+ "cost_per_decision_p50_ms": 2.0}}
+ ok, deltas = report.compare(baseline)
+ assert ok and not {"latency_p50_ms", "cost_per_decision_p50_ms"} & set(deltas)
+ bad, deltas = report.compare(baseline, {"cost_per_decision_p50_ms": 0.5})
+ assert not bad, "a 2 ms drift is outside the 0.5 ms tolerance it named"
+ assert "latency_p50_ms" not in deltas, "the metric nobody named stays out of the comparison"
+ assert deltas["cost_per_decision_p50_ms"]["diff"] == pytest.approx(2.0)
+
+
+def test_docs_and_the_harness_name_the_same_timing_metrics(monkeypatch):
+ """`docs/evals.md` must list exactly the metrics `evaluate` publishes, with their quantities."""
+ import pathlib
+ import re
+
+ page = (pathlib.Path(__file__).resolve().parent.parent / "docs" / "evals.md").read_text()
+ batched, _ = _timed_pair(monkeypatch, 8)
+ published = {metric for metric in batched.overall if metric.endswith("_ms")}
+
+ rows = {}
+ for line in page.splitlines():
+ if line.startswith("| `"):
+ for metric in re.findall(r"`([a-z_0-9]+_ms)`", line):
+ rows[metric] = line
+ assert set(rows) == published, "no metric documented that is not emitted, and none missed"
+ assert "per request" in rows["latency_p50_ms"]
+ assert "waited" in rows["latency_p50_ms"], "the row has to say a request waits the whole call"
+ assert "divided by the rows it carried" in rows["cost_per_decision_p50_ms"]
+ assert "## Batching and timing" in page, "the trade-off the two numbers encode is written down"
+ assert "`config.timing`" in page, "the report's own run facts are documented"
+
+
def test_evaluate_skips_errors_when_asked():
class Boom(StubRunner):
def predict(self, state, questions, model=None):