Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(evals): chunk rows whose criteria dicts differ only in key order

Aashishd67c461568de
diff --git a/laya/evals.py b/laya/evals.py
index 8b780f62..bc1ac495 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -587,6 +587,17 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
examples = dataset.examples
# Only worth grouping if the runner can be handed the group in one call at all.
batch_form = _batch_form(runner) if batch_size is not None and batch_size > 1 else None
+ # The chunk grouping hashes the questions each row carries, but a `questions` dict is
+ # only equal as written: the same choice schema with two `criteria` dicts written in a
+ # different key order is a different signature, so rows that would share one forward pass
+ # are scored alone. Canonicalizing here is safe because chunking is a calling convention,
+ # not a model input -- the runner still receives each example's own `questions` object
+ # untouched. Only dict keys are reordered (`sort_keys=True` reorders dict keys and leaves
+ # lists alone): a list-valued `criteria` is positional, so two orders stay two questions,
+ # exactly as `Router._question_schema` treats them.
+ chunk_signatures = [json.dumps(dict(sorted((example.questions or {}).items())),
+ sort_keys=True, default=str)
+ for example in examples]
# The chunk shape travels with the calls that have one: a chunk of a single example is a plain
# `predict`, which has no batch to reorder, and an off control is not sent at all.
shape = ({"sort_by_length": True}
@@ -598,11 +609,10 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
# same checkpoint and identical questions. Otherwise every example is one predict.
chunk = [examples[index]]
if batch_form:
- signature = (examples[index].model,
- json.dumps(examples[index].questions, sort_keys=False, default=str))
while (index + len(chunk) < len(examples) and len(chunk) < batch_size
and (examples[index + len(chunk)].model,
- json.dumps(examples[index + len(chunk)].questions, sort_keys=False, default=str)) == signature):
+ chunk_signatures[index + len(chunk)]) == (examples[index].model,
+ chunk_signatures[index])):
chunk.append(examples[index + len(chunk)])
# Why these sit above the call and `waits`/`shares` below: the shape counters answer "what
# did the harness issue", which is settled the moment the chunk is grouped, while the
diff --git a/tests/test_evals.py b/tests/test_evals.py
index e9e359a6..f5c5ef55 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -1502,3 +1502,48 @@ def test_parse_revisions_rejects_two_different_bare_commits():
with pytest.raises(EvalError) as exc:
evals_cli._parse_revisions(["abc", "def"])
assert "two commits" in str(exc.value)
+
+
+def test_chunking_ignores_criteria_dict_key_order():
+ """Two choice rows whose criteria dicts differ only in key order share a call.
+
+ A chunk is one forward pass per row, not one shared rendering, and the runner
+ still receives each example's own questions object untouched -- so the grouping
+ signature canonicalizes dict keys. A list-valued criteria is positional, so two
+ orders stay two questions, exactly as Router._question_schema treats them.
+ """
+ def order_of(states, questions):
+ return list(questions["pick"]["criteria"])
+
+ class OrderRecording(StubRunner):
+ def __init__(self, by_state):
+ super().__init__(by_state)
+ self.orders = []
+
+ def predict_batch(self, states, questions, model=None, batch_size=None):
+ self.orders.append(order_of(states, questions))
+ return [{"model": "m", "answers": self.by_state[s]} for s in states]
+
+ answers = {"s1": {"pick": choice_answer("a")}, "s2": {"pick": choice_answer("a")}}
+
+ def rows(criteria_first, criteria_second):
+ first = {"pick": {"type": "choice", "instructions": "Pick",
+ "criteria": criteria_first}}
+ second = {"pick": {"type": "choice", "instructions": "Pick",
+ "criteria": criteria_second}}
+ return Dataset([Example("s1", first, {"pick": "a"}),
+ Example("s2", second, {"pick": "a"})])
+
+ runner = OrderRecording(answers)
+ report = evaluate(runner,
+ rows({"a": "alpha", "b": "beta"}, {"b": "beta", "a": "alpha"}),
+ evaluators=[ChoiceAccuracy()], batch_size=2)
+ assert report.overall["choice_accuracy"] == 1.0
+ assert report.config["timing"]["rows_grouped"] == 2, "one shared call, not two singles"
+ assert runner.orders == [["a", "b"]], "each row's own questions reach the runner"
+
+ listed = OrderRecording(answers)
+ listed_report = evaluate(listed, rows(["alpha", "beta"], ["beta", "alpha"]),
+ evaluators=[ChoiceAccuracy()], batch_size=2)
+ assert listed_report.config["timing"]["rows_grouped"] == 0, \
+ "a reordered list criteria is a different question and still chunks alone"