Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): fold criteria key order only where each row carries its own questions

Aashishf4e6353105f6
diff --git a/laya/evals.py b/laya/evals.py
index bc1ac495..31bbd4d1 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -590,13 +590,17 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
# The chunk grouping hashes the questions each row carries, but a `questions` dict is
# only equal as written: the same choice schema with two `criteria` dicts written in a
# different key order is a different signature, so rows that would share one forward pass
- # are scored alone. Canonicalizing here is safe because chunking is a calling convention,
- # not a model input -- the runner still receives each example's own `questions` object
- # untouched. Only dict keys are reordered (`sort_keys=True` reorders dict keys and leaves
- # lists alone): a list-valued `criteria` is positional, so two orders stay two questions,
- # exactly as `Router._question_schema` treats them.
+ # are scored alone. Canonicalizing the key order is only safe where each row reaches the
+ # runner with its own `questions` object -- the per-request `_BATCH_REQUESTS` form. The
+ # positional `_BATCH_STATES` form hands the whole chunk `chunk[0].questions`, so folding
+ # key order there would render a later row with an earlier row's positional `criteria`
+ # order and silently move the metric. Gate the fold on the call shape: `sort_keys=True`
+ # for per-request runners, `sort_keys=False` (insertion order significant) for positional
+ # ones -- exactly the rule `Router._question_schema` and `_canonical` above keep. A
+ # list-valued `criteria` is positional on both paths, and `sort_keys` leaves lists alone.
+ folded = batch_form == _BATCH_REQUESTS
chunk_signatures = [json.dumps(dict(sorted((example.questions or {}).items())),
- sort_keys=True, default=str)
+ sort_keys=folded, default=str)
for example in examples]
# The chunk shape travels with the calls that have one: a chunk of a single example is a plain
# `predict`, which has no batch to reorder, and an off control is not sent at all.
diff --git a/tests/test_evals.py b/tests/test_evals.py
index f5c5ef55..bf97fd79 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -1504,46 +1504,72 @@ def test_parse_revisions_rejects_two_different_bare_commits():
assert "two commits" in str(exc.value)
-def test_chunking_ignores_criteria_dict_key_order():
- """Two choice rows whose criteria dicts differ only in key order share a call.
+def _criteria_rows(criteria_first, criteria_second):
+ first = {"pick": {"type": "choice", "instructions": "Pick", "criteria": criteria_first}}
+ second = {"pick": {"type": "choice", "instructions": "Pick", "criteria": criteria_second}}
+ return Dataset([Example("s1", first, {"pick": "a"}),
+ Example("s2", second, {"pick": "a"})])
+
+
+ANSWER_A = {"s1": {"pick": choice_answer("a")}, "s2": {"pick": choice_answer("a")}}
- A chunk is one forward pass per row, not one shared rendering, and the runner
- still receives each example's own questions object untouched -- so the grouping
- signature canonicalizes dict keys. A list-valued criteria is positional, so two
- orders stay two questions, exactly as Router._question_schema treats them.
- """
- def order_of(states, questions):
- return list(questions["pick"]["criteria"])
+def test_chunking_keeps_criteria_key_order_on_the_positional_form():
+ """A `predict_batch(states, questions, ...)` runner receives ONE questions dict for the
+ whole chunk. Two rows whose criteria dicts differ only in key order must NOT be grouped
+ there: grouping hands row 2 row 1's positional `criteria` order, and the harness would
+ then report a metric for a question that row never declared. The fold is gated on the
+ call shape precisely so this path keeps `sort_keys=False`."""
class OrderRecording(StubRunner):
def __init__(self, by_state):
super().__init__(by_state)
self.orders = []
def predict_batch(self, states, questions, model=None, batch_size=None):
- self.orders.append(order_of(states, questions))
+ self.orders.append(list(questions["pick"]["criteria"]))
return [{"model": "m", "answers": self.by_state[s]} for s in states]
- answers = {"s1": {"pick": choice_answer("a")}, "s2": {"pick": choice_answer("a")}}
+ runner = OrderRecording(ANSWER_A)
+ report = evaluate(runner, _criteria_rows({"a": "alpha", "b": "beta"},
+ {"b": "beta", "a": "alpha"}),
+ evaluators=[ChoiceAccuracy()], batch_size=2)
+ # Not grouped: the two rows score through their own single predicts, so the batch call
+ # -- the one that would have imposed row 0's order on row 1 -- never runs.
+ assert runner.orders == [], "the positional form must not batch key-order-different rows"
+ assert report.config["timing"]["rows_grouped"] == 0
+ assert report.overall["choice_accuracy"] == 1.0
+
+ listed = OrderRecording(ANSWER_A)
+ listed_report = evaluate(listed, _criteria_rows(["alpha", "beta"], ["beta", "alpha"]),
+ evaluators=[ChoiceAccuracy()], batch_size=2)
+ assert listed_report.config["timing"]["rows_grouped"] == 0, \
+ "a reordered list criteria is a different question and still chunks alone"
- def rows(criteria_first, criteria_second):
- first = {"pick": {"type": "choice", "instructions": "Pick",
- "criteria": criteria_first}}
- second = {"pick": {"type": "choice", "instructions": "Pick",
- "criteria": criteria_second}}
- return Dataset([Example("s1", first, {"pick": "a"}),
- Example("s2", second, {"pick": "a"})])
- runner = OrderRecording(answers)
- report = evaluate(runner,
- rows({"a": "alpha", "b": "beta"}, {"b": "beta", "a": "alpha"}),
+def test_chunking_folds_criteria_key_order_on_the_request_form():
+ """The per-request `predict_batch(requests)` shape carries each row's own `questions`, so
+ folding dict key order is safe here -- these two rows share one call and each request still
+ rides with its own criteria order. This is the only path the canonicalisation belongs on."""
+ class ReqRecording(StubRunner):
+ def __init__(self, by_state):
+ super().__init__(by_state)
+ self.orders = []
+
+ def predict_batch(self, requests, batch_size=None):
+ self.orders.append([list(r["questions"]["pick"]["criteria"]) for r in requests])
+ return [{"model": "m", "answers": self.by_state[r["state"]]} for r in requests]
+
+ runner = ReqRecording(ANSWER_A)
+ report = evaluate(runner, _criteria_rows({"a": "alpha", "b": "beta"},
+ {"b": "beta", "a": "alpha"}),
evaluators=[ChoiceAccuracy()], batch_size=2)
- assert report.overall["choice_accuracy"] == 1.0
assert report.config["timing"]["rows_grouped"] == 2, "one shared call, not two singles"
- assert runner.orders == [["a", "b"]], "each row's own questions reach the runner"
+ # Each request keeps its own criteria order -- row 0 a,b; row 1 b,a -- nothing was rewritten.
+ assert runner.orders == [[["a", "b"], ["b", "a"]]], repr(runner.orders)
+ assert report.overall["choice_accuracy"] == 1.0
- listed = OrderRecording(answers)
- listed_report = evaluate(listed, rows(["alpha", "beta"], ["beta", "alpha"]),
+ listed = ReqRecording(ANSWER_A)
+ listed_report = evaluate(listed, _criteria_rows(["alpha", "beta"], ["beta", "alpha"]),
evaluators=[ChoiceAccuracy()], batch_size=2)
assert listed_report.config["timing"]["rows_grouped"] == 0, \
- "a reordered list criteria is a different question and still chunks alone"
+ "a reordered list criteria stays positional even for the request form"