Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): fold criteria key order only where each row carries its own questions
Aashishf4e6353105f6
diff --git a/laya/evals.py b/laya/evals.pyindex bc1ac495..31bbd4d1 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -590,13 +590,17 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua# The chunk grouping hashes the questions each row carries, but a `questions` dict is# only equal as written: the same choice schema with two `criteria` dicts written in a# different key order is a different signature, so rows that would share one forward pass- # are scored alone. Canonicalizing here is safe because chunking is a calling convention,- # not a model input -- the runner still receives each example's own `questions` object- # untouched. Only dict keys are reordered (`sort_keys=True` reorders dict keys and leaves- # lists alone): a list-valued `criteria` is positional, so two orders stay two questions,- # exactly as `Router._question_schema` treats them.+ # are scored alone. Canonicalizing the key order is only safe where each row reaches the+ # runner with its own `questions` object -- the per-request `_BATCH_REQUESTS` form. The+ # positional `_BATCH_STATES` form hands the whole chunk `chunk[0].questions`, so folding+ # key order there would render a later row with an earlier row's positional `criteria`+ # order and silently move the metric. Gate the fold on the call shape: `sort_keys=True`+ # for per-request runners, `sort_keys=False` (insertion order significant) for positional+ # ones -- exactly the rule `Router._question_schema` and `_canonical` above keep. A+ # list-valued `criteria` is positional on both paths, and `sort_keys` leaves lists alone.+ folded = batch_form == _BATCH_REQUESTSchunk_signatures = [json.dumps(dict(sorted((example.questions or {}).items())),- sort_keys=True, default=str)+ sort_keys=folded, default=str)for example in examples]# The chunk shape travels with the calls that have one: a chunk of a single example is a plain# `predict`, which has no batch to reorder, and an off control is not sent at all.diff --git a/tests/test_evals.py b/tests/test_evals.pyindex f5c5ef55..bf97fd79 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -1504,46 +1504,72 @@ def test_parse_revisions_rejects_two_different_bare_commits():assert "two commits" in str(exc.value)-def test_chunking_ignores_criteria_dict_key_order():- """Two choice rows whose criteria dicts differ only in key order share a call.+def _criteria_rows(criteria_first, criteria_second):+ first = {"pick": {"type": "choice", "instructions": "Pick", "criteria": criteria_first}}+ second = {"pick": {"type": "choice", "instructions": "Pick", "criteria": criteria_second}}+ return Dataset([Example("s1", first, {"pick": "a"}),+ Example("s2", second, {"pick": "a"})])+++ANSWER_A = {"s1": {"pick": choice_answer("a")}, "s2": {"pick": choice_answer("a")}}- A chunk is one forward pass per row, not one shared rendering, and the runner- still receives each example's own questions object untouched -- so the grouping- signature canonicalizes dict keys. A list-valued criteria is positional, so two- orders stay two questions, exactly as Router._question_schema treats them.- """- def order_of(states, questions):- return list(questions["pick"]["criteria"])+def test_chunking_keeps_criteria_key_order_on_the_positional_form():+ """A `predict_batch(states, questions, ...)` runner receives ONE questions dict for the+ whole chunk. Two rows whose criteria dicts differ only in key order must NOT be grouped+ there: grouping hands row 2 row 1's positional `criteria` order, and the harness would+ then report a metric for a question that row never declared. The fold is gated on the+ call shape precisely so this path keeps `sort_keys=False`."""class OrderRecording(StubRunner):def __init__(self, by_state):super().__init__(by_state)self.orders = []def predict_batch(self, states, questions, model=None, batch_size=None):- self.orders.append(order_of(states, questions))+ self.orders.append(list(questions["pick"]["criteria"]))return [{"model": "m", "answers": self.by_state[s]} for s in states]- answers = {"s1": {"pick": choice_answer("a")}, "s2": {"pick": choice_answer("a")}}+ runner = OrderRecording(ANSWER_A)+ report = evaluate(runner, _criteria_rows({"a": "alpha", "b": "beta"},+ {"b": "beta", "a": "alpha"}),+ evaluators=[ChoiceAccuracy()], batch_size=2)+ # Not grouped: the two rows score through their own single predicts, so the batch call+ # -- the one that would have imposed row 0's order on row 1 -- never runs.+ assert runner.orders == [], "the positional form must not batch key-order-different rows"+ assert report.config["timing"]["rows_grouped"] == 0+ assert report.overall["choice_accuracy"] == 1.0++ listed = OrderRecording(ANSWER_A)+ listed_report = evaluate(listed, _criteria_rows(["alpha", "beta"], ["beta", "alpha"]),+ evaluators=[ChoiceAccuracy()], batch_size=2)+ assert listed_report.config["timing"]["rows_grouped"] == 0, \+ "a reordered list criteria is a different question and still chunks alone"- def rows(criteria_first, criteria_second):- first = {"pick": {"type": "choice", "instructions": "Pick",- "criteria": criteria_first}}- second = {"pick": {"type": "choice", "instructions": "Pick",- "criteria": criteria_second}}- return Dataset([Example("s1", first, {"pick": "a"}),- Example("s2", second, {"pick": "a"})])- runner = OrderRecording(answers)- report = evaluate(runner,- rows({"a": "alpha", "b": "beta"}, {"b": "beta", "a": "alpha"}),+def test_chunking_folds_criteria_key_order_on_the_request_form():+ """The per-request `predict_batch(requests)` shape carries each row's own `questions`, so+ folding dict key order is safe here -- these two rows share one call and each request still+ rides with its own criteria order. This is the only path the canonicalisation belongs on."""+ class ReqRecording(StubRunner):+ def __init__(self, by_state):+ super().__init__(by_state)+ self.orders = []++ def predict_batch(self, requests, batch_size=None):+ self.orders.append([list(r["questions"]["pick"]["criteria"]) for r in requests])+ return [{"model": "m", "answers": self.by_state[r["state"]]} for r in requests]++ runner = ReqRecording(ANSWER_A)+ report = evaluate(runner, _criteria_rows({"a": "alpha", "b": "beta"},+ {"b": "beta", "a": "alpha"}),evaluators=[ChoiceAccuracy()], batch_size=2)- assert report.overall["choice_accuracy"] == 1.0assert report.config["timing"]["rows_grouped"] == 2, "one shared call, not two singles"- assert runner.orders == [["a", "b"]], "each row's own questions reach the runner"+ # Each request keeps its own criteria order -- row 0 a,b; row 1 b,a -- nothing was rewritten.+ assert runner.orders == [[["a", "b"], ["b", "a"]]], repr(runner.orders)+ assert report.overall["choice_accuracy"] == 1.0- listed = OrderRecording(answers)- listed_report = evaluate(listed, rows(["alpha", "beta"], ["beta", "alpha"]),+ listed = ReqRecording(ANSWER_A)+ listed_report = evaluate(listed, _criteria_rows(["alpha", "beta"], ["beta", "alpha"]),evaluators=[ChoiceAccuracy()], batch_size=2)assert listed_report.config["timing"]["rows_grouped"] == 0, \- "a reordered list criteria is a different question and still chunks alone"+ "a reordered list criteria stays positional even for the request form"