Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(evals): chunk rows whose criteria dicts differ only in key order
Aashishd67c461568de
diff --git a/laya/evals.py b/laya/evals.pyindex 8b780f62..bc1ac495 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -587,6 +587,17 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluaexamples = dataset.examples# Only worth grouping if the runner can be handed the group in one call at all.batch_form = _batch_form(runner) if batch_size is not None and batch_size > 1 else None+ # The chunk grouping hashes the questions each row carries, but a `questions` dict is+ # only equal as written: the same choice schema with two `criteria` dicts written in a+ # different key order is a different signature, so rows that would share one forward pass+ # are scored alone. Canonicalizing here is safe because chunking is a calling convention,+ # not a model input -- the runner still receives each example's own `questions` object+ # untouched. Only dict keys are reordered (`sort_keys=True` reorders dict keys and leaves+ # lists alone): a list-valued `criteria` is positional, so two orders stay two questions,+ # exactly as `Router._question_schema` treats them.+ chunk_signatures = [json.dumps(dict(sorted((example.questions or {}).items())),+ sort_keys=True, default=str)+ for example in examples]# The chunk shape travels with the calls that have one: a chunk of a single example is a plain# `predict`, which has no batch to reorder, and an off control is not sent at all.shape = ({"sort_by_length": True}@@ -598,11 +609,10 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua# same checkpoint and identical questions. Otherwise every example is one predict.chunk = [examples[index]]if batch_form:- signature = (examples[index].model,- json.dumps(examples[index].questions, sort_keys=False, default=str))while (index + len(chunk) < len(examples) and len(chunk) < batch_sizeand (examples[index + len(chunk)].model,- json.dumps(examples[index + len(chunk)].questions, sort_keys=False, default=str)) == signature):+ chunk_signatures[index + len(chunk)]) == (examples[index].model,+ chunk_signatures[index])):chunk.append(examples[index + len(chunk)])# Why these sit above the call and `waits`/`shares` below: the shape counters answer "what# did the harness issue", which is settled the moment the chunk is grouped, while thediff --git a/tests/test_evals.py b/tests/test_evals.pyindex e9e359a6..f5c5ef55 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -1502,3 +1502,48 @@ def test_parse_revisions_rejects_two_different_bare_commits():with pytest.raises(EvalError) as exc:evals_cli._parse_revisions(["abc", "def"])assert "two commits" in str(exc.value)+++def test_chunking_ignores_criteria_dict_key_order():+ """Two choice rows whose criteria dicts differ only in key order share a call.++ A chunk is one forward pass per row, not one shared rendering, and the runner+ still receives each example's own questions object untouched -- so the grouping+ signature canonicalizes dict keys. A list-valued criteria is positional, so two+ orders stay two questions, exactly as Router._question_schema treats them.+ """+ def order_of(states, questions):+ return list(questions["pick"]["criteria"])++ class OrderRecording(StubRunner):+ def __init__(self, by_state):+ super().__init__(by_state)+ self.orders = []++ def predict_batch(self, states, questions, model=None, batch_size=None):+ self.orders.append(order_of(states, questions))+ return [{"model": "m", "answers": self.by_state[s]} for s in states]++ answers = {"s1": {"pick": choice_answer("a")}, "s2": {"pick": choice_answer("a")}}++ def rows(criteria_first, criteria_second):+ first = {"pick": {"type": "choice", "instructions": "Pick",+ "criteria": criteria_first}}+ second = {"pick": {"type": "choice", "instructions": "Pick",+ "criteria": criteria_second}}+ return Dataset([Example("s1", first, {"pick": "a"}),+ Example("s2", second, {"pick": "a"})])++ runner = OrderRecording(answers)+ report = evaluate(runner,+ rows({"a": "alpha", "b": "beta"}, {"b": "beta", "a": "alpha"}),+ evaluators=[ChoiceAccuracy()], batch_size=2)+ assert report.overall["choice_accuracy"] == 1.0+ assert report.config["timing"]["rows_grouped"] == 2, "one shared call, not two singles"+ assert runner.orders == [["a", "b"]], "each row's own questions reach the runner"++ listed = OrderRecording(answers)+ listed_report = evaluate(listed, rows(["alpha", "beta"], ["beta", "alpha"]),+ evaluators=[ChoiceAccuracy()], batch_size=2)+ assert listed_report.config["timing"]["rows_grouped"] == 0, \+ "a reordered list criteria is a different question and still chunks alone"