Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): a row built in Python gets the checks a JSONL row already had
Aashish4cc57bf2dbb5
diff --git a/laya/evals.py b/laya/evals.pyindex 20f17a84..d969c66c 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -165,6 +165,29 @@ class Dataset:def __init__(self, examples: Iterable[Example]):self.examples: List[Example] = list(examples)+ # `Example.from_dict` refuses a JSONL row whose `questions` or `expected` is not an object, or+ # whose `expected` names a question `questions` does not carry. A row built in Python with+ # `Example(...)` skipped all three, and every reader of it takes the two fields as mappings: a+ # non-object `expected` died as an unhandled ``AttributeError``, in `evaluate` only after the+ # runner had answered (`laya/evals.py:1182`), and a non-object `questions` leaked through+ # `questions_fingerprint` (`laya/evals.py:262`). An `expected` key with no question behind it+ # was answered with "runner returned no answer for question 'x'" -- a verdict about the runner,+ # paid for with a whole inference run, for a mistake in the caller's row. The two opt-in doors+ # refused the same rows before calling anything, but as tracebacks+ # (`laya/evals_shortlist.py:38`, `laya/evals_tournament.py:33`). Checking at the door all four+ # readers share names the row and lands before a single call: a leaked traceback exits 1, the+ # code `laya-evals` reserves for a failed threshold (`laya/evals_cli.py:591`).+ for index, example in enumerate(self.examples):+ if not isinstance(example.questions, dict):+ raise EvalError("example %d 'questions' must be an object, got %s"+ % (index, type(example.questions).__name__))+ if not isinstance(example.expected, dict):+ raise EvalError("example %d 'expected' must be an object keyed by question id, got %s"+ % (index, type(example.expected).__name__))+ unknown = sorted(set(example.expected) - set(example.questions))+ if unknown:+ raise EvalError("example %d 'expected' names unknown question(s): %s"+ % (index, ", ".join(unknown)))def __len__(self) -> int:return len(self.examples)diff --git a/tests/test_evals.py b/tests/test_evals.pyindex f8ad6564..59a9012c 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -757,6 +757,103 @@ def test_dataset_rejects_malformed_json_and_empty(tmp_path):Dataset.from_jsonl(str(empty))+# A row built in Python never passes through `Example.from_dict`, which is the only place these three+# checks were made: `questions` and `expected` must be objects, and every `expected` key must name a+# question that is actually there. Every reader takes the two fields as mappings, so a row that+# skipped the checks reached `evaluate`'s scoring loop (`laya/evals.py:1182`), `questions_fingerprint`+# (`laya/evals.py:262`) or an opt-in door's `_validate_dataset` as an unhandled ``AttributeError`` --+# or got as far as the runner and came back as a verdict *about the runner*. `Dataset` is the one door+# all four readers share, so the refusal names the row and lands before a call is paid for.[email protected]("field, value, fragment", [+ ("expected", True, "'expected' must be an object keyed by question id, got bool"),+ ("expected", "a", "'expected' must be an object keyed by question id, got str"),+ ("expected", ["a"], "'expected' must be an object keyed by question id, got list"),+ ("expected", 1, "'expected' must be an object keyed by question id, got int"),+ ("questions", ["a"], "'questions' must be an object, got list"),+ ("questions", None, "'questions' must be an object, got NoneType"),+ ("questions", "a", "'questions' must be an object, got str"),+])+def test_dataset_refuses_a_row_whose_maps_are_not_objects(field, value, fragment):+ row = {"state": "s1", "questions": Q, "expected": {"intent": "a"}}+ row[field] = value+ with pytest.raises(EvalError) as exc:+ Dataset([Example(**row)])+ assert str(exc.value) == "example 0 " + fragment++[email protected]("expected, names", [+ ({"nope": "a"}, "nope"),+ # Sorted, so the same mistake in two runs reads as one error rather than two.+ ({"intent": "a", "nope": "b", "also": "c"}, "also, nope"),+])+def test_dataset_refuses_an_expected_key_no_question_backs(expected, names):+ with pytest.raises(EvalError) as exc:+ Dataset([Example("s1", Q, expected)])+ assert str(exc.value) == "example 0 'expected' names unknown question(s): " + names+++def test_the_row_refusal_names_the_offender_and_costs_no_inference():+ class Counting:+ """Answers the question ids it was handed, the way a real runner does."""++ def __init__(self):+ self.calls = 0++ def predict(self, state, questions, model=None):+ self.calls += 1+ return {"model": "stub",+ "answers": {qid: choice_answer("a") for qid in (questions or [])}}++ runner, good, bad = Counting(), Example("s1", Q, {"intent": "a"}), Example("s2", Q, True)+ with pytest.raises(EvalError) as exc:+ evaluate(runner, Dataset([good, bad]))+ assert "example 1 'expected'" in str(exc.value), "the offending row is named, not just the class"+ assert runner.calls == 0, "the refusal lands before the dataset is run"+ # All three refusals land with nothing paid for. Measured on `4ac226d5`, every one of these cost a+ # runner call first: a `questions` list whose id is not an expected key came back as "runner+ # returned no answer for question 'intent'", a list carrying the expected id scored and then died+ # in `questions_fingerprint` on the way out, and an unbacked expected key is the same+ # runner-verdict shape `from_dict` has always refused for a JSONL row.+ for row in (Example("s1", ["a"], {"intent": "a"}),+ Example("s1", ["intent"], {"intent": "a"}),+ Example("s1", None, {"intent": "a"}),+ Example("s1", Q, {"nope": "a"}),+ Example("s1", Q, True)):+ silent = Counting()+ with pytest.raises(EvalError) as exc:+ evaluate(silent, Dataset([row]))+ assert silent.calls == 0, row+ assert str(exc.value).startswith("example 0 "), row+ with pytest.raises(EvalError):+ evals.questions_fingerprint(Dataset([Example("s1", ["intent"], {"intent": "a"})]))+ with pytest.raises(EvalError):+ evals.questions_fingerprint(Dataset([bad]))+ assert len(evals.questions_fingerprint(Dataset([good]))) == 64+++def test_the_jsonl_door_and_the_python_door_refuse_the_same_rows():+ # `Example.from_dict` is the JSONL route's door and quotes the row as "row 0"; `Dataset` is the+ # route every Python caller takes. Same check, same reason, different prefix -- which is what+ # makes this one row shape with one meaning rather than two dialects.+ for row in ({"state": "s1", "questions": Q, "expected": True},+ {"state": "s1", "questions": ["intent"], "expected": {"intent": "a"}},+ {"state": "s1", "questions": Q, "expected": {"nope": "a"}}):+ with pytest.raises(EvalError) as via_jsonl:+ Example.from_dict(dict(row), where="row 0")+ with pytest.raises(EvalError) as via_python:+ Dataset([Example(**row)])+ reason = str(via_jsonl.value).removeprefix("row 0 ")+ assert str(via_python.value).removeprefix("example 0 ").startswith(reason), row+++def test_a_row_with_an_empty_expected_map_is_still_accepted():+ # Refusing the type is not refusing an unlabelled row: `expected={}` is a mapping, names no+ # question, and scores nothing. That is the shape this guard has to leave alone.+ dataset = Dataset([Example("s1", Q, {})])+ report = evaluate(StubRunner({"s1": {"intent": choice_answer("a")}}), dataset)+ assert report.cases == []++# --------------------------------------------------------------- evaluatedef test_evaluate_overall_and_slices():dataset = Dataset([diff --git a/tests/test_evals_shortlist.py b/tests/test_evals_shortlist.pyindex c2077aa0..ca8b8af9 100644--- a/tests/test_evals_shortlist.py+++ b/tests/test_evals_shortlist.py@@ -100,6 +100,23 @@ def test_invalid_gold_and_out_of_shortlist_answer_fail():run(ChoiceAgent({}), Dataset([sample("one", "a", "ok")]), TableEmbed(), k=True)+def test_a_row_built_in_python_names_the_row_instead_of_crashing():+ # `Example.from_dict` makes these three checks for a JSONL row and `_validate_dataset` reads the+ # same two fields as mappings, so a row built in Python died on `.items()` / `.get()` -- an+ # unhandled traceback, which `laya-evals` reports with the exit code it reserves for a quality+ # failure -- while an `expected` key with no question behind it was refused for the wrong reason+ # ("shortlist evaluation requires a choice question"). All four now land as one row-naming+ # refusal, before the first embedding and the first call.+ for row, fragment in ((Example("one", QUESTION, True), "'expected' must be an object"),+ (Example("one", ["intent"], {"intent": "a"}), "'questions' must be an object"),+ (Example("one", QUESTION, {"nope": "a"}), "'expected' names unknown question")):+ agent, embed = ChoiceAgent({}), TableEmbed()+ with pytest.raises(EvalError, match=r"example 0 " + fragment):+ run(agent, Dataset([row]), embed)+ assert agent.received == [], row+ assert embed.calls == [], row++def test_malformed_shortlist_metadata_fails(monkeypatch):def malformed(*_args, **_kwargs):return {"answers": {"intent": {"type": "choice", "choice": "a"}},diff --git a/tests/test_evals_tournament.py b/tests/test_evals_tournament.pyindex 9eb32062..432a7a4b 100644--- a/tests/test_evals_tournament.py+++ b/tests/test_evals_tournament.py@@ -90,6 +90,21 @@ def test_invalid_inputs_fail_before_inference():evaluate_tournament(ChoiceAgent(), Dataset([row("ok", "a")]), checkpoint_id=" ")+def test_a_row_built_in_python_names_the_row_instead_of_crashing():+ # Same on this door: `Example.from_dict` makes these three checks for a JSONL row and+ # `_validate_dataset` reads the same two fields as mappings, so a row built in Python died on+ # `.items()` / `.get()` as an unhandled AttributeError, and an `expected` key with no question+ # behind it was refused for the wrong reason ("tournament evaluation requires a choice question").+ for row_shape, fragment in ((Example("ok", WIDE, True), "'expected' must be an object"),+ (Example("ok", ["intent"], {"intent": "a"}), "'questions' must be an object"),+ (Example("ok", WIDE, {"nope": "a"}), "'expected' names unknown question")):+ agent = ChoiceAgent()+ with pytest.raises(EvalError, match=r"example 0 " + fragment):+ evaluate_tournament(agent, Dataset([row_shape]), group_size=2,+ checkpoint_id="checkpoint@123")+ assert agent.received == [], row_shape++def test_malformed_tournament_metadata_fails(monkeypatch):def malformed(*_args, **_kwargs):return {"answers": {"intent": {"type": "choice", "choice": "a"}},