Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): a row built in Python gets the checks a JSONL row already had

Aashish4cc57bf2dbb5
diff --git a/laya/evals.py b/laya/evals.py
index 20f17a84..d969c66c 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -165,6 +165,29 @@ class Dataset:
def __init__(self, examples: Iterable[Example]):
self.examples: List[Example] = list(examples)
+ # `Example.from_dict` refuses a JSONL row whose `questions` or `expected` is not an object, or
+ # whose `expected` names a question `questions` does not carry. A row built in Python with
+ # `Example(...)` skipped all three, and every reader of it takes the two fields as mappings: a
+ # non-object `expected` died as an unhandled ``AttributeError``, in `evaluate` only after the
+ # runner had answered (`laya/evals.py:1182`), and a non-object `questions` leaked through
+ # `questions_fingerprint` (`laya/evals.py:262`). An `expected` key with no question behind it
+ # was answered with "runner returned no answer for question 'x'" -- a verdict about the runner,
+ # paid for with a whole inference run, for a mistake in the caller's row. The two opt-in doors
+ # refused the same rows before calling anything, but as tracebacks
+ # (`laya/evals_shortlist.py:38`, `laya/evals_tournament.py:33`). Checking at the door all four
+ # readers share names the row and lands before a single call: a leaked traceback exits 1, the
+ # code `laya-evals` reserves for a failed threshold (`laya/evals_cli.py:591`).
+ for index, example in enumerate(self.examples):
+ if not isinstance(example.questions, dict):
+ raise EvalError("example %d 'questions' must be an object, got %s"
+ % (index, type(example.questions).__name__))
+ if not isinstance(example.expected, dict):
+ raise EvalError("example %d 'expected' must be an object keyed by question id, got %s"
+ % (index, type(example.expected).__name__))
+ unknown = sorted(set(example.expected) - set(example.questions))
+ if unknown:
+ raise EvalError("example %d 'expected' names unknown question(s): %s"
+ % (index, ", ".join(unknown)))
def __len__(self) -> int:
return len(self.examples)
diff --git a/tests/test_evals.py b/tests/test_evals.py
index f8ad6564..59a9012c 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -757,6 +757,103 @@ def test_dataset_rejects_malformed_json_and_empty(tmp_path):
Dataset.from_jsonl(str(empty))
+# A row built in Python never passes through `Example.from_dict`, which is the only place these three
+# checks were made: `questions` and `expected` must be objects, and every `expected` key must name a
+# question that is actually there. Every reader takes the two fields as mappings, so a row that
+# skipped the checks reached `evaluate`'s scoring loop (`laya/evals.py:1182`), `questions_fingerprint`
+# (`laya/evals.py:262`) or an opt-in door's `_validate_dataset` as an unhandled ``AttributeError`` --
+# or got as far as the runner and came back as a verdict *about the runner*. `Dataset` is the one door
+# all four readers share, so the refusal names the row and lands before a call is paid for.
[email protected]("field, value, fragment", [
+ ("expected", True, "'expected' must be an object keyed by question id, got bool"),
+ ("expected", "a", "'expected' must be an object keyed by question id, got str"),
+ ("expected", ["a"], "'expected' must be an object keyed by question id, got list"),
+ ("expected", 1, "'expected' must be an object keyed by question id, got int"),
+ ("questions", ["a"], "'questions' must be an object, got list"),
+ ("questions", None, "'questions' must be an object, got NoneType"),
+ ("questions", "a", "'questions' must be an object, got str"),
+])
+def test_dataset_refuses_a_row_whose_maps_are_not_objects(field, value, fragment):
+ row = {"state": "s1", "questions": Q, "expected": {"intent": "a"}}
+ row[field] = value
+ with pytest.raises(EvalError) as exc:
+ Dataset([Example(**row)])
+ assert str(exc.value) == "example 0 " + fragment
+
+
[email protected]("expected, names", [
+ ({"nope": "a"}, "nope"),
+ # Sorted, so the same mistake in two runs reads as one error rather than two.
+ ({"intent": "a", "nope": "b", "also": "c"}, "also, nope"),
+])
+def test_dataset_refuses_an_expected_key_no_question_backs(expected, names):
+ with pytest.raises(EvalError) as exc:
+ Dataset([Example("s1", Q, expected)])
+ assert str(exc.value) == "example 0 'expected' names unknown question(s): " + names
+
+
+def test_the_row_refusal_names_the_offender_and_costs_no_inference():
+ class Counting:
+ """Answers the question ids it was handed, the way a real runner does."""
+
+ def __init__(self):
+ self.calls = 0
+
+ def predict(self, state, questions, model=None):
+ self.calls += 1
+ return {"model": "stub",
+ "answers": {qid: choice_answer("a") for qid in (questions or [])}}
+
+ runner, good, bad = Counting(), Example("s1", Q, {"intent": "a"}), Example("s2", Q, True)
+ with pytest.raises(EvalError) as exc:
+ evaluate(runner, Dataset([good, bad]))
+ assert "example 1 'expected'" in str(exc.value), "the offending row is named, not just the class"
+ assert runner.calls == 0, "the refusal lands before the dataset is run"
+ # All three refusals land with nothing paid for. Measured on `4ac226d5`, every one of these cost a
+ # runner call first: a `questions` list whose id is not an expected key came back as "runner
+ # returned no answer for question 'intent'", a list carrying the expected id scored and then died
+ # in `questions_fingerprint` on the way out, and an unbacked expected key is the same
+ # runner-verdict shape `from_dict` has always refused for a JSONL row.
+ for row in (Example("s1", ["a"], {"intent": "a"}),
+ Example("s1", ["intent"], {"intent": "a"}),
+ Example("s1", None, {"intent": "a"}),
+ Example("s1", Q, {"nope": "a"}),
+ Example("s1", Q, True)):
+ silent = Counting()
+ with pytest.raises(EvalError) as exc:
+ evaluate(silent, Dataset([row]))
+ assert silent.calls == 0, row
+ assert str(exc.value).startswith("example 0 "), row
+ with pytest.raises(EvalError):
+ evals.questions_fingerprint(Dataset([Example("s1", ["intent"], {"intent": "a"})]))
+ with pytest.raises(EvalError):
+ evals.questions_fingerprint(Dataset([bad]))
+ assert len(evals.questions_fingerprint(Dataset([good]))) == 64
+
+
+def test_the_jsonl_door_and_the_python_door_refuse_the_same_rows():
+ # `Example.from_dict` is the JSONL route's door and quotes the row as "row 0"; `Dataset` is the
+ # route every Python caller takes. Same check, same reason, different prefix -- which is what
+ # makes this one row shape with one meaning rather than two dialects.
+ for row in ({"state": "s1", "questions": Q, "expected": True},
+ {"state": "s1", "questions": ["intent"], "expected": {"intent": "a"}},
+ {"state": "s1", "questions": Q, "expected": {"nope": "a"}}):
+ with pytest.raises(EvalError) as via_jsonl:
+ Example.from_dict(dict(row), where="row 0")
+ with pytest.raises(EvalError) as via_python:
+ Dataset([Example(**row)])
+ reason = str(via_jsonl.value).removeprefix("row 0 ")
+ assert str(via_python.value).removeprefix("example 0 ").startswith(reason), row
+
+
+def test_a_row_with_an_empty_expected_map_is_still_accepted():
+ # Refusing the type is not refusing an unlabelled row: `expected={}` is a mapping, names no
+ # question, and scores nothing. That is the shape this guard has to leave alone.
+ dataset = Dataset([Example("s1", Q, {})])
+ report = evaluate(StubRunner({"s1": {"intent": choice_answer("a")}}), dataset)
+ assert report.cases == []
+
+
# --------------------------------------------------------------- evaluate
def test_evaluate_overall_and_slices():
dataset = Dataset([
diff --git a/tests/test_evals_shortlist.py b/tests/test_evals_shortlist.py
index c2077aa0..ca8b8af9 100644
--- a/tests/test_evals_shortlist.py
+++ b/tests/test_evals_shortlist.py
@@ -100,6 +100,23 @@ def test_invalid_gold_and_out_of_shortlist_answer_fail():
run(ChoiceAgent({}), Dataset([sample("one", "a", "ok")]), TableEmbed(), k=True)
+def test_a_row_built_in_python_names_the_row_instead_of_crashing():
+ # `Example.from_dict` makes these three checks for a JSONL row and `_validate_dataset` reads the
+ # same two fields as mappings, so a row built in Python died on `.items()` / `.get()` -- an
+ # unhandled traceback, which `laya-evals` reports with the exit code it reserves for a quality
+ # failure -- while an `expected` key with no question behind it was refused for the wrong reason
+ # ("shortlist evaluation requires a choice question"). All four now land as one row-naming
+ # refusal, before the first embedding and the first call.
+ for row, fragment in ((Example("one", QUESTION, True), "'expected' must be an object"),
+ (Example("one", ["intent"], {"intent": "a"}), "'questions' must be an object"),
+ (Example("one", QUESTION, {"nope": "a"}), "'expected' names unknown question")):
+ agent, embed = ChoiceAgent({}), TableEmbed()
+ with pytest.raises(EvalError, match=r"example 0 " + fragment):
+ run(agent, Dataset([row]), embed)
+ assert agent.received == [], row
+ assert embed.calls == [], row
+
+
def test_malformed_shortlist_metadata_fails(monkeypatch):
def malformed(*_args, **_kwargs):
return {"answers": {"intent": {"type": "choice", "choice": "a"}},
diff --git a/tests/test_evals_tournament.py b/tests/test_evals_tournament.py
index 9eb32062..432a7a4b 100644
--- a/tests/test_evals_tournament.py
+++ b/tests/test_evals_tournament.py
@@ -90,6 +90,21 @@ def test_invalid_inputs_fail_before_inference():
evaluate_tournament(ChoiceAgent(), Dataset([row("ok", "a")]), checkpoint_id=" ")
+def test_a_row_built_in_python_names_the_row_instead_of_crashing():
+ # Same on this door: `Example.from_dict` makes these three checks for a JSONL row and
+ # `_validate_dataset` reads the same two fields as mappings, so a row built in Python died on
+ # `.items()` / `.get()` as an unhandled AttributeError, and an `expected` key with no question
+ # behind it was refused for the wrong reason ("tournament evaluation requires a choice question").
+ for row_shape, fragment in ((Example("ok", WIDE, True), "'expected' must be an object"),
+ (Example("ok", ["intent"], {"intent": "a"}), "'questions' must be an object"),
+ (Example("ok", WIDE, {"nope": "a"}), "'expected' names unknown question")):
+ agent = ChoiceAgent()
+ with pytest.raises(EvalError, match=r"example 0 " + fragment):
+ evaluate_tournament(agent, Dataset([row_shape]), group_size=2,
+ checkpoint_id="checkpoint@123")
+ assert agent.received == [], row_shape
+
+
def test_malformed_tournament_metadata_fails(monkeypatch):
def malformed(*_args, **_kwargs):
return {"answers": {"intent": {"type": "choice", "choice": "a"}},