Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): read a student's abstention from the gate that writes it
aashish9b47d34339cb
diff --git a/laya/_agreement.py b/laya/_agreement.pyindex 07eff427..b8ad4bb0 100644--- a/laya/_agreement.py+++ b/laya/_agreement.py@@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optionalimport numpy as np+from .confidence import GATE_ABSTAINEDfrom .evals import EvalError@@ -282,6 +283,12 @@ def evaluate_agreement(runner: Any, reference: Dict, repeat: Dict, *,Provenance describes the caller's runner; no checkpoint revision is inferred or fabricated.The runner receives copied inputs and its exceptions propagate without dropping cases.++ An abstention reaches this through the gate's own vocabulary: an answer carrying+ ``abstention`` equal to :data:`laya.confidence.GATE_ABSTAINED` becomes a null, which+ `compare_agreement` then counts as uncovered. The gate's two other states do not. `passed` is+ a clearance, and `unevaluated` means the gate had no usable confidence to compare -- the+ student still answered, so the choice is kept."""_controls(bootstrap_samples, seed)ref, rep = _teachers(reference, repeat)@@ -304,7 +311,11 @@ def evaluate_agreement(runner: Any, reference: Dict, repeat: Dict, *,choice = answer.get("choice")if not isinstance(choice, str) or choice not in _labels(question, where):raise EvalError("%s returned an invalid choice label" % where)- choices[qid] = None if answer.get("abstention") == "abstained" else choice+ # The marker belongs to the gate's vocabulary, not to this module: of the three states+ # `laya.confidence` can write, only GATE_ABSTAINED means the student declined. A caller+ # that read the field instead of the value ("if answer.get(...)") would drop the+ # `unevaluated` answers -- gated with no usable confidence, choice intact -- from coverage.+ choices[qid] = None if answer.get("abstention") == GATE_ABSTAINED else choicecases.append(dict(deepcopy(case), answers=choices))run_id, suffix = "student", 0while run_id in (reference["run_id"], repeat["run_id"]):diff --git a/tests/test_evals_agreement.py b/tests/test_evals_agreement.pyindex 394bf0aa..edb5e173 100644--- a/tests/test_evals_agreement.py+++ b/tests/test_evals_agreement.py@@ -7,7 +7,7 @@ import sysimport pytest-from laya import evals+from laya import confidence, evalsdef run_document(run_id, rows, *, teacher=True):@@ -266,11 +266,41 @@ def test_evaluate_forwards_model_and_isolates_inputs_from_runner_mutation():def test_evaluate_treats_abstained_argmax_as_uncovered():runner = RecordingRunner({"answers": {"q0": {"type": "choice", "choice": "a",- "abstention": "abstained"}}})+ "abstention": confidence.GATE_ABSTAINED}}})overall = evaluate(runner, documents())["overall"]assert overall["student_coverage"] == overall["student_teacher_agreement"] == 0.0+def test_the_gate_writes_the_marker_and_the_report_reads_it():+ """Abstention is `laya.confidence`'s vocabulary, so this result comes from that gate.++ The fixture above types the state by hand, which pins nothing about the field a gated runner+ actually returns: renaming the state, renaming the key, or reading the key as "any marker+ present" all leave a hand-typed test green while the report counts a declined answer as+ covered. Here the gate produces all three states and the report has to single out one of them.+ """+ answers = {+ "q0": {"type": "choice", "choice": "a", "answer_confidence": 0.9}, # clears the gate+ "q1": {"type": "choice", "choice": "a", "answer_confidence": 0.2}, # falls below it+ "q2": {"type": "choice", "choice": "a"}, # nothing to compare+ }+ confidence.apply_confidence_gate([{"answers": answers}], min_confidence=0.5)+ assert [answers[qid]["abstention"] for qid in ("q0", "q1", "q2")] == list(confidence.GATE_STATES)+ declined = [qid for qid in answers if answers[qid]["abstention"] == confidence.GATE_ABSTAINED]+ assert declined == ["q1"]++ report = evaluate(RecordingRunner({"answers": answers}), documents((("a", "a", "a"),)))+ overall = report["overall"]+ assert overall["n_decisions"] == 3+ assert overall["student_coverage"] == pytest.approx(2 / 3)+ assert overall["student_teacher_agreement"] == pytest.approx(2 / 3)+ assert overall["teacher_repeat_agreement"] == pytest.approx(1.0)+ assert overall["paired_agreement_delta"] == pytest.approx(-1 / 3)+ assert [(case["qid"], case["student"], case["accepted"]) for case in report["cases"]] == [+ ("q0", "a", True), ("q1", None, False), ("q2", "a", True),+ ]++@pytest.mark.parametrize("result", [None, "invalid", {}, {"answers": {}},{"answers": {"q0": {"type": "choice", "choice": "unlisted"}}},{"answers": {"q0": {"type": "score", "score": 0.4}}},