Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): read a student's abstention from the gate that writes it

aashish9b47d34339cb
diff --git a/laya/_agreement.py b/laya/_agreement.py
index 07eff427..b8ad4bb0 100644
--- a/laya/_agreement.py
+++ b/laya/_agreement.py
@@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional
import numpy as np
+from .confidence import GATE_ABSTAINED
from .evals import EvalError
@@ -282,6 +283,12 @@ def evaluate_agreement(runner: Any, reference: Dict, repeat: Dict, *,
Provenance describes the caller's runner; no checkpoint revision is inferred or fabricated.
The runner receives copied inputs and its exceptions propagate without dropping cases.
+
+ An abstention reaches this through the gate's own vocabulary: an answer carrying
+ ``abstention`` equal to :data:`laya.confidence.GATE_ABSTAINED` becomes a null, which
+ `compare_agreement` then counts as uncovered. The gate's two other states do not. `passed` is
+ a clearance, and `unevaluated` means the gate had no usable confidence to compare -- the
+ student still answered, so the choice is kept.
"""
_controls(bootstrap_samples, seed)
ref, rep = _teachers(reference, repeat)
@@ -304,7 +311,11 @@ def evaluate_agreement(runner: Any, reference: Dict, repeat: Dict, *,
choice = answer.get("choice")
if not isinstance(choice, str) or choice not in _labels(question, where):
raise EvalError("%s returned an invalid choice label" % where)
- choices[qid] = None if answer.get("abstention") == "abstained" else choice
+ # The marker belongs to the gate's vocabulary, not to this module: of the three states
+ # `laya.confidence` can write, only GATE_ABSTAINED means the student declined. A caller
+ # that read the field instead of the value ("if answer.get(...)") would drop the
+ # `unevaluated` answers -- gated with no usable confidence, choice intact -- from coverage.
+ choices[qid] = None if answer.get("abstention") == GATE_ABSTAINED else choice
cases.append(dict(deepcopy(case), answers=choices))
run_id, suffix = "student", 0
while run_id in (reference["run_id"], repeat["run_id"]):
diff --git a/tests/test_evals_agreement.py b/tests/test_evals_agreement.py
index 394bf0aa..edb5e173 100644
--- a/tests/test_evals_agreement.py
+++ b/tests/test_evals_agreement.py
@@ -7,7 +7,7 @@ import sys
import pytest
-from laya import evals
+from laya import confidence, evals
def run_document(run_id, rows, *, teacher=True):
@@ -266,11 +266,41 @@ def test_evaluate_forwards_model_and_isolates_inputs_from_runner_mutation():
def test_evaluate_treats_abstained_argmax_as_uncovered():
runner = RecordingRunner({"answers": {"q0": {"type": "choice", "choice": "a",
- "abstention": "abstained"}}})
+ "abstention": confidence.GATE_ABSTAINED}}})
overall = evaluate(runner, documents())["overall"]
assert overall["student_coverage"] == overall["student_teacher_agreement"] == 0.0
+def test_the_gate_writes_the_marker_and_the_report_reads_it():
+ """Abstention is `laya.confidence`'s vocabulary, so this result comes from that gate.
+
+ The fixture above types the state by hand, which pins nothing about the field a gated runner
+ actually returns: renaming the state, renaming the key, or reading the key as "any marker
+ present" all leave a hand-typed test green while the report counts a declined answer as
+ covered. Here the gate produces all three states and the report has to single out one of them.
+ """
+ answers = {
+ "q0": {"type": "choice", "choice": "a", "answer_confidence": 0.9}, # clears the gate
+ "q1": {"type": "choice", "choice": "a", "answer_confidence": 0.2}, # falls below it
+ "q2": {"type": "choice", "choice": "a"}, # nothing to compare
+ }
+ confidence.apply_confidence_gate([{"answers": answers}], min_confidence=0.5)
+ assert [answers[qid]["abstention"] for qid in ("q0", "q1", "q2")] == list(confidence.GATE_STATES)
+ declined = [qid for qid in answers if answers[qid]["abstention"] == confidence.GATE_ABSTAINED]
+ assert declined == ["q1"]
+
+ report = evaluate(RecordingRunner({"answers": answers}), documents((("a", "a", "a"),)))
+ overall = report["overall"]
+ assert overall["n_decisions"] == 3
+ assert overall["student_coverage"] == pytest.approx(2 / 3)
+ assert overall["student_teacher_agreement"] == pytest.approx(2 / 3)
+ assert overall["teacher_repeat_agreement"] == pytest.approx(1.0)
+ assert overall["paired_agreement_delta"] == pytest.approx(-1 / 3)
+ assert [(case["qid"], case["student"], case["accepted"]) for case in report["cases"]] == [
+ ("q0", "a", True), ("q1", None, False), ("q2", "a", True),
+ ]
+
+
@pytest.mark.parametrize("result", [None, "invalid", {}, {"answers": {}},
{"answers": {"q0": {"type": "choice", "choice": "unlisted"}}},
{"answers": {"q0": {"type": "score", "score": 0.4}}},