Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): make laya-evals run --min-confidence reach the abstention gate

Aashishd3830a813e46
diff --git a/docs/evals.md b/docs/evals.md
index 0e11aefc..e8a25553 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -132,6 +132,26 @@ A run with no `--batch-size` asks for something that cannot happen, and says so
`sent: false`; a runner whose `predict_batch` predates the knob is scored unsorted rather than
raising `TypeError` halfway through a long run.
+### The abstention gate at a threshold
+
+`--min-confidence T` forwards core's opt-in abstention threshold (#361) to every call the run
+makes, so `Router` and `ONNXAgent` mark answers whose `answer_confidence` falls below `T` with
+`low_confidence: True` before the harness sees them. Unlike grouping, this changes the answers
+that score: the same run at `T=0` and `T=0.7` is a different experiment, and a `precision@coverage`
+sweep is a series of these, not a single baseline drifting.
+
+The accepted range is core's `laya.confidence.check_min_confidence` -- `[0.0, 1.0]`, finite, not
+a bool -- rather than a copy here, so a value the gate itself would reject fails as a usage error
+(exit 2) before any checkpoint loads. `0.0` is a legal ask: it is the control arm for a
+`precision@coverage` sweep, and a check that dropped it would hide the sweep's own floor.
+
+A runner whose `predict` or (for a batched run) whose `predict_batch` predates the gate is
+**refused with a named `EvalError`**, not scored without the threshold. Silently dropping a
+scoring control is the class of lie this harness exists to prevent: the report would publish a
+`precision@coverage` figure for a policy that never ran. `config.timing` records both the ask and
+the fact: `min_confidence` is the threshold that was requested, `min_confidence_sent` says whether
+any call this run made actually carried it.
+
## Slices
`compare` and `run` report overall numbers and, for `--slice language|model|qid|tag`, the same
diff --git a/laya/evals.py b/laya/evals.py
index 8b780f62..f758b004 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -31,6 +31,8 @@ from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
import numpy as np
+from .confidence import check_min_confidence
+
#: The shape of the report this module writes, so a consumer can refuse one it cannot read.
#: ``research/evals/act_head_eval.py`` publishes a report under its own tag off this prefix.
REPORT_SCHEMA = "laya-evals-report/1"
@@ -547,9 +549,31 @@ def _takes_sort_by_length(runner: Any) -> bool:
return any(p.name == "sort_by_length" or p.kind is p.VAR_KEYWORD for p in params)
+def _takes_min_confidence(runner: Any, fn_name: str = "predict_batch") -> bool:
+ """Whether `runner`'s entry point accepts an abstention threshold on the call.
+
+ The same signature check `_takes_sort_by_length` makes, applied to whichever entry point the
+ harness is about to call: `min_confidence` changes the answer (an abstention overwrites a
+ low-confidence choice), so it is a scoring control, not an optimisation. A runner that predates
+ the gate (#361) must still be scoreable -- silently dropping the threshold and reporting the
+ same run would give a `precision@coverage` number for a policy that never ran -- so when the
+ guard is false the harness raises rather than lies. The CLI catches the raise into a
+ pre-flight message before any checkpoint loads.
+ """
+ fn = getattr(runner, fn_name, None)
+ if fn is None:
+ return False
+ try:
+ params = inspect.signature(fn).parameters.values()
+ except (TypeError, ValueError):
+ return False
+ return any(p.name == "min_confidence" or p.kind is p.VAR_KEYWORD for p in params)
+
+
def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluator]] = None,
batch_size: Optional[int] = None, on_error: str = "fail",
- config: Optional[Dict[str, Any]] = None, sort_by_length: bool = False) -> EvalReport:
+ config: Optional[Dict[str, Any]] = None, sort_by_length: bool = False,
+ min_confidence: Optional[float] = None) -> EvalReport:
"""Run `runner` over `dataset`, aggregating per-answer metrics overall and per slice.
`runner` needs a ``predict(state, questions, model=...)`` method, and for `batch_size` above 1
@@ -565,6 +589,14 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
runner that predates the knob is unaffected by a run that does not ask. Nothing about the
scored answers changes -- the results come back in chunk order either way.
+ `min_confidence` is the abstention threshold `Router` and `ONNXAgent` apply to
+ `answer_confidence` (#361): answers below it come back abstained, so the run scores the
+ policy at that threshold, not the raw argmax. Unlike `sort_by_length` this changes the
+ answers, so a runner whose batch entry point (or whose single ``predict``, on the fallback
+ path) predates the gate is refused rather than silently scored without it -- the report
+ would otherwise publish a `precision@coverage` figure for an abstention policy that never
+ ran.
+
`on_error` is ``"fail"`` (re-raise a runner error) or ``"skip"`` (record it and continue),
the latter for evaluating a flaky fleet without aborting the whole run.
@@ -578,6 +610,34 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
"""
if on_error not in ("fail", "skip"):
raise EvalError("on_error must be 'fail' or 'skip', got %r" % on_error)
+ # `min_confidence` is validated here -- and, unlike `sort_by_length`, a run that asks for it
+ # on a runner that does not accept it is refused -- because an abstention threshold changes
+ # which answers score as correct. Silently dropping it would publish a `precision@coverage`
+ # number for a policy that never ran, which is exactly the class of lie a baseline report is
+ # supposed to prevent. `laya.confidence.check_min_confidence` is the same validator the Router
+ # uses, so the accepted range cannot drift from what the gate itself enforces.
+ try:
+ mc = check_min_confidence(min_confidence) if min_confidence is not None else None
+ except ValueError as exc:
+ # Core's validator raises a bare ValueError; the CLI's usage-error handler catches EvalError
+ # and turns it into exit 2 with a printed message. Re-raising here keeps a mistyped
+ # `--min-confidence 1.5` on the same path as a bad tolerance, instead of a traceback.
+ raise EvalError(str(exc)) from exc
+ if mc is not None:
+ # Whichever entry points this run can actually reach all have to take the threshold. A
+ # batched run still sends a chunk of one through `predict`, so requiring only
+ # ``predict_batch`` would let the runner silently score those rows without the gate --
+ # the exact class of lie the guard exists to prevent.
+ targets = ["predict"]
+ if batch_size is not None and batch_size > 1 and _batch_form(runner):
+ targets.append("predict_batch")
+ for target in targets:
+ if not _takes_min_confidence(runner, target):
+ raise EvalError(
+ "evaluate(min_confidence=%r) refused: this runner's %s does not accept the "
+ "abstention threshold. Either use a Router/ONNXAgent that gates on "
+ "answer_confidence (#361), or drop the threshold -- the report would "
+ "otherwise score a policy that never ran." % (min_confidence, target))
evaluators = list(evaluators) if evaluators is not None else default_evaluators()
cases: List[Dict[str, Any]] = []
waits: List[float] = [] # what each request actually waited: its chunk's whole call
@@ -591,6 +651,13 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
# `predict`, which has no batch to reorder, and an off control is not sent at all.
shape = ({"sort_by_length": True}
if sort_by_length and batch_form and _takes_sort_by_length(runner) else {})
+ # `min_confidence` joins the batch shape whenever a chunk can go through `predict_batch`, and
+ # the single-`predict` path picks it up below. The pre-flight refusal above has already
+ # proven every entry point this run can reach accepts it, so no re-check sits here: a run
+ # that reaches this line either asked for the gate and will get it on every call, or asked
+ # for nothing.
+ if mc is not None and batch_form:
+ shape = dict(shape, min_confidence=mc)
index = 0
while index < len(examples):
@@ -631,7 +698,11 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
model=chunk[0].model, batch_size=batch_size,
**shape)
else:
- results = [runner.predict(chunk[0].state, chunk[0].questions, model=chunk[0].model)]
+ # A chunk of one still sees the gate: the pre-flight refusal proved `predict`
+ # accepts it whenever `mc` is set, so nothing here has to re-check the signature.
+ single_kwargs = {"min_confidence": mc} if mc is not None else {}
+ results = [runner.predict(chunk[0].state, chunk[0].questions,
+ model=chunk[0].model, **single_kwargs)]
except Exception as exc: # noqa: BLE001 -- honoured by on_error
if on_error == "fail":
raise
@@ -686,7 +757,18 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
"sort_by_length": sort_by_length,
# Requested and sent are different claims: `sort_by_length` needs a chunk to reorder, so a
# run with no batch form asked for something this harness cannot do.
- "sort_by_length_sent": bool(shape),
+ "sort_by_length_sent": bool(shape.get("sort_by_length")),
+ # Same shape for `min_confidence`: a run that asked for a threshold and landed on a
+ # chunked path where the runner accepted it gets a different report from one that was
+ # refused at the guard, and the report should say which. The `_sent` value reflects
+ # whether at least one call actually carried the threshold, not whether it was requested.
+ "min_confidence": min_confidence,
+ "min_confidence_sent": (
+ # The pre-flight refusal proved the entry points this run reaches accept the
+ # threshold, so a non-None `mc` on a run that made at least one call is exactly the
+ # claim "the threshold was on every call". An empty dataset sends nothing.
+ mc is not None and chunks > 0
+ ),
"chunks": chunks,
"rows_grouped": rows_grouped,
"rows_alone": rows_alone,
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index a58375d0..fcb4a033 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -27,15 +27,22 @@ class RouterRunner:
def __init__(self, router: Any):
self.router = router
- def predict(self, state: Any, questions: Dict[str, Any], model: Optional[str] = None) -> Dict[str, Any]:
+ def predict(self, state: Any, questions: Dict[str, Any], model: Optional[str] = None,
+ min_confidence: Optional[float] = None) -> Dict[str, Any]:
+ if min_confidence is not None:
+ return self.router.predict(state, questions, model=model,
+ min_confidence=min_confidence)
return self.router.predict(state, questions, model=model)
def predict_batch(self, states: Sequence[Any], questions: Dict[str, Any],
model: Optional[str] = None, batch_size: Optional[int] = None,
- sort_by_length: bool = False) -> List[Dict[str, Any]]:
+ sort_by_length: bool = False,
+ min_confidence: Optional[float] = None) -> List[Dict[str, Any]]:
requests = [{"state": state, "questions": questions, "model": model} for state in states]
- return self.router.predict_batch(requests, batch_size=batch_size,
- sort_by_length=sort_by_length)
+ kwargs = {"batch_size": batch_size, "sort_by_length": sort_by_length}
+ if min_confidence is not None:
+ kwargs["min_confidence"] = min_confidence
+ return self.router.predict_batch(requests, **kwargs)
class OnnxRunner:
@@ -56,23 +63,30 @@ class OnnxRunner:
"the ONNX runner serves only %r, but this example asks for %r; "
"run them separately or drop --onnx" % (self.agent.model_id, model))
- def predict(self, state: Any, questions: Dict[str, Any], model: Optional[str] = None) -> Dict[str, Any]:
+ def predict(self, state: Any, questions: Dict[str, Any], model: Optional[str] = None,
+ min_confidence: Optional[float] = None) -> Dict[str, Any]:
self._check_model(model)
+ if min_confidence is not None:
+ return self.agent.predict(state, questions, min_confidence=min_confidence)
return self.agent.predict(state, questions)
def predict_batch(self, states: Sequence[Any], questions: Dict[str, Any],
model: Optional[str] = None, batch_size: Optional[int] = None,
- sort_by_length: bool = False) -> List[Dict[str, Any]]:
+ sort_by_length: bool = False,
+ min_confidence: Optional[float] = None) -> List[Dict[str, Any]]:
self._check_model(model)
agent_batch = getattr(self.agent, "predict_batch", None)
+ extra = {}
+ if min_confidence is not None:
+ extra["min_confidence"] = min_confidence
if agent_batch is not None:
if sort_by_length:
# Asked for, so it has to reach the agent's own grouping. The per-state fallback
# below cannot honour it: there is no batch to reorder.
return agent_batch(list(states), questions, batch_size=batch_size,
- sort_by_length=True)
- return agent_batch(list(states), questions, batch_size=batch_size)
- return [self.agent.predict(state, questions) for state in states]
+ sort_by_length=True, **extra)
+ return agent_batch(list(states), questions, batch_size=batch_size, **extra)
+ return [self.agent.predict(state, questions, **extra) for state in states]
def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:
@@ -155,6 +169,13 @@ def _build_parser() -> argparse.ArgumentParser:
help="with --batch-size N where 1 < N < the run, group similarly sized "
"examples into the same forward pass so each pads to a shorter maximum; "
"scores the same answers, in the same order")
+ run.add_argument("--min-confidence", dest="min_confidence", type=float, metavar="THRESHOLD",
+ help="abstention threshold on `answer_confidence` (#361): answers below it "
+ "come back abstained, so the run scores the policy at that threshold "
+ "rather than the raw argmax. Accepted range is core's -- "
+ "`laya.confidence.check_min_confidence` -- not a copy of it here, and a "
+ "runner that predates the gate is refused with a named error rather "
+ "than silently scored without it")
run.add_argument("--on-error", choices=("fail", "skip"), default="fail")
run.add_argument("--baseline", help="a baseline report JSON to compare against")
run.add_argument("--tolerance", action="append", metavar="METRIC=VALUE",
@@ -301,7 +322,8 @@ def _cmd_run(args) -> int:
config["score_within"] = [evaluator.tolerance for evaluator in extra]
report = evals.evaluate(runner, dataset, evaluators=evals.default_evaluators() + extra,
batch_size=args.batch_size, on_error=args.on_error, config=config,
- sort_by_length=args.sort_by_length)
+ sort_by_length=args.sort_by_length,
+ min_confidence=args.min_confidence)
# Which commit answered belongs in the artifact a baseline is, and it can only be read after
# the run: `preload=False` means no checkpoint is resident before the first row.
# `loaded_revisions` reports the commit each resident agent came from -- the pin when there is
diff --git a/tests/test_evals.py b/tests/test_evals.py
index e9e359a6..7f4cb055 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -432,6 +432,207 @@ def test_a_pass_through_wrapper_is_given_the_knob():
assert report.config["timing"]["sort_by_length_sent"] is True
+# ------------------------------------------------------- abstention gate (#361 knob)
+#
+# `Router.predict` / `Router.predict_batch` and `ONNXAgent.predict` gate on `answer_confidence`
+# and mark answers below `min_confidence` with `low_confidence: True` (#361). That is a scoring
+# control: an answer below the threshold is not the same decision as one above. A `laya-evals run`
+# that could not pass it through had no way to measure `precision@coverage` at any threshold
+# except by wrapping a Router by hand -- the exact class of thing this harness exists to be.
+
+UNSET = object() # no caller sends it, so `mc is UNSET` distinguishes "absent" from 0.0 or None
+
+
+class GatedRunner(RequestsRunner):
+ """A `Router`-shaped runner that has the gate and records exactly what each call carried."""
+
+ def __init__(self, by_state):
+ super().__init__(by_state)
+ self.thresholds = []
+
+ def predict(self, state, questions, model=None, min_confidence=UNSET):
+ self.thresholds.append({"path": "predict", "min_confidence": min_confidence})
+ return {"model": model or "m", "answers": self.by_state[state]}
+
+ def predict_batch(self, requests, batch_size=None, min_confidence=UNSET):
+ self.thresholds.append({"path": "predict_batch", "min_confidence": min_confidence})
+ return [{"model": r.get("model") or "m", "answers": self.by_state[r["state"]]}
+ for r in requests]
+
+
+class GatedStatesRunner(StubRunner):
+ """The positional batch shape, same gate + same recording."""
+
+ def __init__(self, by_state):
+ super().__init__(by_state)
+ self.thresholds = []
+
+ def predict(self, state, questions, model=None, min_confidence=UNSET):
+ self.thresholds.append({"path": "predict", "min_confidence": min_confidence})
+ return {"model": model or "m", "answers": self.by_state[state]}
+
+ def predict_batch(self, states, questions, model=None, batch_size=None,
+ min_confidence=UNSET):
+ self.thresholds.append({"path": "predict_batch", "min_confidence": min_confidence})
+ return [{"model": model or "m", "answers": self.by_state[s]} for s in states]
+
+
+def test_min_confidence_reaches_a_requests_shaped_runner_when_asked():
+ runner = GatedRunner(ANSWERS3)
+ report = evaluate(runner, _three(), evaluators=[ChoiceAccuracy()], batch_size=2,
+ min_confidence=0.7)
+ assert runner.thresholds[0] == {"path": "predict_batch", "min_confidence": 0.7}
+ assert report.config["timing"]["min_confidence"] == 0.7
+ assert report.config["timing"]["min_confidence_sent"] is True
+
+
+def test_min_confidence_reaches_a_states_shaped_runner_too():
+ runner = GatedStatesRunner(ANSWERS3)
+ report = evaluate(runner, _three(), evaluators=[ChoiceAccuracy()], batch_size=2,
+ min_confidence=0.7)
+ # The first two rows share a batch; the third stands alone because batch_size=2.
+ assert runner.thresholds[0] == {"path": "predict_batch", "min_confidence": 0.7}
+ assert runner.thresholds[1] == {"path": "predict", "min_confidence": 0.7}
+ assert report.config["timing"]["min_confidence_sent"] is True
+
+
+def test_min_confidence_reaches_the_single_predict_fallback():
+ """A run with no `batch_size` still has to see the gate; the abstention path is per-decision."""
+ runner = GatedRunner(ANSWERS3)
+ report = evaluate(runner, _three(), evaluators=[ChoiceAccuracy()], min_confidence=0.7)
+ assert all(t["path"] == "predict" and t["min_confidence"] == 0.7 for t in runner.thresholds)
+ assert report.config["timing"]["min_confidence_sent"] is True
+
+
+def test_min_confidence_is_not_invented_for_a_run_that_did_not_ask():
+ """The kwarg is dropped when unset, so an older runner sees the call it always saw."""
+ runner = GatedRunner(ANSWERS3)
+ evaluate(runner, _three(), evaluators=[ChoiceAccuracy()], batch_size=2)
+ # chunk 1 shares a batch (rows s1, s2), row s3 falls to `predict`. Neither sees the threshold.
+ assert runner.thresholds == [{"path": "predict_batch", "min_confidence": UNSET},
+ {"path": "predict", "min_confidence": UNSET}]
+
+
+def test_min_confidence_zero_is_still_a_threshold():
+ """`if min_confidence:` would drop 0.0; a gate at zero is a legal, meaningful ask."""
+ runner = GatedRunner(ANSWERS3)
+ evaluate(runner, _three(), evaluators=[ChoiceAccuracy()], batch_size=2, min_confidence=0.0)
+ assert runner.thresholds[0] == {"path": "predict_batch", "min_confidence": 0.0}
+
+
+def test_a_runner_without_the_gate_is_refused_not_silently_scored():
+ """Silently dropping a scoring control would report `precision@coverage` for a policy that
+ never ran. `RequestsRunner`'s `predict_batch(requests, batch_size=None)` is exactly what a
+ pre-#361 runner looks like, so a run that asks for a threshold on it must fail loudly."""
+ with pytest.raises(EvalError) as exc:
+ evaluate(RequestsRunner(ANSWERS3), _three(), evaluators=[ChoiceAccuracy()],
+ batch_size=2, min_confidence=0.7)
+ assert "min_confidence" in str(exc.value)
+ assert "does not accept the abstention threshold" in str(exc.value)
+
+
+def test_the_single_predict_path_refuses_too_when_only_predict_lacks_the_gate():
+ """Batching on a gate-aware `predict_batch` does not excuse a gate-blind `predict`: a chunk
+ of one still goes through that entry point, and would be scored without the threshold."""
+ class OnlyBatchGated(GatedRunner):
+ def predict(self, state, questions, model=None): # predates #361
+ return {"model": model or "m", "answers": self.by_state[state]}
+
+ # batch_size=2 on a three-row dataset leaves the third row to `predict`; the guard sees that
+ # before issuing anything.
+ with pytest.raises(EvalError) as exc:
+ evaluate(OnlyBatchGated(ANSWERS3), _three(), evaluators=[ChoiceAccuracy()],
+ batch_size=2, min_confidence=0.7)
+ assert "predict" in str(exc.value)
+
+
+def test_min_confidence_validated_by_core():
+ """The accepted range is core's validator, so the harness cannot drift from the gate itself."""
+ for bad in (1.5, -0.1, "high", True, float("nan")):
+ with pytest.raises(EvalError) as exc:
+ evaluate(GatedRunner(ANSWERS3), _three(), evaluators=[ChoiceAccuracy()],
+ min_confidence=bad)
+ assert "min_confidence must be a float in [0.0, 1.0]" in str(exc.value)
+
+
+def test_min_confidence_changes_the_call_not_the_score_of_a_stub():
+ """The stub ignores the threshold, so the answers match. This is the parity check on the
+ control-flow side -- the answer-shape witness is `flag_low_confidence`'s own suite (#361)."""
+ asked = GatedRunner(ANSWERS3)
+ gated = evaluate(asked, _three(), evaluators=[ChoiceAccuracy()], batch_size=2,
+ min_confidence=0.7)
+ plain = GatedRunner(ANSWERS3)
+ ungated = evaluate(plain, _three(), evaluators=[ChoiceAccuracy()], batch_size=2)
+ assert _labels(gated) == _labels(ungated)
+ # The calls really differed: one carried the threshold, one did not.
+ assert asked.thresholds != plain.thresholds
+
+
+def test_min_confidence_and_sort_by_length_are_independent_controls():
+ """A run can ask for both: the sort is dropped for a runner that predates #294 while the
+ gate is honoured for the same runner's #361 support, and vice versa."""
+ class OnlyGated(GatedRunner):
+ def predict_batch(self, requests, batch_size=None, min_confidence=UNSET):
+ # No `sort_by_length` in the signature, but the threshold is real.
+ self.thresholds.append({"path": "predict_batch", "min_confidence": min_confidence})
+ return [{"model": r.get("model") or "m", "answers": self.by_state[r["state"]]}
+ for r in requests]
+
+ runner = OnlyGated(ANSWERS3)
+ report = evaluate(runner, _three(), evaluators=[ChoiceAccuracy()], batch_size=2,
+ sort_by_length=True, min_confidence=0.7)
+ assert report.config["timing"]["sort_by_length_sent"] is False
+ assert report.config["timing"]["min_confidence_sent"] is True
+
+
+def test_cli_min_confidence_forwards_through_evaluate(monkeypatch, tmp_path):
+ """The flag has to actually reach `evaluate`, not sit in the parser."""
+ from laya import evals_cli
+
+ seen = {}
+ real_evaluate = evals_cli.evals.evaluate
+
+ def spy(runner, dataset, **kwargs):
+ seen.update(kwargs)
+ return real_evaluate(runner, dataset, **kwargs)
+
+ monkeypatch.setattr(evals_cli.evals, "evaluate", spy)
+ _fake_router(monkeypatch, {})
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--min-confidence", "0.7"]) == 0
+ assert seen["min_confidence"] == 0.7
+
+
+def test_cli_min_confidence_out_of_range_is_a_usage_error(monkeypatch, tmp_path, capsys):
+ """A mistyped `--min-confidence 1.5` must exit 2 with core's message, not traceback."""
+ from laya import evals_cli
+
+ _fake_router(monkeypatch, {})
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--min-confidence", "1.5"]) == 2
+ err = capsys.readouterr().err
+ assert "min_confidence must be a float" in err
+ assert "Traceback" not in err
+
+
+def test_cli_min_confidence_zero_is_accepted(monkeypatch, tmp_path):
+ """A gate at zero is a legal ask; `if args.min_confidence` would drop it before the call."""
+ from laya import evals_cli
+
+ seen = {}
+ real_evaluate = evals_cli.evals.evaluate
+
+ def spy(runner, dataset, **kwargs):
+ seen.update(kwargs)
+ return real_evaluate(runner, dataset, **kwargs)
+
+ monkeypatch.setattr(evals_cli.evals, "evaluate", spy)
+ _fake_router(monkeypatch, {})
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--min-confidence", "0.0"]) == 0
+ assert seen["min_confidence"] == 0.0
+
+
# --------------------------------------------------------------- timing (#585)
FORWARD_MS = 100.0
SHARED = 0.6 # a batch of n costs SHARED * n * FORWARD_MS, as one call
@@ -1360,9 +1561,10 @@ def test_docs_and_the_cli_name_the_same_flags():
assert taught <= registered, "the page teaches %s, which no subcommand registers" % sorted(
taught - registered)
# The other half of parity: a registered flag nobody documents is unreachable in practice. The
- # page that teaches `--batch-size` has to teach the grouping that makes a bounded pass cheaper.
- assert {"--batch-size", "--sort-by-length"} <= taught, \
- "the evals page teaches the batch size but not the grouping knob"
+ # page that teaches `--batch-size` has to teach the grouping that makes a bounded pass cheaper,
+ # and the abstention threshold that changes which answers score at all.
+ assert {"--batch-size", "--sort-by-length", "--min-confidence"} <= taught, \
+ "the evals page teaches the batch size but not the grouping or abstention knobs"
assert "--score-within" in quickstart, "the tolerance metric has to be reachable from the quickstart"
metrics = page.split("## Metrics", 1)[1].split("\n## ", 1)[0]
assert "score_within" in metrics and "--score-within" in metrics, \
@@ -1370,6 +1572,10 @@ def test_docs_and_the_cli_name_the_same_flags():
grouping = page.split("### Grouping the rows inside a batch", 1)[1].split("\n#", 1)[0]
assert "--sort-by-length" in grouping and "sort_by_length_sent" in grouping, \
"the subsection that explains the grouping has to name the flag and what it reports"
+ gate = page.split("### The abstention gate at a threshold", 1)[1].split("\n#", 1)[0]
+ assert "--min-confidence" in gate and "min_confidence_sent" in gate, \
+ "the subsection that explains the gate has to name the flag and what it reports"
+ assert "refused" in gate, "the docs have to teach that a gate-blind runner is refused, not silently scored"
# --------------------------------------------------------------- --revision pinning
@@ -1390,7 +1596,10 @@ def _fake_router(monkeypatch, recorded):
recorded.update(device=device, preload=preload, revision=revision,
revisions={normalise_name(k): v for k, v in (revisions or {}).items()})
- def predict(self, state, questions, model=None):
+ def predict(self, state, questions, model=None, min_confidence=None):
+ # The gate is a real part of the Router interface since #361; a stub without it would
+ # turn a `--min-confidence` run into a TypeError and hide whether the CLI really
+ # forwarded, rather than only parsing, the flag.
return {"model": model or "english", "answers": {"intent": choice_answer("a")}}
loaded_revisions = {"english": SHA}
diff --git a/tests/test_evals_api.py b/tests/test_evals_api.py
index 26742062..b65c0a4f 100644
--- a/tests/test_evals_api.py
+++ b/tests/test_evals_api.py
@@ -55,16 +55,28 @@ check("EvalReport fields are unchanged", [f.name for f in dataclasses.fields(eva
# names and order are the contract. `sort_by_length` is core's #294 forward-pass grouping knob
# reaching a scored run through `laya-evals run --sort-by-length`; it is appended, never inserted,
# because a caller that passed `on_error` or `config` positionally must still mean the same thing.
+# `min_confidence` extends the same rule: it is appended after `sort_by_length`, defaults to None
+# (an unset abstention threshold means "score the raw argmax", which is what a pre-#361 run did),
+# and sits after the batch size the calls that carry it can group inside.
_sig = inspect.signature(evals.evaluate)
check("evaluate signature", [p.name for p in _sig.parameters.values()],
- ["runner", "dataset", "evaluators", "batch_size", "on_error", "config", "sort_by_length"])
+ ["runner", "dataset", "evaluators", "batch_size", "on_error", "config",
+ "sort_by_length", "min_confidence"])
check("evaluate: grouping defaults to off", _sig.parameters["sort_by_length"].default, False)
+check("evaluate: abstention threshold defaults to unset",
+ _sig.parameters["min_confidence"].default, None)
check_true("evaluate: the knob sits after the batch size it groups inside",
list(_sig.parameters).index("batch_size")
< list(_sig.parameters).index("sort_by_length"))
+check_true("evaluate: the abstention knob sits after the batch size it may apply within",
+ list(_sig.parameters).index("batch_size")
+ < list(_sig.parameters).index("min_confidence"))
# The two batch entry points the CLI wires up: both have to take the knob by the same name, or a
-# `--sort-by-length` run reports `sort_by_length_sent: false` for the surface it ships.
+# `--sort-by-length` run reports `sort_by_length_sent: false` for the surface it ships. The same
+# is true for the abstention threshold; a `--min-confidence` run that silently dropped the
+# argument would publish a `precision@coverage` figure for a policy that never ran, so the CLI
+# runner shapes have to accept the kwarg under the same name the guard on `evaluate` checks.
from laya import evals_cli # noqa: E402
for label, fn in (("RouterRunner.predict_batch", evals_cli.RouterRunner.predict_batch),
@@ -72,6 +84,18 @@ for label, fn in (("RouterRunner.predict_batch", evals_cli.RouterRunner.predict_
params = inspect.signature(fn).parameters
check_true("%s takes the grouping knob" % label, "sort_by_length" in params)
check("%s defaults it to off" % label, params["sort_by_length"].default, False)
+ check_true("%s takes the abstention threshold" % label, "min_confidence" in params)
+ check("%s defaults the threshold to unset" % label,
+ params["min_confidence"].default, None)
+
+# The single-predict fallback path takes the threshold under the same name on both runner shapes,
+# so a run without --batch-size still scores the gate rather than silently skipping it.
+for label, fn in (("RouterRunner.predict", evals_cli.RouterRunner.predict),
+ ("OnnxRunner.predict", evals_cli.OnnxRunner.predict)):
+ params = inspect.signature(fn).parameters
+ check_true("%s takes the abstention threshold" % label, "min_confidence" in params)
+ check("%s defaults the threshold to unset" % label,
+ params["min_confidence"].default, None)
# A signature is not a wiring: a runner that declares the knob and drops it on the way through
# would pass both checks above. `OnnxRunner`'s call is gated in test_evals_onnx.py against a
@@ -80,8 +104,10 @@ class _RecordingRouter:
def __init__(self):
self.calls = []
- def predict_batch(self, requests, batch_size=None, sort_by_length=False):
- self.calls.append({"batch_size": batch_size, "sort_by_length": sort_by_length})
+ def predict_batch(self, requests, batch_size=None, sort_by_length=False,
+ min_confidence=None):
+ self.calls.append({"batch_size": batch_size, "sort_by_length": sort_by_length,
+ "min_confidence": min_confidence})
return [{"model": "m", "answers": {}} for _ in requests]