Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(calibrate): fit abstention cuts on the 4-decimal confidences the gate reads

sameedkhan17ac2b30d8a623
diff --git a/laya/calibrate.py b/laya/calibrate.py
index 868d7b03..d7582ece 100644
--- a/laya/calibrate.py
+++ b/laya/calibrate.py
@@ -382,6 +382,10 @@ def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],
# The runtime bins before anyone reads `answer_confidence` (`Agent._decode_answers`),
# so the cut has to be chosen on the binned scale or it gates a different quantity.
conf = apply_binning_map(conf, bucket, binning_map)
+ # `_decode_answers` reports `answer_confidence` rounded to 4 decimals, after binning, and
+ # the gate compares that. A cut picked at full precision can sit above the rounded value of
+ # its own cohort (bin 5/6 is reported as 0.8333 < 0.83333...), which abstains all of it.
+ conf = round(conf, 4)
by_bucket.setdefault(bucket, []).append((conf, int(int(p.argmax()) == y)))
out: Dict[str, float] = {}
for key, pairs in by_bucket.items():
diff --git a/tests/test_conformal_abstention.py b/tests/test_conformal_abstention.py
index 3d4b8ed2..baae4592 100644
--- a/tests/test_conformal_abstention.py
+++ b/tests/test_conformal_abstention.py
@@ -203,6 +203,55 @@ check_true("binning/the two cuts genuinely differ", abs(_cut_binned - _cut_raw)
check("binning/no map leaves the fit unchanged",
fit_abstention_thresholds(_binning_records, _temps, _tbo, binning_map=None,
target_error=0.10)["choice:11+"], _cut_raw)
+
+
+# --------------------------------------------------------------- thresholds vs 4-decimal reporting
+# `_decode_answers` reports `answer_confidence` rounded to 4 decimals and the gate compares that, so
+# a cut chosen at full precision can sit above every value the runtime reports for its own cohort:
+# seed 2's cut is the bin value 5/6 = 0.83333..., the runtime reports that bin as 0.8333, and the
+# whole bin abstains. Gated through the real decoder, the fitted cut must keep the coverage and
+# error the fitter judged it on.
+from laya.agent import Agent # noqa: E402
+
+_rng = np.random.default_rng(2)
+_round_records = []
+for _ in range(1200):
+ _true = int(_rng.integers(12))
+ _z = _rng.normal(0.0, 1.0, 12)
+ _z[_true] += 2.2
+ _t = [0.0] * 12
+ _t[_true] = 1.0
+ _round_records.append((0, _z.tolist(), _t, 12))
+
+_round_bmap = fit_binning_map(_round_records, _temps, _tbo)
+_round_cut = fit_abstention_thresholds(_round_records, _temps, _tbo, binning_map=_round_bmap,
+ target_error=0.10)["choice:11+"]
+
+_dec = Agent.__new__(Agent)
+_dec.temperature, _dec.temperature_by_options, _dec.lang_temperatures = list(_temps), {}, {}
+_dec.binning_map = _round_bmap
+_round_ids = ["q%d" % i for i in range(len(_round_records))]
+_q = Agent._to_internal({"type": "choice", "instructions": "?", "criteria": ["o%d" % i for i in range(12)]})
+_decoded = _dec._decode_answers(np.asarray([z for _, z, _, _ in _round_records], dtype=np.float64),
+ np.zeros((len(_round_records), 2)), [{"markers": list(range(12))}] * len(_round_ids),
+ _round_ids, {qid: _q for qid in _round_ids}, 0)
+_round_results = [{"answers": _decoded}]
+apply_confidence_gate(_round_results, {"choice:11+": _round_cut})
+
+_fit_acc = _gate_acc = _gate_wrong = 0
+for qid, (_qt, z, t, k) in zip(_round_ids, _round_records):
+ p = np.exp(np.asarray(z) - max(z))
+ p = p / p.sum()
+ _fit_acc += apply_binning_map(float(p.max()), "choice:11+", _round_bmap) >= _round_cut
+ if _decoded[qid]["abstention"] == GATE_PASSED:
+ _gate_acc += 1
+ _gate_wrong += int(int(p.argmax()) != int(np.argmax(t)))
+
+check_true("rounding/the cut is a value the runtime can report", round(_round_cut, 4) == _round_cut,
+ "cut=%r" % (_round_cut,))
+check("rounding/the gate keeps every answer of the cut bin", _gate_acc, _fit_acc)
+check_true("rounding/the gate holds the target error", _gate_acc > 0 and _gate_wrong / _gate_acc <= 0.10,
+ "accepted=%d wrong=%d" % (_gate_acc, _gate_wrong))
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)