Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(calibrate): keep a ragged pool's fit from vanishing into 1.0
aashish97bb3a605b57
diff --git a/laya/calibrate.py b/laya/calibrate.pyindex bbed428c..b24b871f 100644--- a/laya/calibrate.py+++ b/laya/calibrate.py@@ -39,6 +39,12 @@ MIN_BUCKET_N = 2000MIN_TYPE_N = 10# Fraction of each bucket held out when `compute_ece=True`. Unused otherwise.ECE_HOLDOUT_FRAC = 0.2+# Log-space search margin, in e-folds, outside the `[TEMP_MIN, TEMP_MAX]` answer band.+# The optimiser runs on log T with no bound of its own; any scale this far below+# `TEMP_MIN` already clamps to `TEMP_MIN`, so the band costs no real minimum, while an+# unbounded walk reaches `exp()` underflow (~-88 in float32), divides the objective by+# zero, and hands `clamp_temperature` a NaN -- which it reports as the neutral 1.0.+LOG_T_MARGIN = 10.0# Payload schema. Files written before identity was recorded have no version key;# readers treat that absence as version 1.CALIBRATION_VERSION = 2@@ -85,17 +91,21 @@ def _validated_pair(logits, target, k=None):return logits, target, int(k)-def _pairs_to_tensors(pairs: Sequence) -> Tuple[torch.Tensor, torch.Tensor]:- kmax = max(len(_vec(z)) for z, _ in pairs)- z_mat = torch.full((len(pairs), kmax), -1e4)- t_mat = torch.zeros((len(pairs), kmax))- for i, (z, t) in enumerate(pairs):- z = _vec(z)- t = _vec(t)- n = len(z)- z_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(z[:n]))- t_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(t[:n]))- return z_mat, t_mat+def _pairs_by_width(pairs: Sequence) -> List[Tuple[torch.Tensor, torch.Tensor]]:+ """One (logits, target) matrix per distinct option count, narrowest first.++ A ragged pool -- a question type holding 2-option and 12-option decisions -- cannot be+ one matrix. Any value used to square off the short rows is a real column in the single+ softmax every row then goes through, so it competes for the probability mass the fit is+ trying to allocate; at `-1e4` it wins the row it pads once the row's own logits drop+ below it, and it overflows to `inf`/`NaN` in float32 when the scale gets small. Grouping+ keeps each row inside its own option count, which is what the objective means.+ """+ groups: Dict[int, List[Tuple[np.ndarray, np.ndarray]]] = {}+ for z, t in pairs:+ groups.setdefault(len(_vec(z)), []).append((_vec(z), _vec(t)))+ return [(torch.from_numpy(np.stack([z for z, _ in groups[w]])),+ torch.from_numpy(np.stack([t for _t, t in groups[w]]))) for w in sorted(groups)]def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:@@ -103,6 +113,10 @@ def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:The result is `clamp_temperature` of the optimised scale, so it lies in`[TEMP_MIN, TEMP_MAX]` (or is the neutral 1.0 when the value is not a number).+ The search is bounded to `LOG_T_MARGIN` e-folds outside that band, which covers any+ minimum the answer can use; past it the float32 scale drifts toward `exp()` underflow,+ where the objective divides by zero and every gradient becomes NaN, and+ `clamp_temperature` reads that NaN back as the neutral 1.0.Returns 1.0 when fewer than `min_n` pairs are given. `min_n` defaults to`MIN_BUCKET_N` (the per-bucket floor). Type-level fits pass `MIN_TYPE_N`, whichis lower, so a dataset that fills no bucket still gets a scalar instead of@@ -122,19 +136,25 @@ def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:sel.append((z, t))if len(sel) < min_n:return 1.0- z_mat, t_mat = _pairs_to_tensors(sel)+ groups = _pairs_by_width(sel)+ n_rows = float(len(sel))+ log_lo = float(np.log(TEMP_MIN)) - LOG_T_MARGIN+ log_hi = float(np.log(TEMP_MAX)) + LOG_T_MARGINwith torch.enable_grad():log_t = torch.zeros(1, requires_grad=True)opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)def closure():opt.zero_grad()- loss = -(t_mat * torch.log_softmax(z_mat / log_t.exp(), -1)).sum(-1).mean()+ scale = log_t.clamp(min=log_lo, max=log_hi).exp()+ nll = sum(-(t * torch.log_softmax(z / scale, -1)).sum(-1).sum()+ for z, t in groups)+ loss = nll / n_rowsloss.backward()return lossopt.step(closure)- fitted = float(log_t.detach().exp().item())+ fitted = float(log_t.detach().clamp(min=log_lo, max=log_hi).exp().item())return clamp_temperature(fitted, TEMP_MIN, TEMP_MAX)diff --git a/tests/test_calibrate.py b/tests/test_calibrate.pyindex d93f0a92..3b0e100c 100644--- a/tests/test_calibrate.py+++ b/tests/test_calibrate.py@@ -23,12 +23,14 @@ from laya.agent import Agent, load # noqa: E402from laya.calibrate import ( # noqa: E402CALIBRATION_VERSION,ECE_HOLDOUT_FRAC,+ LOG_T_MARGIN,MIN_BUCKET_N,MIN_TYPE_N,MIN_BINNING_BUCKET_N,_ece_report,_ece_split,_iter_records,+ _pairs_by_width,_softmax,apply_binning_map,apply_calibration_payload,@@ -177,6 +179,107 @@ check_true()+# --------------------------------------------------- ragged pools, pad columns, runaway scales+# A type-level fit pools every option count that type has seen, so `fit_one_temperature`+# regularly receives rows of different lengths. Two things used to cost such a pool its+# answer, and both surface the same way -- the neutral 1.0, or a bound, instead of a fit:+# 1. squaring off the short rows put the pad value in the same softmax those rows run+# through, so a row whose own logits sat under it stopped being about that row;+# 2. the search runs on log T with no bound, and a pool whose optimum is below+# `TEMP_MIN` walks `log_t` past float32's `exp()` underflow, divides the objective+# by zero, and hands `clamp_temperature` a NaN, which it reports as 1.0.+def _spread_pool(width, scale, n, seed=0):+ """`n` argmax-correct rows at one width, re-seeded per width."""+ rs = np.random.RandomState(seed)+ out = []+ for _ in range(n):+ z = list(rs.randn(width) * scale)+ t = [0.0] * width+ t[int(np.argmax(z))] = 1.0+ out.append((z, t))+ return out+++def _mixed_rows(k, rng, acc, mag):+ """`k`-option rows that are right with probability `acc`, logits at +-mag."""+ out = []+ for _ in range(400):+ y = int(rng.randint(0, k))+ pred = y if rng.rand() < acc else int(rng.randint(0, k))+ z, t = peaked(k, pred, mag)+ t = np.zeros(k, dtype=np.float32)+ t[y] = 1.0+ out.append((z, t))+ return out+++two, nine = _spread_pool(2, 1.0, 24), _spread_pool(9, 1.0, 24)+t_two = fit_one_temperature(two, min_n=10)+t_nine = fit_one_temperature(nine, min_n=10)+t_ragged = fit_one_temperature(two + nine, min_n=10)+check_true("runaway/every width on its own fits the floor", (t_two, t_nine) == (0.5, 0.5),+ "two=%r nine=%r" % (t_two, t_nine))+check_true("runaway/ragged pool is a number", bool(np.isfinite(t_ragged)), "T=%r" % t_ragged)+check_true("runaway/ragged pool is not the neutral 1.0", t_ragged != 1.0, "T=%r" % t_ragged)+check_true("runaway/ragged pool keeps the floor", abs(t_ragged - TEMP_MIN) < 1e-9,+ "T=%r" % t_ragged)++# The same runaway at one width: only the walk to zero was wrong, not the raggedness.+t_soft = fit_one_temperature(_mixed_rows(3, np.random.RandomState(51), 0.7, 0.3), min_n=10)+check_true("runaway/low-spread pool is not the neutral 1.0", t_soft != 1.0, "T=%r" % t_soft)+check_true("runaway/low-spread pool keeps the floor", abs(t_soft - TEMP_MIN) < 1e-9,+ "T=%r" % t_soft)++# Rows whose every logit sits under the pad value: pooled, they used to report the ceiling.+low_rows = [([-20000.0 - i, -20001.0 - i], [1.0, 0.0]) for i in range(12)]+t_low_alone = fit_one_temperature(low_rows, min_n=10)+t_low_pool = fit_one_temperature(low_rows + _spread_pool(4, 1.0, 12), min_n=10)+check_true("pad/low rows alone fit the floor", abs(t_low_alone - TEMP_MIN) < 1e-9,+ "T=%r" % t_low_alone)+check("pad/low rows pooled fit what they fit alone", t_low_pool, t_low_alone)+check_true("pad/pool is not the ceiling", t_low_pool != TEMP_MAX, "T=%r" % t_low_pool)+check("pad/one matrix per width, narrowest first",+ [int(z.shape[1]) for z, _ in _pairs_by_width(low_rows + _spread_pool(4, 1.0, 12))], [2, 4])+check("pad/row totals survive the split",+ sum(int(z.shape[0]) for z, _ in _pairs_by_width(low_rows + _spread_pool(4, 1.0, 12))), 24)++# What a question type stores through the public API for the same rows.+ragged_recs = ([(1, z, t, len(z)) for z, t in low_rows * 34]+ + [(1, z, t, len(z)) for z, t in _spread_pool(5, 1.0, 400, 1)])+t_public = fit_temperature_map(ragged_recs)["temperature"][1]+check_true("public/ragged type keeps the floor", abs(t_public - TEMP_MIN) < 1e-9,+ "T=%r" % t_public)+check_true("public/ragged type is neither neutral nor the ceiling",+ t_public not in (1.0, TEMP_MAX), "T=%r" % t_public)++# Bounding the search must not flatten a minimum that sits inside the band: the fit is+# checked against the objective it is supposed to minimise, in float64, not against a+# hard-coded number.+mid = _mixed_rows(20, np.random.RandomState(293), 0.7, 1.0)+t_mid = fit_one_temperature(mid, min_n=10)+++def _nll_at(rows, scale):+ z = np.asarray([r[0] for r in rows], dtype=np.float64) / float(scale)+ t = np.asarray([r[1] for r in rows], dtype=np.float64)+ z = z - z.max(1, keepdims=True)+ p = np.exp(z)+ p = p / p.sum(1, keepdims=True)+ return float(-(t * np.log(np.clip(p, 1e-300, None))).sum(1).mean())+++check_true("band/interior fit stays interior", TEMP_MIN < t_mid < TEMP_MAX, "T=%r" % t_mid)+check_true("band/interior fit beats the floor", _nll_at(mid, t_mid) < _nll_at(mid, TEMP_MIN),+ "T=%r nll=%r floor=%r" % (t_mid, _nll_at(mid, t_mid), _nll_at(mid, TEMP_MIN)))+check_true("band/interior fit beats either side of itself",+ _nll_at(mid, t_mid) < min(_nll_at(mid, t_mid * 1.1), _nll_at(mid, t_mid / 1.1)))+src = inspect.getsource(fit_one_temperature)+check_true("band/search is bounded by LOG_T_MARGIN", "LOG_T_MARGIN" in src)+check_true("band/the bound stops before float32 exp() underflows",+ float(np.log(TEMP_MIN)) - LOG_T_MARGIN > -80.0,+ "lower edge=%r" % (float(np.log(TEMP_MIN)) - LOG_T_MARGIN))++# --------------------------------------------------------------- ECE on a held-out split, not the fit rowsreport = fitted["report"]check_true("ece/after < before", report["ece_after"] < report["ece_before"],