Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(calibrate): keep a ragged pool's fit from vanishing into 1.0

aashish97bb3a605b57
diff --git a/laya/calibrate.py b/laya/calibrate.py
index bbed428c..b24b871f 100644
--- a/laya/calibrate.py
+++ b/laya/calibrate.py
@@ -39,6 +39,12 @@ MIN_BUCKET_N = 2000
MIN_TYPE_N = 10
# Fraction of each bucket held out when `compute_ece=True`. Unused otherwise.
ECE_HOLDOUT_FRAC = 0.2
+# Log-space search margin, in e-folds, outside the `[TEMP_MIN, TEMP_MAX]` answer band.
+# The optimiser runs on log T with no bound of its own; any scale this far below
+# `TEMP_MIN` already clamps to `TEMP_MIN`, so the band costs no real minimum, while an
+# unbounded walk reaches `exp()` underflow (~-88 in float32), divides the objective by
+# zero, and hands `clamp_temperature` a NaN -- which it reports as the neutral 1.0.
+LOG_T_MARGIN = 10.0
# Payload schema. Files written before identity was recorded have no version key;
# readers treat that absence as version 1.
CALIBRATION_VERSION = 2
@@ -85,17 +91,21 @@ def _validated_pair(logits, target, k=None):
return logits, target, int(k)
-def _pairs_to_tensors(pairs: Sequence) -> Tuple[torch.Tensor, torch.Tensor]:
- kmax = max(len(_vec(z)) for z, _ in pairs)
- z_mat = torch.full((len(pairs), kmax), -1e4)
- t_mat = torch.zeros((len(pairs), kmax))
- for i, (z, t) in enumerate(pairs):
- z = _vec(z)
- t = _vec(t)
- n = len(z)
- z_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(z[:n]))
- t_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(t[:n]))
- return z_mat, t_mat
+def _pairs_by_width(pairs: Sequence) -> List[Tuple[torch.Tensor, torch.Tensor]]:
+ """One (logits, target) matrix per distinct option count, narrowest first.
+
+ A ragged pool -- a question type holding 2-option and 12-option decisions -- cannot be
+ one matrix. Any value used to square off the short rows is a real column in the single
+ softmax every row then goes through, so it competes for the probability mass the fit is
+ trying to allocate; at `-1e4` it wins the row it pads once the row's own logits drop
+ below it, and it overflows to `inf`/`NaN` in float32 when the scale gets small. Grouping
+ keeps each row inside its own option count, which is what the objective means.
+ """
+ groups: Dict[int, List[Tuple[np.ndarray, np.ndarray]]] = {}
+ for z, t in pairs:
+ groups.setdefault(len(_vec(z)), []).append((_vec(z), _vec(t)))
+ return [(torch.from_numpy(np.stack([z for z, _ in groups[w]])),
+ torch.from_numpy(np.stack([t for _t, t in groups[w]]))) for w in sorted(groups)]
def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
@@ -103,6 +113,10 @@ def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
The result is `clamp_temperature` of the optimised scale, so it lies in
`[TEMP_MIN, TEMP_MAX]` (or is the neutral 1.0 when the value is not a number).
+ The search is bounded to `LOG_T_MARGIN` e-folds outside that band, which covers any
+ minimum the answer can use; past it the float32 scale drifts toward `exp()` underflow,
+ where the objective divides by zero and every gradient becomes NaN, and
+ `clamp_temperature` reads that NaN back as the neutral 1.0.
Returns 1.0 when fewer than `min_n` pairs are given. `min_n` defaults to
`MIN_BUCKET_N` (the per-bucket floor). Type-level fits pass `MIN_TYPE_N`, which
is lower, so a dataset that fills no bucket still gets a scalar instead of
@@ -122,19 +136,25 @@ def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
sel.append((z, t))
if len(sel) < min_n:
return 1.0
- z_mat, t_mat = _pairs_to_tensors(sel)
+ groups = _pairs_by_width(sel)
+ n_rows = float(len(sel))
+ log_lo = float(np.log(TEMP_MIN)) - LOG_T_MARGIN
+ log_hi = float(np.log(TEMP_MAX)) + LOG_T_MARGIN
with torch.enable_grad():
log_t = torch.zeros(1, requires_grad=True)
opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)
def closure():
opt.zero_grad()
- loss = -(t_mat * torch.log_softmax(z_mat / log_t.exp(), -1)).sum(-1).mean()
+ scale = log_t.clamp(min=log_lo, max=log_hi).exp()
+ nll = sum(-(t * torch.log_softmax(z / scale, -1)).sum(-1).sum()
+ for z, t in groups)
+ loss = nll / n_rows
loss.backward()
return loss
opt.step(closure)
- fitted = float(log_t.detach().exp().item())
+ fitted = float(log_t.detach().clamp(min=log_lo, max=log_hi).exp().item())
return clamp_temperature(fitted, TEMP_MIN, TEMP_MAX)
diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py
index d93f0a92..3b0e100c 100644
--- a/tests/test_calibrate.py
+++ b/tests/test_calibrate.py
@@ -23,12 +23,14 @@ from laya.agent import Agent, load # noqa: E402
from laya.calibrate import ( # noqa: E402
CALIBRATION_VERSION,
ECE_HOLDOUT_FRAC,
+ LOG_T_MARGIN,
MIN_BUCKET_N,
MIN_TYPE_N,
MIN_BINNING_BUCKET_N,
_ece_report,
_ece_split,
_iter_records,
+ _pairs_by_width,
_softmax,
apply_binning_map,
apply_calibration_payload,
@@ -177,6 +179,107 @@ check_true(
)
+# --------------------------------------------------- ragged pools, pad columns, runaway scales
+# A type-level fit pools every option count that type has seen, so `fit_one_temperature`
+# regularly receives rows of different lengths. Two things used to cost such a pool its
+# answer, and both surface the same way -- the neutral 1.0, or a bound, instead of a fit:
+# 1. squaring off the short rows put the pad value in the same softmax those rows run
+# through, so a row whose own logits sat under it stopped being about that row;
+# 2. the search runs on log T with no bound, and a pool whose optimum is below
+# `TEMP_MIN` walks `log_t` past float32's `exp()` underflow, divides the objective
+# by zero, and hands `clamp_temperature` a NaN, which it reports as 1.0.
+def _spread_pool(width, scale, n, seed=0):
+ """`n` argmax-correct rows at one width, re-seeded per width."""
+ rs = np.random.RandomState(seed)
+ out = []
+ for _ in range(n):
+ z = list(rs.randn(width) * scale)
+ t = [0.0] * width
+ t[int(np.argmax(z))] = 1.0
+ out.append((z, t))
+ return out
+
+
+def _mixed_rows(k, rng, acc, mag):
+ """`k`-option rows that are right with probability `acc`, logits at +-mag."""
+ out = []
+ for _ in range(400):
+ y = int(rng.randint(0, k))
+ pred = y if rng.rand() < acc else int(rng.randint(0, k))
+ z, t = peaked(k, pred, mag)
+ t = np.zeros(k, dtype=np.float32)
+ t[y] = 1.0
+ out.append((z, t))
+ return out
+
+
+two, nine = _spread_pool(2, 1.0, 24), _spread_pool(9, 1.0, 24)
+t_two = fit_one_temperature(two, min_n=10)
+t_nine = fit_one_temperature(nine, min_n=10)
+t_ragged = fit_one_temperature(two + nine, min_n=10)
+check_true("runaway/every width on its own fits the floor", (t_two, t_nine) == (0.5, 0.5),
+ "two=%r nine=%r" % (t_two, t_nine))
+check_true("runaway/ragged pool is a number", bool(np.isfinite(t_ragged)), "T=%r" % t_ragged)
+check_true("runaway/ragged pool is not the neutral 1.0", t_ragged != 1.0, "T=%r" % t_ragged)
+check_true("runaway/ragged pool keeps the floor", abs(t_ragged - TEMP_MIN) < 1e-9,
+ "T=%r" % t_ragged)
+
+# The same runaway at one width: only the walk to zero was wrong, not the raggedness.
+t_soft = fit_one_temperature(_mixed_rows(3, np.random.RandomState(51), 0.7, 0.3), min_n=10)
+check_true("runaway/low-spread pool is not the neutral 1.0", t_soft != 1.0, "T=%r" % t_soft)
+check_true("runaway/low-spread pool keeps the floor", abs(t_soft - TEMP_MIN) < 1e-9,
+ "T=%r" % t_soft)
+
+# Rows whose every logit sits under the pad value: pooled, they used to report the ceiling.
+low_rows = [([-20000.0 - i, -20001.0 - i], [1.0, 0.0]) for i in range(12)]
+t_low_alone = fit_one_temperature(low_rows, min_n=10)
+t_low_pool = fit_one_temperature(low_rows + _spread_pool(4, 1.0, 12), min_n=10)
+check_true("pad/low rows alone fit the floor", abs(t_low_alone - TEMP_MIN) < 1e-9,
+ "T=%r" % t_low_alone)
+check("pad/low rows pooled fit what they fit alone", t_low_pool, t_low_alone)
+check_true("pad/pool is not the ceiling", t_low_pool != TEMP_MAX, "T=%r" % t_low_pool)
+check("pad/one matrix per width, narrowest first",
+ [int(z.shape[1]) for z, _ in _pairs_by_width(low_rows + _spread_pool(4, 1.0, 12))], [2, 4])
+check("pad/row totals survive the split",
+ sum(int(z.shape[0]) for z, _ in _pairs_by_width(low_rows + _spread_pool(4, 1.0, 12))), 24)
+
+# What a question type stores through the public API for the same rows.
+ragged_recs = ([(1, z, t, len(z)) for z, t in low_rows * 34]
+ + [(1, z, t, len(z)) for z, t in _spread_pool(5, 1.0, 400, 1)])
+t_public = fit_temperature_map(ragged_recs)["temperature"][1]
+check_true("public/ragged type keeps the floor", abs(t_public - TEMP_MIN) < 1e-9,
+ "T=%r" % t_public)
+check_true("public/ragged type is neither neutral nor the ceiling",
+ t_public not in (1.0, TEMP_MAX), "T=%r" % t_public)
+
+# Bounding the search must not flatten a minimum that sits inside the band: the fit is
+# checked against the objective it is supposed to minimise, in float64, not against a
+# hard-coded number.
+mid = _mixed_rows(20, np.random.RandomState(293), 0.7, 1.0)
+t_mid = fit_one_temperature(mid, min_n=10)
+
+
+def _nll_at(rows, scale):
+ z = np.asarray([r[0] for r in rows], dtype=np.float64) / float(scale)
+ t = np.asarray([r[1] for r in rows], dtype=np.float64)
+ z = z - z.max(1, keepdims=True)
+ p = np.exp(z)
+ p = p / p.sum(1, keepdims=True)
+ return float(-(t * np.log(np.clip(p, 1e-300, None))).sum(1).mean())
+
+
+check_true("band/interior fit stays interior", TEMP_MIN < t_mid < TEMP_MAX, "T=%r" % t_mid)
+check_true("band/interior fit beats the floor", _nll_at(mid, t_mid) < _nll_at(mid, TEMP_MIN),
+ "T=%r nll=%r floor=%r" % (t_mid, _nll_at(mid, t_mid), _nll_at(mid, TEMP_MIN)))
+check_true("band/interior fit beats either side of itself",
+ _nll_at(mid, t_mid) < min(_nll_at(mid, t_mid * 1.1), _nll_at(mid, t_mid / 1.1)))
+src = inspect.getsource(fit_one_temperature)
+check_true("band/search is bounded by LOG_T_MARGIN", "LOG_T_MARGIN" in src)
+check_true("band/the bound stops before float32 exp() underflows",
+ float(np.log(TEMP_MIN)) - LOG_T_MARGIN > -80.0,
+ "lower edge=%r" % (float(np.log(TEMP_MIN)) - LOG_T_MARGIN))
+
+
# --------------------------------------------------------------- ECE on a held-out split, not the fit rows
report = fitted["report"]
check_true("ece/after < before", report["ece_after"] < report["ece_before"],