Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(calibrate): fit, save and load per-bucket temperatures
Matt Van Hornd29b918b0809
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 7a9cca5f..5822a576 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -118,6 +118,7 @@ jobs:python tests/test_identifier_complexity.pypython tests/test_lang_stats.pypython tests/test_calibration_persistence.py+ python tests/test_calibrate.pypython tests/test_context_manager.pypython tests/test_criteria_normalization.pypython tests/test_docker_entrypoint.py@@ -236,7 +237,7 @@ jobs:for t in test_router test_criteria test_download test_shortlist \test_decision_model test_packaging test_lang_guess \test_identifier_complexity \- test_calibration_persistence test_context_manager \+ test_calibration_persistence test_calibrate test_context_manager \test_criteria_normalization test_docker_entrypoint \test_empty_questions test_router_memory test_shortlist_cosine \test_temperature_loading; dodiff --git a/laya/__init__.py b/laya/__init__.pyindex b6462e54..4f9e2e38 100644--- a/laya/__init__.py+++ b/laya/__init__.py@@ -23,6 +23,9 @@ _LAZY_ATTRS = {"Agent": (".agent", "Agent"),"RLAgent": (".agent", "RLAgent"),"load": (".agent", "load"),+ "fit_temperatures": (".calibrate", "fit_temperatures"),+ "fit_one_temperature": (".calibrate", "fit_one_temperature"),+ "fit_temperature_map": (".calibrate", "fit_temperature_map"),"proper_reward": (".common", "proper_reward"),"td_lambda_targets": (".common", "td_lambda_targets"),"ece_score": (".common", "ece_score"),@@ -63,6 +66,9 @@ __all__ = ["Agent","RLAgent","load",+ "fit_temperatures",+ "fit_one_temperature",+ "fit_temperature_map","Router","RouteDecision","DEFAULT_MODELS",diff --git a/laya/agent.py b/laya/agent.pyindex 1b58de6f..49f96ddd 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -11,6 +11,12 @@ from typing import Any, Dict, List, Optional, Unionimport numpy as npimport torch+from .calibrate import (+ _install_temperatures,+ apply_calibration_payload,+ calibration_payload,+ fit_temperature_map,+)from .common import (QTYPES,TEMP_MAX,@@ -216,6 +222,15 @@ def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype:return amp_dtype(checkpoint_default)+def _option_logits(logits, items, offset):+ """Raw per-option logits, the rows `_decode_answers` divides by temperature.++ Calibration record collection slices with this same helper, so a fitted map sees the+ option width the decoder scales and not a second tokenization of the state.+ """+ return [logits[offset + j, : len(item["markers"])] for j, item in enumerate(items)]++class Agent(HookRegistry):"""System 1 decision model runtime: fast, non-autoregressive, calibrated decisions."""@@ -250,6 +265,7 @@ class Agent(HookRegistry):hooks_raise: bool = True,hooks_concurrent: bool = True,hooks_timeout: Optional[float] = None,+ calibration: Optional[str] = None,):"""Load a Laya checkpoint.@@ -267,6 +283,10 @@ class Agent(HookRegistry):`Agent("convaiinnovations/laya", subfolder="multilingual")`. Only that subfolder isdownloaded, so bundling does not cost every user the whole family.+ `calibration` is an optional JSON path with `temperature` and `temperature_by_options`.+ It is applied after the checkpoint config, so a fitted map overrides shipped scalars+ without rewriting `model.safetensors`.+`hooks` / `on_predict_start` / `on_predict_end` observe or shape every prediction; see`laya.hooks`. `hooks_raise=False` warns and continues when a hook fails,`hooks_concurrent=False` serialises hooks that are not safe to run in parallel, and@@ -279,6 +299,9 @@ class Agent(HookRegistry):self._hooks_lock = threading.RLock() if not hooks_concurrent else Noneself._hooks_mutex = threading.Lock()self.model_id = model_id_or_path+ # Retained so `save_calibration` can record which checkpoint the map was fitted for.+ self.model_id_or_path = model_id_or_path+ self.subfolder = subfolderfrom safetensors.torch import load_file@@ -424,6 +447,8 @@ class Agent(HookRegistry):"using %s. Treat confidence from the affected entries as uncalibrated."% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),RuntimeWarning, stacklevel=2)+ if calibration:+ self.load_calibration(calibration)# Autocast policy. CUDA, MPS and XPU all support fp16/bf16 autocast and the shipped# checkpoints are trained in reduced precision; on CUDA the checkpoint's `amp_dtype`# (bf16) is the default and LAYA_CUDA_AMP=fp16|bf16 overrides it. CPU bf16 is only a win@@ -760,6 +785,7 @@ class Agent(HookRegistry):internal: Dict[str, Dict], offset: int, lang: Optional[str] = None) -> Dict[str, Any]:"""Turn one state's logit rows (starting at `offset`) into typed answers."""answers = {}+ raw_rows = _option_logits(logits, items, offset)for j, qid in enumerate(ids):r = offset + jq = internal[qid]@@ -769,7 +795,7 @@ class Agent(HookRegistry):if lang and lang.split("-")[0].lower() in self.lang_temperatures:l_cfg = self.lang_temperatures[lang.split("-")[0].lower()]t_scale = l_cfg["temperature_by_options"].get(temp_bucket(qt, k), l_cfg["temperature"][qt])- z = logits[r, :k] / t_scale+ z = raw_rows[j] / t_scalep = np.exp(z - z.max())p = p / p.sum()@@ -1126,6 +1152,44 @@ class Agent(HookRegistry):predict = system_one+ def fit_temperatures(self, records, compute_ece: bool = False, seed: int = 0) -> Dict[str, Any]:+ """Fit per-bucket temperatures from CPU records and store them on this agent.++ `records` are `(qtype, logits, target, k)`. Build them with+ `laya.calibrate.records_from_labeled` when you have labeled forwards; this method+ does not download weights or write `model.safetensors`. `seed` only affects the+ held-out ECE split when `compute_ece` is true. The checkpoint `cfg` is left as loaded.+ """+ result = fit_temperature_map(records, compute_ece=compute_ece, seed=seed)+ # Already clamped inside the fitter; don't report that as a bad calibration file.+ _install_temperatures(self, result["temperature"], result["temperature_by_options"], warn=False)+ return result++ def save_calibration(self, path: str) -> None:+ """Write temperatures and the checkpoint they were fitted for. Does not write weights."""+ payload = calibration_payload(+ self.temperature,+ self.temperature_by_options,+ model_id_or_path=getattr(self, "model_id_or_path", None),+ subfolder=getattr(self, "subfolder", None),+ config=getattr(self, "cfg", None),+ )+ with open(path, "w") as f:+ json.dump(payload, f, indent=2)+ f.write("\n")++ def load_calibration(self, path: str) -> None:+ """Read a JSON map written by `save_calibration` onto this agent.++ A file with no `version` is treated as version 1 and still loads. A newer file+ whose recorded checkpoint does not match this agent warns and still loads.+ Values that are not numbers, or that sit outside `[TEMP_MIN, TEMP_MAX]`, are clamped+ with `clamp_temperature` the same way checkpoint load is.+ """+ with open(path) as f:+ payload = json.load(f)+ apply_calibration_payload(self, payload)+RLAgent = Agent@@ -1136,7 +1200,8 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]lang_temperatures: Optional[Dict[str, Dict[str, Any]]] = None,hooks=None, on_predict_start=None, on_predict_end=None,hooks_raise: bool = True, hooks_concurrent: bool = True,- hooks_timeout: Optional[float] = None) -> Agent:+ hooks_timeout: Optional[float] = None,+ calibration: Optional[str] = None) -> Agent:"""Load a Laya agent.`subfolder` picks one checkpoint out of a repo that bundles several:@@ -1147,11 +1212,11 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]`revision`/`expected_sha256` pin and verify the downloaded artifacts; see `Agent`.`hooks` / `on_predict_start` / `on_predict_end` observe or shape every prediction; see- `laya.hooks`.+ `laya.hooks`. `calibration` is the same optional JSON path accepted by `Agent`."""return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, fast=fast,revision=revision, expected_sha256=expected_sha256,lang_temperatures=lang_temperatures,hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent,- hooks_timeout=hooks_timeout)+ hooks_timeout=hooks_timeout, calibration=calibration)diff --git a/laya/calibrate.py b/laya/calibrate.pynew file mode 100644index 00000000..cd437755--- /dev/null+++ b/laya/calibrate.py@@ -0,0 +1,410 @@+"""Fit and persist per-bucket temperatures for Laya logits.++Ports the notebook LBFGS fitter (NLL on softmax(z/T)) and groups records by+`temp_bucket(qtype, k)` so the runtime's `temperature_by_options` lookup has a writer.+Fitting is CPU-side: records are `(qtype, logits, target, k)` and do not load Hub weights.++Two sample floors, on purpose:++- `MIN_BUCKET_N` is the per-bucket floor. Buckets with fewer examples are omitted from+ `temperature_by_options`; the type-level scalar covers them at lookup time.+- `MIN_TYPE_N` is the lower floor used only for that type-level scalar. It stays at the+ previous floor of 10 so a realistic dataset that fills no single bucket still gets a+ temperature instead of remaining at 1.0. Do not raise it to `MIN_BUCKET_N`.++Fitted temperatures go through `clamp_temperature` and therefore `[TEMP_MIN, TEMP_MAX]`+from `laya.common`, the same guard checkpoint load uses. There is no second pair of+bounds in this module.+"""+import warnings+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple++import numpy as np+import torch++from .common import (+ QTYPES,+ TEMP_MAX,+ TEMP_MIN,+ clamp_temperature,+ collate_items,+ ece_score,+ temp_bucket,+)++# Per-bucket floor. Below this, `temperature_by_options` omits the bucket.+MIN_BUCKET_N = 2000+# Type-level floor only. Lower than MIN_BUCKET_N; see the module docstring.+MIN_TYPE_N = 10+# Fraction of each bucket held out when `compute_ece=True`. Unused otherwise.+ECE_HOLDOUT_FRAC = 0.2+# Payload schema. Files written before identity was recorded have no version key;+# readers treat that absence as version 1.+CALIBRATION_VERSION = 2+N_QTYPES = len(QTYPES)++# (qtype:int, logits:1d, target:1d, k:int)+Record = Tuple[int, Any, Any, int]++_CONFIG_IDENTITY_SKIP = ("temperature", "temperature_by_options")+++def _vec(x) -> np.ndarray:+ if isinstance(x, torch.Tensor):+ x = x.detach().cpu().numpy()+ return np.asarray(x, dtype=np.float32).reshape(-1)+++def _pairs_to_tensors(pairs: Sequence) -> Tuple[torch.Tensor, torch.Tensor]:+ kmax = max(len(_vec(z)) for z, _ in pairs)+ z_mat = torch.full((len(pairs), kmax), -1e4)+ t_mat = torch.zeros((len(pairs), kmax))+ for i, (z, t) in enumerate(pairs):+ z = _vec(z)+ t = _vec(t)+ n = min(len(z), len(t))+ z_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(z[:n]))+ t_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(t[:n]))+ return z_mat, t_mat+++def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:+ """Fit one scalar T by NLL + LBFGS on log T.++ The result is `clamp_temperature` of the optimised scale, so it lies in+ `[TEMP_MIN, TEMP_MAX]` (or is the neutral 1.0 when the value is not a number).+ Returns 1.0 when fewer than `min_n` pairs are given. `min_n` defaults to+ `MIN_BUCKET_N` (the per-bucket floor). Type-level fits pass `MIN_TYPE_N`, which+ is lower, so a dataset that fills no bucket still gets a scalar instead of+ staying at 1.0.+ """+ if min_n is None:+ min_n = MIN_BUCKET_N+ sel = list(pairs)+ if len(sel) < min_n:+ return 1.0+ z_mat, t_mat = _pairs_to_tensors(sel)+ with torch.enable_grad():+ log_t = torch.zeros(1, requires_grad=True)+ opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)++ def closure():+ opt.zero_grad()+ loss = -(t_mat * torch.log_softmax(z_mat / log_t.exp(), -1)).sum(-1).mean()+ loss.backward()+ return loss++ opt.step(closure)+ fitted = float(log_t.detach().exp().item())+ return clamp_temperature(fitted, TEMP_MIN, TEMP_MAX)+++def _iter_records(records: Iterable) -> List[Tuple[int, np.ndarray, np.ndarray, int]]:+ out = []+ for rec in records:+ if len(rec) == 4:+ qtype, logits, target, k = rec+ elif len(rec) == 3:+ qtype, logits, target = rec+ k = len(_vec(logits))+ else:+ raise ValueError("record must be (qtype, logits, target[, k])")+ logits = _vec(logits)+ target = _vec(target)+ k = int(k)+ if k < 1:+ raise ValueError("k must be >= 1")+ out.append((int(qtype), logits[:k], target[:k], k))+ return out+++def _softmax(z, t_scale: float) -> np.ndarray:+ z = np.asarray(z, dtype=np.float64) / max(1e-3, float(t_scale))+ z = z - z.max()+ p = np.exp(z)+ return p / p.sum()+++def _ece_report(recs, temperature, temperature_by_options) -> Dict[str, Any]:+ conf_b, cor_b, conf_a, cor_a = [], [], [], []+ for qt, z, t, k in recs:+ y = int(np.argmax(t[:k]))+ p0 = _softmax(z[:k], 1.0)+ conf_b.append(float(p0.max()))+ cor_b.append(float(int(p0.argmax()) == y))+ t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])+ p1 = _softmax(z[:k], t_scale)+ conf_a.append(float(p1.max()))+ cor_a.append(float(int(p1.argmax()) == y))+ return {+ "ece_before": ece_score(np.asarray(conf_b), np.asarray(cor_b)),+ "ece_after": ece_score(np.asarray(conf_a), np.asarray(cor_a)),+ "n": float(len(recs)),+ }+++def _group_records(recs):+ by_type = {qt: [] for qt in range(N_QTYPES)}+ by_bucket = {}+ for qt, z, t, k in recs:+ if qt in by_type:+ by_type[qt].append((z, t))+ key = temp_bucket(qt, k)+ by_bucket.setdefault(key, []).append((z, t))+ return by_type, by_bucket+++def _ece_split(recs, seed):+ """Split records into fit and eval, stratified by `temp_bucket`.++ Returns `(fit_recs, eval_recs, excluded_buckets)`. The split is a deterministic+ partition (`numpy.random.RandomState(seed)`): every record is in exactly one side.+ A bucket whose fit side would fall below `MIN_BUCKET_N` is not split; all of its+ records stay in `fit_recs` and its key is listed in `excluded_buckets`.+ """+ by_bucket = {}+ for i, rec in enumerate(recs):+ qt, _z, _t, k = rec+ by_bucket.setdefault(temp_bucket(qt, k), []).append(i)++ rng = np.random.RandomState(seed)+ fit_idx = []+ eval_idx = []+ excluded = []+ for key in sorted(by_bucket):+ idxs = by_bucket[key]+ n = len(idxs)+ n_eval = int(round(n * ECE_HOLDOUT_FRAC))+ if n - n_eval < MIN_BUCKET_N:+ fit_idx.extend(idxs)+ excluded.append(key)+ continue+ perm = rng.permutation(n)+ for j in perm[:n_eval]:+ eval_idx.append(idxs[int(j)])+ for j in perm[n_eval:]:+ fit_idx.append(idxs[int(j)])++ fit_idx.sort()+ eval_idx.sort()+ fit_recs = [recs[i] for i in fit_idx]+ eval_recs = [recs[i] for i in eval_idx]+ return fit_recs, eval_recs, excluded+++def _fit_groups(by_type, by_bucket):+ temperature = [1.0] * N_QTYPES+ for qt in range(N_QTYPES):+ pairs = by_type[qt]+ # MIN_TYPE_N, not MIN_BUCKET_N: the scalar has to exist when every bucket is small.+ if len(pairs) >= MIN_TYPE_N:+ temperature[qt] = fit_one_temperature(pairs, min_n=MIN_TYPE_N)++ temperature_by_options = {}+ for key, pairs in by_bucket.items():+ if len(pairs) < MIN_BUCKET_N:+ continue+ temperature_by_options[key] = fit_one_temperature(pairs)+ return temperature, temperature_by_options+++def fit_temperature_map(records: Iterable, compute_ece: bool = False, seed: int = 0) -> Dict[str, Any]:+ """Fit type-level scalars and per-bucket temperatures.++ `MIN_BUCKET_N` is the per-bucket floor: smaller buckets are omitted from+ `temperature_by_options` and the type-level scalar covers them. `MIN_TYPE_N` is+ the separate, lower floor for that scalar only.++ `compute_ece=False` (the default, and the path `Agent.fit_temperatures` stores)+ fits on every record and returns no `report` key. `seed` is ignored on this path.++ `compute_ece=True` holds out `ECE_HOLDOUT_FRAC` of each bucket, stratified by+ `temp_bucket`, using `seed` so the same records always split the same way. Temperatures+ are fit on the remainder only and ECE is scored only on the held-out records.+ `report["n"]` is the number of records passed in; `report["n_eval"]` is the held-out+ count the ECE rests on. A bucket that would drop below `MIN_BUCKET_N` after the+ holdout is fit on all of its records, left out of the eval set, and named in+ `report["buckets_excluded_from_eval"]` instead of being dropped. `n_by_bucket`+ always counts the full input, including when the fit itself used a subset.+ """+ recs = _iter_records(records)+ all_by_type, all_buckets = _group_records(recs)+ n_by_bucket = {key: len(pairs) for key, pairs in all_buckets.items()}++ if compute_ece:+ _fit_recs, eval_recs, excluded = _ece_split(recs, seed)+ by_type, by_bucket = _group_records(_fit_recs)+ else:+ eval_recs = None+ excluded = []+ by_type, by_bucket = all_by_type, all_buckets++ temperature, temperature_by_options = _fit_groups(by_type, by_bucket)+ out: Dict[str, Any] = {+ "temperature": temperature,+ "temperature_by_options": temperature_by_options,+ "n_by_bucket": n_by_bucket,+ }+ if compute_ece:+ report = _ece_report(eval_recs, temperature, temperature_by_options)+ # `n` stays the size of the input. `n_eval` is the held-out count ECE used.+ report["n"] = float(len(recs))+ report["n_eval"] = float(len(eval_recs))+ report["buckets_excluded_from_eval"] = excluded+ out["report"] = report+ return out+++fit_temperatures = fit_temperature_map+++def records_from_labeled(agent, pairs: Sequence) -> List[Record]:+ """Collect CPU records from `(state, questions, targets)`.++ Logits are the raw rows `_decode_answers` scales, not the probabilities it returns.+ `_encode_state` builds one state's items, `_forward` runs them, and `_option_logits`+ slices each question to the same marker width `_decode_answers` divides by temperature.+ No Hub download and no `model.safetensors` write. Tests stay weight-free by passing+ synthetic records to `fit_temperature_map` instead of calling this.+ """+ # Imported lazily: `agent` imports this module at load, and `_option_logits` is defined+ # on the way through that import.+ from .agent import _option_logits++ records: List[Record] = []+ for state, questions, targets in pairs:+ ids = list(questions.keys())+ if not ids:+ continue+ for qid in ids:+ agent._check_question(qid, questions[qid])+ internal = {qid: agent._to_internal(questions[qid]) for qid in ids}+ items = agent._encode_state(state, ids, internal)+ batch = collate_items([items], agent.tok.pad_token_id)+ if batch is None:+ continue+ logits, _act = agent._forward(batch)+ rows = _option_logits(logits, items, 0)+ for j, qid in enumerate(ids):+ k = len(items[j]["markers"])+ qt = int(items[j]["qtype"])+ records.append((qt, _vec(rows[j]), _vec(targets[qid])[:k], k))+ return records+++def _config_identity(cfg):+ """Checkpoint config without the temperature fields this file itself stores."""+ if cfg is None:+ return None+ return {str(k): cfg[k] for k in cfg if k not in _CONFIG_IDENTITY_SKIP}+++def calibration_payload(+ temperature,+ temperature_by_options,+ model_id_or_path=None,+ subfolder=None,+ config=None,+) -> Dict[str, Any]:+ """JSON body written by `Agent.save_calibration` (no weights).++ `version` is `CALIBRATION_VERSION`. Payloads written before identity was recorded have+ no version key; `apply_calibration_payload` treats that as version 1 and still loads.++ `model_id_or_path`, `subfolder`, and `config` say which checkpoint the map was fitted+ against. `config` is `rl_agent_config.json` without `temperature` /+ `temperature_by_options` (those live at the top of this payload).+ """+ return {+ "version": CALIBRATION_VERSION,+ "temperature": [float(x) for x in temperature],+ "temperature_by_options": {+ str(k): float(v) for k, v in dict(temperature_by_options).items()+ },+ "model_id_or_path": model_id_or_path,+ "subfolder": subfolder,+ "config": _config_identity(config),+ }+++def _warn_if_identity_mismatch(obj, payload) -> None:+ recorded_model = payload.get("model_id_or_path")+ recorded_sub = payload.get("subfolder")+ recorded_cfg = payload.get("config")+ current_model = getattr(obj, "model_id_or_path", None)+ current_sub = getattr(obj, "subfolder", None)+ current_cfg = _config_identity(getattr(obj, "cfg", None))+ if (recorded_model, recorded_sub, recorded_cfg) == (current_model, current_sub, current_cfg):+ return+ cfg_note = ""+ if recorded_cfg != current_cfg:+ cfg_note = " Checkpoint config identity differs."+ warnings.warn(+ "calibration was fitted for model_id_or_path=%r subfolder=%r but this agent is "+ "model_id_or_path=%r subfolder=%r.%s Loading the temperatures anyway."+ % (recorded_model, recorded_sub, current_model, current_sub, cfg_note),+ UserWarning,+ stacklevel=3,+ )+++def _rejected_temperatures(entries):+ """Names whose raw value is not the number `clamp_temperature` kept, matching checkpoint load."""+ rejected = []+ for name, raw, applied in entries:+ try:+ if float(raw) == applied:+ continue+ except (TypeError, ValueError):+ pass+ rejected.append("%s=%r -> %g" % (name, raw, applied))+ return rejected+++def _install_temperatures(obj, temperature, temperature_by_options, warn: bool = True) -> None:+ """Store a map the way checkpoint load does: raw values, and the clamped ones inference reads.++ `warn=False` is the fitter path: those values were already passed through `clamp_temperature`,+ so a second pass must not report them as a bad file.+ """+ raw_t = list(temperature)+ applied_t = [clamp_temperature(t) for t in raw_t]+ raw_b = {str(k): v for k, v in dict(temperature_by_options).items()}+ applied_b = {k: clamp_temperature(v) for k, v in raw_b.items()}+ obj.temperature_raw = raw_t+ obj.temperature = applied_t+ obj.temperature_by_options_raw = raw_b+ obj.temperature_by_options = applied_b+ if not warn:+ return+ entries = [("temperature[%d]" % i, raw, applied_t[i]) for i, raw in enumerate(raw_t)]+ entries += [(key, raw_b[key], applied_b[key]) for key in raw_b]+ rejected = _rejected_temperatures(entries)+ if rejected:+ warnings.warn(+ "laya: calibration temperatures are invalid or outside [%g, %g]; "+ "using %s. Treat confidence from the affected entries as uncalibrated."+ % (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),+ RuntimeWarning,+ stacklevel=3,+ )+++def apply_calibration_payload(obj, payload: Dict[str, Any]) -> None:+ """Copy `temperature` and `temperature_by_options` from a calibration payload onto `obj`.++ A missing `version` is version 1 (temperatures only, no checkpoint identity). Version+ `CALIBRATION_VERSION` records the checkpoint the map was fitted for; a mismatch warns+ and still loads, so an older file never becomes a hard failure. Each value is passed+ through `clamp_temperature`, so a non-numeric or out-of-range entry cannot crash a later+ forward the way an unclamped zero used to.+ """+ version = payload.get("version", 1)+ if version is None:+ version = 1+ temps = payload.get("temperature")+ if temps is None or len(temps) != N_QTYPES:+ raise ValueError("calibration JSON must contain temperature: [3 floats]")+ if int(version) >= CALIBRATION_VERSION:+ _warn_if_identity_mismatch(obj, payload)+ _install_temperatures(obj, temps, payload.get("temperature_by_options") or {})diff --git a/tests/test_calibrate.py b/tests/test_calibrate.pynew file mode 100644index 00000000..7fe6a717--- /dev/null+++ b/tests/test_calibrate.py@@ -0,0 +1,458 @@+"""Per-bucket temperature fitting. No model weights are loaded.++Records are CPU tuples (qtype, logits, target, k). The fitter is the notebook's NLL+LBFGS+on log T, grouped by `temp_bucket`. Bounds are `TEMP_MIN` / `TEMP_MAX` via `clamp_temperature`.+Logits for a labeled agent come from `_encode_state` and the raw rows `_decode_answers` scales.+"""+import inspect+import json+import os+import sys+import tempfile+import warnings++import numpy as np++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya.agent import Agent, load # noqa: E402+from laya.calibrate import ( # noqa: E402+ CALIBRATION_VERSION,+ ECE_HOLDOUT_FRAC,+ MIN_BUCKET_N,+ MIN_TYPE_N,+ _ece_report,+ _ece_split,+ _iter_records,+ apply_calibration_payload,+ calibration_payload,+ fit_one_temperature,+ fit_temperature_map,+ fit_temperatures,+ records_from_labeled,+)+from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, temp_bucket # noqa: E402+import laya.calibrate as _calibrate # noqa: E402++PASS, FAIL = [], []++# Large enough that ECE_HOLDOUT_FRAC still leaves MIN_BUCKET_N rows to fit.+BUCKET_N = 2500+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))+++def check_true(name, cond, detail=""):+ if cond:+ PASS.append(name)+ else:+ FAIL.append("%s %s" % (name, detail))+++def peaked(k, idx, mag):+ z = np.full(k, -mag, dtype=np.float32)+ z[idx] = mag+ t = np.zeros(k, dtype=np.float32)+ t[idx] = 1.0+ return z, t+++def overconfident_records(n, k, qtype, seed=0, acc=0.6, mag=8.0):+ rng = np.random.RandomState(seed)+ recs = []+ for i in range(n):+ y = int(rng.randint(0, k))+ pred = y if rng.rand() < acc else int((y + 1) % k)+ z = np.full(k, -mag, dtype=np.float32)+ z[pred] = mag+ t = np.zeros(k, dtype=np.float32)+ t[y] = 1.0+ recs.append((qtype, z, t, k))+ return recs+++check("floor/MIN_BUCKET_N", MIN_BUCKET_N, 2000)+check("floor/MIN_TYPE_N", MIN_TYPE_N, 10)+check("clamp/TEMP_MIN", TEMP_MIN, 0.5)+check("clamp/TEMP_MAX", TEMP_MAX, 5.0)+check_true("clamp/no local TEMP_LO", not hasattr(_calibrate, "TEMP_LO"))+check_true("clamp/no local TEMP_HI", not hasattr(_calibrate, "TEMP_HI"))+check_true(+ "setup/holdout leaves a fittable bucket",+ BUCKET_N - int(round(BUCKET_N * ECE_HOLDOUT_FRAC)) >= MIN_BUCKET_N,+)+++# --------------------------------------------------------------- overconfident -> T > 1+choice_k2 = overconfident_records(BUCKET_N, 2, QTYPES["choice"], seed=1, acc=0.6, mag=8.0)+t_choice = fit_one_temperature([(z, t) for _, z, t, _ in choice_k2])+check_true("overconfident/T>1", t_choice > 1.0, "T=%r" % t_choice)+check_true("overconfident/T<=TEMP_MAX", t_choice <= TEMP_MAX, "T=%r" % t_choice)+++# --------------------------------------------------------------- bucket keys + omit n<MIN_BUCKET_N+noul_k2 = overconfident_records(BUCKET_N, 2, QTYPES["noul"], seed=2, acc=0.6, mag=8.0)+choice_k12 = overconfident_records(BUCKET_N, 12, QTYPES["choice"], seed=3, acc=0.6, mag=8.0)+# Exactly the per-bucket floor: fitted, but too small to hold any rows out for ECE.+choice_k4 = overconfident_records(MIN_BUCKET_N, 4, QTYPES["choice"], seed=6, acc=0.6, mag=8.0)+tiny_score = overconfident_records(5, 4, QTYPES["score"], seed=4, acc=0.6, mag=8.0)+mixed = choice_k2 + noul_k2 + choice_k12 + choice_k4 + tiny_score+fitted = fit_temperature_map(mixed, compute_ece=True)+keys = set(fitted["temperature_by_options"])+check_true("keys/choice:2", "choice:2" in keys, keys)+check_true("keys/noul:2", "noul:2" in keys, keys)+check_true("keys/choice:11+", "choice:11+" in keys, keys)+check_true("keys/choice:3-5 at floor", "choice:3-5" in keys, keys)+check("keys/temp_bucket choice:2", temp_bucket(QTYPES["choice"], 2), "choice:2")+check("keys/temp_bucket noul:2", temp_bucket(QTYPES["noul"], 2), "noul:2")+check("keys/temp_bucket choice:11+", temp_bucket(QTYPES["choice"], 12), "choice:11+")+check_true("omit/n<MIN_BUCKET_N score:3-5 not in map", "score:3-5" not in keys, keys)+check("omit/n_by_bucket still counts tiny", fitted["n_by_bucket"].get("score:3-5"), 5)+check("omit/n_by_bucket counts floor bucket", fitted["n_by_bucket"].get("choice:3-5"), MIN_BUCKET_N)+check_true("omit/not NaN", all(np.isfinite(v) for v in fitted["temperature_by_options"].values()))+check_true("alias/fit_temperatures is fit_temperature_map", fit_temperatures is fit_temperature_map)+++# --------------------------------------------------------------- type floor stays below the bucket floor+# 50 is above MIN_TYPE_N and below MIN_BUCKET_N, so the scalar moves and the bucket is omitted.+medium_score = overconfident_records(50, 4, QTYPES["score"], seed=5, acc=0.6, mag=8.0)+med = fit_temperature_map(medium_score, compute_ece=False, seed=0)+med_other_seed = fit_temperature_map(medium_score, compute_ece=False, seed=1)+check_true(+ "type-floor/fitted below bucket floor",+ med["temperature"][QTYPES["score"]] > 1.0,+ med["temperature"],+)+check_true("type-floor/bucket omitted", "score:3-5" not in med["temperature_by_options"])+check_true("ece/off has no report", "report" not in med)+check("ece/off ignores seed", med["temperature"], med_other_seed["temperature"])+++# --------------------------------------------------------------- clamp 0.5-5+# 50% accurate, extreme logits: NLL wants a large T, clamped at TEMP_MAX.+rng = np.random.RandomState(7)+hi_pairs = []+for i in range(MIN_BUCKET_N):+ y = int(rng.randint(0, 2))+ pred = y if i % 2 == 0 else 1 - y+ z, t = peaked(2, pred, 80.0)+ t = np.zeros(2, dtype=np.float32)+ t[y] = 1.0+ hi_pairs.append((z, t))+t_hi = fit_one_temperature(hi_pairs)+check_true("clamp/high in [0.5, 5]", TEMP_MIN <= t_hi <= TEMP_MAX, "T=%r" % t_hi)+check_true("clamp/high at TEMP_MAX", abs(t_hi - TEMP_MAX) < 1e-4, "T=%r" % t_hi)++# Always-correct, mild logits: NLL wants a small T, clamped at TEMP_MIN.+lo_pairs = []+for i in range(MIN_BUCKET_N):+ z, t = peaked(2, i % 2, 0.3)+ lo_pairs.append((z, t))+t_lo = fit_one_temperature(lo_pairs)+check_true("clamp/low in [0.5, 5]", TEMP_MIN <= t_lo <= TEMP_MAX, "T=%r" % t_lo)+check_true("clamp/low at TEMP_MIN", abs(t_lo - TEMP_MIN) < 1e-4, "T=%r" % t_lo)++src = inspect.getsource(fit_one_temperature)+check_true("clamp/source uses clamp_temperature", "clamp_temperature" in src)+check_true("clamp/source uses TEMP_MIN", "TEMP_MIN" in src)+check_true("clamp/source uses TEMP_MAX", "TEMP_MAX" in src)+check_true("clamp/n<MIN_BUCKET_N returns 1.0", fit_one_temperature(lo_pairs[:5]) == 1.0)+check_true(+ "clamp/n=MIN_BUCKET_N-1 returns 1.0",+ fit_one_temperature(lo_pairs[: MIN_BUCKET_N - 1]) == 1.0,+)+++# --------------------------------------------------------------- ECE on a held-out split, not the fit rows+report = fitted["report"]+check_true("ece/after < before", report["ece_after"] < report["ece_before"],+ "before=%r after=%r" % (report["ece_before"], report["ece_after"]))+check_true("ece/before finite", np.isfinite(report["ece_before"]))+check_true("ece/after finite", np.isfinite(report["ece_after"]))+check("ece/n is the full input", report["n"], float(len(mixed)))++parsed = _iter_records(mixed)+fit_recs, eval_recs, excluded = _ece_split(parsed, 0)+fit_ids = [id(r) for r in fit_recs]+eval_ids = [id(r) for r in eval_recs]+check_true("split/disjoint", set(fit_ids).isdisjoint(eval_ids))+check("split/partition", len(fit_ids) + len(eval_ids), len(parsed))+again_fit, again_eval, again_ex = _ece_split(parsed, 0)+check("split/deterministic eval", [id(r) for r in again_eval], eval_ids)+check("split/deterministic excluded", again_ex, excluded)+other_fit, other_eval, _other_ex = _ece_split(parsed, 1)+check_true("split/seed changes holdout", [id(r) for r in other_eval] != eval_ids)+check_true("split/floor bucket excluded", "choice:3-5" in excluded, excluded)+check_true("split/tiny excluded", "score:3-5" in excluded, excluded)+check_true("split/big bucket held out", "choice:2" not in excluded, excluded)+check("ece/n_eval", report["n_eval"], float(len(eval_recs)))+check_true("ece/n_eval < n", report["n_eval"] < report["n"])+check("ece/excluded noted", report["buckets_excluded_from_eval"], excluded)+# The map itself is the one fit on the non-held-out rows.+refit = fit_temperature_map(fit_recs, compute_ece=False)+check("split/fit ignores eval buckets", fitted["temperature_by_options"], refit["temperature_by_options"])+check("split/fit ignores eval types", fitted["temperature"], refit["temperature"])+holdout_ece = _ece_report(eval_recs, fitted["temperature"], fitted["temperature_by_options"])+check("ece/after is the holdout", report["ece_after"], holdout_ece["ece_after"])+check("ece/before is the holdout", report["ece_before"], holdout_ece["ece_before"])+full_ece = _ece_report(parsed, fitted["temperature"], fitted["temperature_by_options"])+check_true(+ "ece/not scored on the fit pool",+ full_ece["ece_after"] != report["ece_after"],+ "full=%r holdout=%r" % (full_ece["ece_after"], report["ece_after"]),+)+again_fit_map = fit_temperature_map(mixed, compute_ece=True, seed=0)+check("ece/fit deterministic", again_fit_map["report"], fitted["report"])+++# --------------------------------------------------------------- live Agent map + JSON round-trip+_CFG = {+ "encoder": "answerdotai/ModernBERT-large",+ "head_layers": 2,+ "max_len": 512,+ "model_name": "laya",+ "temperature": [9.0, 9.0, 9.0],+ "temperature_by_options": {"choice:2": 9.0},+}+agent = Agent.__new__(Agent)+agent.temperature = [1.0, 1.0, 1.0]+agent.temperature_by_options = {}+agent.model_id_or_path = "convaiinnovations/laya"+agent.subfolder = None+agent.cfg = dict(_CFG)+live = agent.fit_temperatures(mixed, compute_ece=True)+check_true("agent/live temperature_by_options updates",+ "choice:2" in agent.temperature_by_options, agent.temperature_by_options)+check("agent/live matches result", agent.temperature_by_options, live["temperature_by_options"])+check_true("agent/type-level choice T>1", agent.temperature[0] > 1.0, agent.temperature)+check_true("agent/forwards seed", "seed=seed" in inspect.getsource(Agent.fit_temperatures))++td = tempfile.mkdtemp()+calib_path = os.path.join(td, "calibration.json")+agent.save_calibration(calib_path)+check_true("save/no model.safetensors", not os.path.exists(os.path.join(td, "model.safetensors")))+check_true("save/only the json file", os.listdir(td) == ["calibration.json"], os.listdir(td))+with open(calib_path) as f:+ payload = json.load(f)+check(+ "save/keys",+ sorted(payload.keys()),+ ["config", "model_id_or_path", "subfolder", "temperature", "temperature_by_options", "version"],+)+check("save/version", payload["version"], CALIBRATION_VERSION)+check("save/model_id", payload["model_id_or_path"], "convaiinnovations/laya")+check("save/subfolder", payload["subfolder"], None)+check("save/config encoder", payload["config"]["encoder"], "answerdotai/ModernBERT-large")+check("save/config model_name", payload["config"]["model_name"], "laya")+check_true("save/config omits temperature", "temperature" not in payload["config"])+check_true(+ "save/config omits temperature_by_options",+ "temperature_by_options" not in payload["config"],+)+check_true("save/no weights key", "model.safetensors" not in json.dumps(payload))++other = Agent.__new__(Agent)+other.model_id_or_path = agent.model_id_or_path+other.subfolder = agent.subfolder+other.cfg = dict(_CFG)+with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ other.load_calibration(calib_path)+check_true("load/matching identity is silent", not caught, [str(w.message) for w in caught])+check("load/temperature", other.temperature, agent.temperature)+check("load/by_options", other.temperature_by_options, agent.temperature_by_options)++mismatch = Agent.__new__(Agent)+mismatch.model_id_or_path = "convaiinnovations/laya-multilingual"+mismatch.subfolder = "multilingual"+mismatch.cfg = {"encoder": "jhu-clsp/mmBERT-base", "head_layers": 2}+with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ mismatch.load_calibration(calib_path)+check_true(+ "load/mismatch warns",+ any(issubclass(w.category, UserWarning) for w in caught),+ [str(w.message) for w in caught],+)+check("load/mismatch still applies temperature", mismatch.temperature, agent.temperature)+check("load/mismatch still applies by_options", mismatch.temperature_by_options, agent.temperature_by_options)++cfg_only = Agent.__new__(Agent)+cfg_only.model_id_or_path = agent.model_id_or_path+cfg_only.subfolder = agent.subfolder+cfg_only.cfg = {"encoder": "some-other-encoder", "head_layers": 4, "temperature": [1.0, 1.0, 1.0]}+with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ cfg_only.load_calibration(calib_path)+check_true(+ "load/config mismatch warns",+ any("config identity differs" in str(w.message) for w in caught),+ [str(w.message) for w in caught],+)+check("load/config mismatch still applies", cfg_only.temperature, agent.temperature)++# module helpers round-trip on a stub+stub = type("Stub", (), {})()+apply_calibration_payload(stub, calibration_payload([1.2, 1.1, 1.3], {"choice:2": 1.4}))+check("stub/temperature", stub.temperature, [1.2, 1.1, 1.3])+check("stub/by_options", stub.temperature_by_options, {"choice:2": 1.4})++# A payload with no version is the original schema and must still load, even onto an+# agent that has its own identity. No warning: there is no recorded checkpoint to disagree with.+legacy = {"temperature": [1.4, 1.2, 1.1], "temperature_by_options": {"noul:2": 1.5}}+legacy_agent = Agent.__new__(Agent)+legacy_agent.model_id_or_path = "convaiinnovations/laya"+legacy_agent.subfolder = None+legacy_agent.cfg = {"encoder": "answerdotai/ModernBERT-large"}+with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ apply_calibration_payload(legacy_agent, legacy)+check("version/fallback temperature", legacy_agent.temperature, [1.4, 1.2, 1.1])+check("version/fallback by_options", legacy_agent.temperature_by_options, {"noul:2": 1.5})+check_true("version/missing is silent", not caught, [str(w.message) for w in caught])++# Invalid entries clamp instead of crashing the load (#142, same guard as a checkpoint).+clamped = Agent.__new__(Agent)+clamped.model_id_or_path = "convaiinnovations/laya"+clamped.subfolder = None+clamped.cfg = {"encoder": "answerdotai/ModernBERT-large"}+bad_payload = {+ "version": CALIBRATION_VERSION,+ "temperature": [0.1, "nope", 9],+ "temperature_by_options": {"choice:2": 0, "noul:2": None},+ "model_id_or_path": "convaiinnovations/laya",+ "subfolder": None,+ "config": {"encoder": "answerdotai/ModernBERT-large"},+}+with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ apply_calibration_payload(clamped, bad_payload)+check("clamp-load/temperature", clamped.temperature, [TEMP_MIN, 1.0, TEMP_MAX])+check("clamp-load/by_options", clamped.temperature_by_options, {"choice:2": TEMP_MIN, "noul:2": 1.0})+check_true(+ "clamp-load/warns",+ any(issubclass(w.category, RuntimeWarning) and "choice:2" in str(w.message) for w in caught),+ [str(w.message) for w in caught],+)+bad_path = os.path.join(td, "bad.json")+Agent.save_calibration(clamped, bad_path)+with open(bad_path) as f:+ bad_saved = json.load(f)+check("clamp-load/saved temperature is applied", bad_saved["temperature"], [TEMP_MIN, 1.0, TEMP_MAX])+check(+ "clamp-load/saved buckets are applied",+ bad_saved["temperature_by_options"],+ {"choice:2": TEMP_MIN, "noul:2": 1.0},+)+++# --------------------------------------------------------------- constructor wiring (no Hub download)+init_src = inspect.getsource(Agent.__init__)+check_true("init/calibration kwarg", "calibration: Optional[str] = None" in init_src)+check_true("init/load_calibration call", "self.load_calibration(calibration)" in init_src)+check_true("init/does not write safetensors", "save_file" not in init_src)+check_true("init/stores model_id_or_path", "self.model_id_or_path = model_id_or_path" in init_src)+check_true("init/stores subfolder", "self.subfolder = subfolder" in init_src)+load_src = inspect.getsource(load)+check_true("load/calibration kwarg", "calibration" in load_src)+check_true("load/forwards calibration", "calibration=calibration" in load_src)++rec_src = inspect.getsource(records_from_labeled)+check_true("records/_encode_state", "_encode_state" in rec_src)+check_true("records/_decode_answers path", "_decode_answers" in rec_src)+check_true("records/no temp_bucket divide", "temp_bucket" not in rec_src)+check_true("records/_option_logits", "_option_logits" in rec_src)+check_true("decode/uses _option_logits", "_option_logits" in inspect.getsource(Agent._decode_answers))+sys_one = inspect.getsource(Agent.system_one)+check_true("system_one/uses predict_batch", "predict_batch" in sys_one)+check_true("system_one/does not fit", "fit_temperature" not in sys_one)+batch_src = inspect.getsource(Agent.predict_batch)+check_true("predict_batch/_encode_state", "_encode_state" in batch_src)+check_true("predict_batch/_decode_answers", "_decode_answers" in batch_src)+++# --------------------------------------------------------------- labeled records use raw logits+class _RecAgent:+ """Stand-in encoder: real question checks, fake forward, temperatures that must not scale logits."""++ temperature = [9.0, 9.0, 9.0]+ temperature_by_options = {"choice:2": 9.0}+ tok = type("Tok", (), {"pad_token_id": 0})()+ calls = []++ def _check_question(self, qid, qdef):+ Agent._check_question(qid, qdef)++ def _to_internal(self, qdef):+ return Agent._to_internal(qdef)++ def _encode_state(self, state, ids, internal):+ self.calls.append(state)+ items = []+ for qid in ids:+ q = internal[qid]+ if q["t"] == "noul":+ k = 2+ else:+ k = len(q["crit"])+ items.append({"ids": [4, 5, 6], "markers": list(range(k)), "qtype": QTYPES[q["t"]]})+ return items++ def _forward(self, batch):+ n = int(batch["input_ids"].shape[0])+ kmax = int(batch["marker_pos"].shape[1])+ logits = np.full((n, kmax), 4.0, dtype=np.float32)+ for i in range(n):+ logits[i, 0] = 1.0 + i+ act = np.zeros((n, 2), dtype=np.float32)+ return logits, act+++rec_agent = _RecAgent()+questions = {+ "c": {"type": "choice", "instructions": "pick", "criteria": ["a", "b"]},+ "n": {"type": "noul", "instructions": "yes?"},+}+targets = {+ "c": np.array([1.0, 0.0], dtype=np.float32),+ "n": np.array([0.0, 1.0], dtype=np.float32),+}+labeled = records_from_labeled(rec_agent, [("hello", questions, targets)])+check("records/state reaches _encode_state", rec_agent.calls, ["hello"])+check("records/choice qtype", labeled[0][0], QTYPES["choice"])+check("records/choice k", labeled[0][3], 2)+check("records/choice logits unscaled", labeled[0][1].tolist(), [1.0, 4.0])+check("records/choice target", labeled[0][2].tolist(), [1.0, 0.0])+check("records/noul qtype", labeled[1][0], QTYPES["noul"])+check("records/noul k", labeled[1][3], 2)+check("records/noul logits unscaled", labeled[1][1].tolist(), [2.0, 4.0])+++# public exports stay lazy: importing the package must not import torch for these names+import laya as _laya # noqa: E402+for name in ("fit_temperatures", "fit_one_temperature", "fit_temperature_map"):+ check_true("export/%s in __all__" % name, name in _laya.__all__)+ check_true("export/%s attr" % name, hasattr(_laya, name))+ check(+ "export/%s lazy table" % name,+ _laya._LAZY_ATTRS.get(name),+ (".calibrate", name),+ )+++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL " + f)+if not FAIL:+ print("synthetic ECE before=%.4f after=%.4f choice:2 T=%.3f noul:2 T=%.3f choice:11+ T=%.3f" % (+ report["ece_before"], report["ece_after"],+ fitted["temperature_by_options"]["choice:2"],+ fitted["temperature_by_options"]["noul:2"],+ fitted["temperature_by_options"]["choice:11+"],+ ))+sys.exit(1 if FAIL else 0)