Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(calibrate): fit, save and load per-bucket temperatures

Matt Van Hornd29b918b0809
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7a9cca5f..5822a576 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -118,6 +118,7 @@ jobs:
python tests/test_identifier_complexity.py
python tests/test_lang_stats.py
python tests/test_calibration_persistence.py
+ python tests/test_calibrate.py
python tests/test_context_manager.py
python tests/test_criteria_normalization.py
python tests/test_docker_entrypoint.py
@@ -236,7 +237,7 @@ jobs:
for t in test_router test_criteria test_download test_shortlist \
test_decision_model test_packaging test_lang_guess \
test_identifier_complexity \
- test_calibration_persistence test_context_manager \
+ test_calibration_persistence test_calibrate test_context_manager \
test_criteria_normalization test_docker_entrypoint \
test_empty_questions test_router_memory test_shortlist_cosine \
test_temperature_loading; do
diff --git a/laya/__init__.py b/laya/__init__.py
index b6462e54..4f9e2e38 100644
--- a/laya/__init__.py
+++ b/laya/__init__.py
@@ -23,6 +23,9 @@ _LAZY_ATTRS = {
"Agent": (".agent", "Agent"),
"RLAgent": (".agent", "RLAgent"),
"load": (".agent", "load"),
+ "fit_temperatures": (".calibrate", "fit_temperatures"),
+ "fit_one_temperature": (".calibrate", "fit_one_temperature"),
+ "fit_temperature_map": (".calibrate", "fit_temperature_map"),
"proper_reward": (".common", "proper_reward"),
"td_lambda_targets": (".common", "td_lambda_targets"),
"ece_score": (".common", "ece_score"),
@@ -63,6 +66,9 @@ __all__ = [
"Agent",
"RLAgent",
"load",
+ "fit_temperatures",
+ "fit_one_temperature",
+ "fit_temperature_map",
"Router",
"RouteDecision",
"DEFAULT_MODELS",
diff --git a/laya/agent.py b/laya/agent.py
index 1b58de6f..49f96ddd 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -11,6 +11,12 @@ from typing import Any, Dict, List, Optional, Union
import numpy as np
import torch
+from .calibrate import (
+ _install_temperatures,
+ apply_calibration_payload,
+ calibration_payload,
+ fit_temperature_map,
+)
from .common import (
QTYPES,
TEMP_MAX,
@@ -216,6 +222,15 @@ def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype:
return amp_dtype(checkpoint_default)
+def _option_logits(logits, items, offset):
+ """Raw per-option logits, the rows `_decode_answers` divides by temperature.
+
+ Calibration record collection slices with this same helper, so a fitted map sees the
+ option width the decoder scales and not a second tokenization of the state.
+ """
+ return [logits[offset + j, : len(item["markers"])] for j, item in enumerate(items)]
+
+
class Agent(HookRegistry):
"""System 1 decision model runtime: fast, non-autoregressive, calibrated decisions."""
@@ -250,6 +265,7 @@ class Agent(HookRegistry):
hooks_raise: bool = True,
hooks_concurrent: bool = True,
hooks_timeout: Optional[float] = None,
+ calibration: Optional[str] = None,
):
"""Load a Laya checkpoint.
@@ -267,6 +283,10 @@ class Agent(HookRegistry):
`Agent("convaiinnovations/laya", subfolder="multilingual")`. Only that subfolder is
downloaded, so bundling does not cost every user the whole family.
+ `calibration` is an optional JSON path with `temperature` and `temperature_by_options`.
+ It is applied after the checkpoint config, so a fitted map overrides shipped scalars
+ without rewriting `model.safetensors`.
+
`hooks` / `on_predict_start` / `on_predict_end` observe or shape every prediction; see
`laya.hooks`. `hooks_raise=False` warns and continues when a hook fails,
`hooks_concurrent=False` serialises hooks that are not safe to run in parallel, and
@@ -279,6 +299,9 @@ class Agent(HookRegistry):
self._hooks_lock = threading.RLock() if not hooks_concurrent else None
self._hooks_mutex = threading.Lock()
self.model_id = model_id_or_path
+ # Retained so `save_calibration` can record which checkpoint the map was fitted for.
+ self.model_id_or_path = model_id_or_path
+ self.subfolder = subfolder
from safetensors.torch import load_file
@@ -424,6 +447,8 @@ class Agent(HookRegistry):
"using %s. Treat confidence from the affected entries as uncalibrated."
% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
RuntimeWarning, stacklevel=2)
+ if calibration:
+ self.load_calibration(calibration)
# Autocast policy. CUDA, MPS and XPU all support fp16/bf16 autocast and the shipped
# checkpoints are trained in reduced precision; on CUDA the checkpoint's `amp_dtype`
# (bf16) is the default and LAYA_CUDA_AMP=fp16|bf16 overrides it. CPU bf16 is only a win
@@ -760,6 +785,7 @@ class Agent(HookRegistry):
internal: Dict[str, Dict], offset: int, lang: Optional[str] = None) -> Dict[str, Any]:
"""Turn one state's logit rows (starting at `offset`) into typed answers."""
answers = {}
+ raw_rows = _option_logits(logits, items, offset)
for j, qid in enumerate(ids):
r = offset + j
q = internal[qid]
@@ -769,7 +795,7 @@ class Agent(HookRegistry):
if lang and lang.split("-")[0].lower() in self.lang_temperatures:
l_cfg = self.lang_temperatures[lang.split("-")[0].lower()]
t_scale = l_cfg["temperature_by_options"].get(temp_bucket(qt, k), l_cfg["temperature"][qt])
- z = logits[r, :k] / t_scale
+ z = raw_rows[j] / t_scale
p = np.exp(z - z.max())
p = p / p.sum()
@@ -1126,6 +1152,44 @@ class Agent(HookRegistry):
predict = system_one
+ def fit_temperatures(self, records, compute_ece: bool = False, seed: int = 0) -> Dict[str, Any]:
+ """Fit per-bucket temperatures from CPU records and store them on this agent.
+
+ `records` are `(qtype, logits, target, k)`. Build them with
+ `laya.calibrate.records_from_labeled` when you have labeled forwards; this method
+ does not download weights or write `model.safetensors`. `seed` only affects the
+ held-out ECE split when `compute_ece` is true. The checkpoint `cfg` is left as loaded.
+ """
+ result = fit_temperature_map(records, compute_ece=compute_ece, seed=seed)
+ # Already clamped inside the fitter; don't report that as a bad calibration file.
+ _install_temperatures(self, result["temperature"], result["temperature_by_options"], warn=False)
+ return result
+
+ def save_calibration(self, path: str) -> None:
+ """Write temperatures and the checkpoint they were fitted for. Does not write weights."""
+ payload = calibration_payload(
+ self.temperature,
+ self.temperature_by_options,
+ model_id_or_path=getattr(self, "model_id_or_path", None),
+ subfolder=getattr(self, "subfolder", None),
+ config=getattr(self, "cfg", None),
+ )
+ with open(path, "w") as f:
+ json.dump(payload, f, indent=2)
+ f.write("\n")
+
+ def load_calibration(self, path: str) -> None:
+ """Read a JSON map written by `save_calibration` onto this agent.
+
+ A file with no `version` is treated as version 1 and still loads. A newer file
+ whose recorded checkpoint does not match this agent warns and still loads.
+ Values that are not numbers, or that sit outside `[TEMP_MIN, TEMP_MAX]`, are clamped
+ with `clamp_temperature` the same way checkpoint load is.
+ """
+ with open(path) as f:
+ payload = json.load(f)
+ apply_calibration_payload(self, payload)
+
RLAgent = Agent
@@ -1136,7 +1200,8 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]
lang_temperatures: Optional[Dict[str, Dict[str, Any]]] = None,
hooks=None, on_predict_start=None, on_predict_end=None,
hooks_raise: bool = True, hooks_concurrent: bool = True,
- hooks_timeout: Optional[float] = None) -> Agent:
+ hooks_timeout: Optional[float] = None,
+ calibration: Optional[str] = None) -> Agent:
"""Load a Laya agent.
`subfolder` picks one checkpoint out of a repo that bundles several:
@@ -1147,11 +1212,11 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]
`revision`/`expected_sha256` pin and verify the downloaded artifacts; see `Agent`.
`hooks` / `on_predict_start` / `on_predict_end` observe or shape every prediction; see
- `laya.hooks`.
+ `laya.hooks`. `calibration` is the same optional JSON path accepted by `Agent`.
"""
return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, fast=fast,
revision=revision, expected_sha256=expected_sha256,
lang_temperatures=lang_temperatures,
hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,
hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent,
- hooks_timeout=hooks_timeout)
+ hooks_timeout=hooks_timeout, calibration=calibration)
diff --git a/laya/calibrate.py b/laya/calibrate.py
new file mode 100644
index 00000000..cd437755
--- /dev/null
+++ b/laya/calibrate.py
@@ -0,0 +1,410 @@
+"""Fit and persist per-bucket temperatures for Laya logits.
+
+Ports the notebook LBFGS fitter (NLL on softmax(z/T)) and groups records by
+`temp_bucket(qtype, k)` so the runtime's `temperature_by_options` lookup has a writer.
+Fitting is CPU-side: records are `(qtype, logits, target, k)` and do not load Hub weights.
+
+Two sample floors, on purpose:
+
+- `MIN_BUCKET_N` is the per-bucket floor. Buckets with fewer examples are omitted from
+ `temperature_by_options`; the type-level scalar covers them at lookup time.
+- `MIN_TYPE_N` is the lower floor used only for that type-level scalar. It stays at the
+ previous floor of 10 so a realistic dataset that fills no single bucket still gets a
+ temperature instead of remaining at 1.0. Do not raise it to `MIN_BUCKET_N`.
+
+Fitted temperatures go through `clamp_temperature` and therefore `[TEMP_MIN, TEMP_MAX]`
+from `laya.common`, the same guard checkpoint load uses. There is no second pair of
+bounds in this module.
+"""
+import warnings
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
+
+import numpy as np
+import torch
+
+from .common import (
+ QTYPES,
+ TEMP_MAX,
+ TEMP_MIN,
+ clamp_temperature,
+ collate_items,
+ ece_score,
+ temp_bucket,
+)
+
+# Per-bucket floor. Below this, `temperature_by_options` omits the bucket.
+MIN_BUCKET_N = 2000
+# Type-level floor only. Lower than MIN_BUCKET_N; see the module docstring.
+MIN_TYPE_N = 10
+# Fraction of each bucket held out when `compute_ece=True`. Unused otherwise.
+ECE_HOLDOUT_FRAC = 0.2
+# Payload schema. Files written before identity was recorded have no version key;
+# readers treat that absence as version 1.
+CALIBRATION_VERSION = 2
+N_QTYPES = len(QTYPES)
+
+# (qtype:int, logits:1d, target:1d, k:int)
+Record = Tuple[int, Any, Any, int]
+
+_CONFIG_IDENTITY_SKIP = ("temperature", "temperature_by_options")
+
+
+def _vec(x) -> np.ndarray:
+ if isinstance(x, torch.Tensor):
+ x = x.detach().cpu().numpy()
+ return np.asarray(x, dtype=np.float32).reshape(-1)
+
+
+def _pairs_to_tensors(pairs: Sequence) -> Tuple[torch.Tensor, torch.Tensor]:
+ kmax = max(len(_vec(z)) for z, _ in pairs)
+ z_mat = torch.full((len(pairs), kmax), -1e4)
+ t_mat = torch.zeros((len(pairs), kmax))
+ for i, (z, t) in enumerate(pairs):
+ z = _vec(z)
+ t = _vec(t)
+ n = min(len(z), len(t))
+ z_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(z[:n]))
+ t_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(t[:n]))
+ return z_mat, t_mat
+
+
+def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
+ """Fit one scalar T by NLL + LBFGS on log T.
+
+ The result is `clamp_temperature` of the optimised scale, so it lies in
+ `[TEMP_MIN, TEMP_MAX]` (or is the neutral 1.0 when the value is not a number).
+ Returns 1.0 when fewer than `min_n` pairs are given. `min_n` defaults to
+ `MIN_BUCKET_N` (the per-bucket floor). Type-level fits pass `MIN_TYPE_N`, which
+ is lower, so a dataset that fills no bucket still gets a scalar instead of
+ staying at 1.0.
+ """
+ if min_n is None:
+ min_n = MIN_BUCKET_N
+ sel = list(pairs)
+ if len(sel) < min_n:
+ return 1.0
+ z_mat, t_mat = _pairs_to_tensors(sel)
+ with torch.enable_grad():
+ log_t = torch.zeros(1, requires_grad=True)
+ opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)
+
+ def closure():
+ opt.zero_grad()
+ loss = -(t_mat * torch.log_softmax(z_mat / log_t.exp(), -1)).sum(-1).mean()
+ loss.backward()
+ return loss
+
+ opt.step(closure)
+ fitted = float(log_t.detach().exp().item())
+ return clamp_temperature(fitted, TEMP_MIN, TEMP_MAX)
+
+
+def _iter_records(records: Iterable) -> List[Tuple[int, np.ndarray, np.ndarray, int]]:
+ out = []
+ for rec in records:
+ if len(rec) == 4:
+ qtype, logits, target, k = rec
+ elif len(rec) == 3:
+ qtype, logits, target = rec
+ k = len(_vec(logits))
+ else:
+ raise ValueError("record must be (qtype, logits, target[, k])")
+ logits = _vec(logits)
+ target = _vec(target)
+ k = int(k)
+ if k < 1:
+ raise ValueError("k must be >= 1")
+ out.append((int(qtype), logits[:k], target[:k], k))
+ return out
+
+
+def _softmax(z, t_scale: float) -> np.ndarray:
+ z = np.asarray(z, dtype=np.float64) / max(1e-3, float(t_scale))
+ z = z - z.max()
+ p = np.exp(z)
+ return p / p.sum()
+
+
+def _ece_report(recs, temperature, temperature_by_options) -> Dict[str, Any]:
+ conf_b, cor_b, conf_a, cor_a = [], [], [], []
+ for qt, z, t, k in recs:
+ y = int(np.argmax(t[:k]))
+ p0 = _softmax(z[:k], 1.0)
+ conf_b.append(float(p0.max()))
+ cor_b.append(float(int(p0.argmax()) == y))
+ t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])
+ p1 = _softmax(z[:k], t_scale)
+ conf_a.append(float(p1.max()))
+ cor_a.append(float(int(p1.argmax()) == y))
+ return {
+ "ece_before": ece_score(np.asarray(conf_b), np.asarray(cor_b)),
+ "ece_after": ece_score(np.asarray(conf_a), np.asarray(cor_a)),
+ "n": float(len(recs)),
+ }
+
+
+def _group_records(recs):
+ by_type = {qt: [] for qt in range(N_QTYPES)}
+ by_bucket = {}
+ for qt, z, t, k in recs:
+ if qt in by_type:
+ by_type[qt].append((z, t))
+ key = temp_bucket(qt, k)
+ by_bucket.setdefault(key, []).append((z, t))
+ return by_type, by_bucket
+
+
+def _ece_split(recs, seed):
+ """Split records into fit and eval, stratified by `temp_bucket`.
+
+ Returns `(fit_recs, eval_recs, excluded_buckets)`. The split is a deterministic
+ partition (`numpy.random.RandomState(seed)`): every record is in exactly one side.
+ A bucket whose fit side would fall below `MIN_BUCKET_N` is not split; all of its
+ records stay in `fit_recs` and its key is listed in `excluded_buckets`.
+ """
+ by_bucket = {}
+ for i, rec in enumerate(recs):
+ qt, _z, _t, k = rec
+ by_bucket.setdefault(temp_bucket(qt, k), []).append(i)
+
+ rng = np.random.RandomState(seed)
+ fit_idx = []
+ eval_idx = []
+ excluded = []
+ for key in sorted(by_bucket):
+ idxs = by_bucket[key]
+ n = len(idxs)
+ n_eval = int(round(n * ECE_HOLDOUT_FRAC))
+ if n - n_eval < MIN_BUCKET_N:
+ fit_idx.extend(idxs)
+ excluded.append(key)
+ continue
+ perm = rng.permutation(n)
+ for j in perm[:n_eval]:
+ eval_idx.append(idxs[int(j)])
+ for j in perm[n_eval:]:
+ fit_idx.append(idxs[int(j)])
+
+ fit_idx.sort()
+ eval_idx.sort()
+ fit_recs = [recs[i] for i in fit_idx]
+ eval_recs = [recs[i] for i in eval_idx]
+ return fit_recs, eval_recs, excluded
+
+
+def _fit_groups(by_type, by_bucket):
+ temperature = [1.0] * N_QTYPES
+ for qt in range(N_QTYPES):
+ pairs = by_type[qt]
+ # MIN_TYPE_N, not MIN_BUCKET_N: the scalar has to exist when every bucket is small.
+ if len(pairs) >= MIN_TYPE_N:
+ temperature[qt] = fit_one_temperature(pairs, min_n=MIN_TYPE_N)
+
+ temperature_by_options = {}
+ for key, pairs in by_bucket.items():
+ if len(pairs) < MIN_BUCKET_N:
+ continue
+ temperature_by_options[key] = fit_one_temperature(pairs)
+ return temperature, temperature_by_options
+
+
+def fit_temperature_map(records: Iterable, compute_ece: bool = False, seed: int = 0) -> Dict[str, Any]:
+ """Fit type-level scalars and per-bucket temperatures.
+
+ `MIN_BUCKET_N` is the per-bucket floor: smaller buckets are omitted from
+ `temperature_by_options` and the type-level scalar covers them. `MIN_TYPE_N` is
+ the separate, lower floor for that scalar only.
+
+ `compute_ece=False` (the default, and the path `Agent.fit_temperatures` stores)
+ fits on every record and returns no `report` key. `seed` is ignored on this path.
+
+ `compute_ece=True` holds out `ECE_HOLDOUT_FRAC` of each bucket, stratified by
+ `temp_bucket`, using `seed` so the same records always split the same way. Temperatures
+ are fit on the remainder only and ECE is scored only on the held-out records.
+ `report["n"]` is the number of records passed in; `report["n_eval"]` is the held-out
+ count the ECE rests on. A bucket that would drop below `MIN_BUCKET_N` after the
+ holdout is fit on all of its records, left out of the eval set, and named in
+ `report["buckets_excluded_from_eval"]` instead of being dropped. `n_by_bucket`
+ always counts the full input, including when the fit itself used a subset.
+ """
+ recs = _iter_records(records)
+ all_by_type, all_buckets = _group_records(recs)
+ n_by_bucket = {key: len(pairs) for key, pairs in all_buckets.items()}
+
+ if compute_ece:
+ _fit_recs, eval_recs, excluded = _ece_split(recs, seed)
+ by_type, by_bucket = _group_records(_fit_recs)
+ else:
+ eval_recs = None
+ excluded = []
+ by_type, by_bucket = all_by_type, all_buckets
+
+ temperature, temperature_by_options = _fit_groups(by_type, by_bucket)
+ out: Dict[str, Any] = {
+ "temperature": temperature,
+ "temperature_by_options": temperature_by_options,
+ "n_by_bucket": n_by_bucket,
+ }
+ if compute_ece:
+ report = _ece_report(eval_recs, temperature, temperature_by_options)
+ # `n` stays the size of the input. `n_eval` is the held-out count ECE used.
+ report["n"] = float(len(recs))
+ report["n_eval"] = float(len(eval_recs))
+ report["buckets_excluded_from_eval"] = excluded
+ out["report"] = report
+ return out
+
+
+fit_temperatures = fit_temperature_map
+
+
+def records_from_labeled(agent, pairs: Sequence) -> List[Record]:
+ """Collect CPU records from `(state, questions, targets)`.
+
+ Logits are the raw rows `_decode_answers` scales, not the probabilities it returns.
+ `_encode_state` builds one state's items, `_forward` runs them, and `_option_logits`
+ slices each question to the same marker width `_decode_answers` divides by temperature.
+ No Hub download and no `model.safetensors` write. Tests stay weight-free by passing
+ synthetic records to `fit_temperature_map` instead of calling this.
+ """
+ # Imported lazily: `agent` imports this module at load, and `_option_logits` is defined
+ # on the way through that import.
+ from .agent import _option_logits
+
+ records: List[Record] = []
+ for state, questions, targets in pairs:
+ ids = list(questions.keys())
+ if not ids:
+ continue
+ for qid in ids:
+ agent._check_question(qid, questions[qid])
+ internal = {qid: agent._to_internal(questions[qid]) for qid in ids}
+ items = agent._encode_state(state, ids, internal)
+ batch = collate_items([items], agent.tok.pad_token_id)
+ if batch is None:
+ continue
+ logits, _act = agent._forward(batch)
+ rows = _option_logits(logits, items, 0)
+ for j, qid in enumerate(ids):
+ k = len(items[j]["markers"])
+ qt = int(items[j]["qtype"])
+ records.append((qt, _vec(rows[j]), _vec(targets[qid])[:k], k))
+ return records
+
+
+def _config_identity(cfg):
+ """Checkpoint config without the temperature fields this file itself stores."""
+ if cfg is None:
+ return None
+ return {str(k): cfg[k] for k in cfg if k not in _CONFIG_IDENTITY_SKIP}
+
+
+def calibration_payload(
+ temperature,
+ temperature_by_options,
+ model_id_or_path=None,
+ subfolder=None,
+ config=None,
+) -> Dict[str, Any]:
+ """JSON body written by `Agent.save_calibration` (no weights).
+
+ `version` is `CALIBRATION_VERSION`. Payloads written before identity was recorded have
+ no version key; `apply_calibration_payload` treats that as version 1 and still loads.
+
+ `model_id_or_path`, `subfolder`, and `config` say which checkpoint the map was fitted
+ against. `config` is `rl_agent_config.json` without `temperature` /
+ `temperature_by_options` (those live at the top of this payload).
+ """
+ return {
+ "version": CALIBRATION_VERSION,
+ "temperature": [float(x) for x in temperature],
+ "temperature_by_options": {
+ str(k): float(v) for k, v in dict(temperature_by_options).items()
+ },
+ "model_id_or_path": model_id_or_path,
+ "subfolder": subfolder,
+ "config": _config_identity(config),
+ }
+
+
+def _warn_if_identity_mismatch(obj, payload) -> None:
+ recorded_model = payload.get("model_id_or_path")
+ recorded_sub = payload.get("subfolder")
+ recorded_cfg = payload.get("config")
+ current_model = getattr(obj, "model_id_or_path", None)
+ current_sub = getattr(obj, "subfolder", None)
+ current_cfg = _config_identity(getattr(obj, "cfg", None))
+ if (recorded_model, recorded_sub, recorded_cfg) == (current_model, current_sub, current_cfg):
+ return
+ cfg_note = ""
+ if recorded_cfg != current_cfg:
+ cfg_note = " Checkpoint config identity differs."
+ warnings.warn(
+ "calibration was fitted for model_id_or_path=%r subfolder=%r but this agent is "
+ "model_id_or_path=%r subfolder=%r.%s Loading the temperatures anyway."
+ % (recorded_model, recorded_sub, current_model, current_sub, cfg_note),
+ UserWarning,
+ stacklevel=3,
+ )
+
+
+def _rejected_temperatures(entries):
+ """Names whose raw value is not the number `clamp_temperature` kept, matching checkpoint load."""
+ rejected = []
+ for name, raw, applied in entries:
+ try:
+ if float(raw) == applied:
+ continue
+ except (TypeError, ValueError):
+ pass
+ rejected.append("%s=%r -> %g" % (name, raw, applied))
+ return rejected
+
+
+def _install_temperatures(obj, temperature, temperature_by_options, warn: bool = True) -> None:
+ """Store a map the way checkpoint load does: raw values, and the clamped ones inference reads.
+
+ `warn=False` is the fitter path: those values were already passed through `clamp_temperature`,
+ so a second pass must not report them as a bad file.
+ """
+ raw_t = list(temperature)
+ applied_t = [clamp_temperature(t) for t in raw_t]
+ raw_b = {str(k): v for k, v in dict(temperature_by_options).items()}
+ applied_b = {k: clamp_temperature(v) for k, v in raw_b.items()}
+ obj.temperature_raw = raw_t
+ obj.temperature = applied_t
+ obj.temperature_by_options_raw = raw_b
+ obj.temperature_by_options = applied_b
+ if not warn:
+ return
+ entries = [("temperature[%d]" % i, raw, applied_t[i]) for i, raw in enumerate(raw_t)]
+ entries += [(key, raw_b[key], applied_b[key]) for key in raw_b]
+ rejected = _rejected_temperatures(entries)
+ if rejected:
+ warnings.warn(
+ "laya: calibration temperatures are invalid or outside [%g, %g]; "
+ "using %s. Treat confidence from the affected entries as uncalibrated."
+ % (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
+ RuntimeWarning,
+ stacklevel=3,
+ )
+
+
+def apply_calibration_payload(obj, payload: Dict[str, Any]) -> None:
+ """Copy `temperature` and `temperature_by_options` from a calibration payload onto `obj`.
+
+ A missing `version` is version 1 (temperatures only, no checkpoint identity). Version
+ `CALIBRATION_VERSION` records the checkpoint the map was fitted for; a mismatch warns
+ and still loads, so an older file never becomes a hard failure. Each value is passed
+ through `clamp_temperature`, so a non-numeric or out-of-range entry cannot crash a later
+ forward the way an unclamped zero used to.
+ """
+ version = payload.get("version", 1)
+ if version is None:
+ version = 1
+ temps = payload.get("temperature")
+ if temps is None or len(temps) != N_QTYPES:
+ raise ValueError("calibration JSON must contain temperature: [3 floats]")
+ if int(version) >= CALIBRATION_VERSION:
+ _warn_if_identity_mismatch(obj, payload)
+ _install_temperatures(obj, temps, payload.get("temperature_by_options") or {})
diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py
new file mode 100644
index 00000000..7fe6a717
--- /dev/null
+++ b/tests/test_calibrate.py
@@ -0,0 +1,458 @@
+"""Per-bucket temperature fitting. No model weights are loaded.
+
+Records are CPU tuples (qtype, logits, target, k). The fitter is the notebook's NLL+LBFGS
+on log T, grouped by `temp_bucket`. Bounds are `TEMP_MIN` / `TEMP_MAX` via `clamp_temperature`.
+Logits for a labeled agent come from `_encode_state` and the raw rows `_decode_answers` scales.
+"""
+import inspect
+import json
+import os
+import sys
+import tempfile
+import warnings
+
+import numpy as np
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.agent import Agent, load # noqa: E402
+from laya.calibrate import ( # noqa: E402
+ CALIBRATION_VERSION,
+ ECE_HOLDOUT_FRAC,
+ MIN_BUCKET_N,
+ MIN_TYPE_N,
+ _ece_report,
+ _ece_split,
+ _iter_records,
+ apply_calibration_payload,
+ calibration_payload,
+ fit_one_temperature,
+ fit_temperature_map,
+ fit_temperatures,
+ records_from_labeled,
+)
+from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, temp_bucket # noqa: E402
+import laya.calibrate as _calibrate # noqa: E402
+
+PASS, FAIL = [], []
+
+# Large enough that ECE_HOLDOUT_FRAC still leaves MIN_BUCKET_N rows to fit.
+BUCKET_N = 2500
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s %s" % (name, detail))
+
+
+def peaked(k, idx, mag):
+ z = np.full(k, -mag, dtype=np.float32)
+ z[idx] = mag
+ t = np.zeros(k, dtype=np.float32)
+ t[idx] = 1.0
+ return z, t
+
+
+def overconfident_records(n, k, qtype, seed=0, acc=0.6, mag=8.0):
+ rng = np.random.RandomState(seed)
+ recs = []
+ for i in range(n):
+ y = int(rng.randint(0, k))
+ pred = y if rng.rand() < acc else int((y + 1) % k)
+ z = np.full(k, -mag, dtype=np.float32)
+ z[pred] = mag
+ t = np.zeros(k, dtype=np.float32)
+ t[y] = 1.0
+ recs.append((qtype, z, t, k))
+ return recs
+
+
+check("floor/MIN_BUCKET_N", MIN_BUCKET_N, 2000)
+check("floor/MIN_TYPE_N", MIN_TYPE_N, 10)
+check("clamp/TEMP_MIN", TEMP_MIN, 0.5)
+check("clamp/TEMP_MAX", TEMP_MAX, 5.0)
+check_true("clamp/no local TEMP_LO", not hasattr(_calibrate, "TEMP_LO"))
+check_true("clamp/no local TEMP_HI", not hasattr(_calibrate, "TEMP_HI"))
+check_true(
+ "setup/holdout leaves a fittable bucket",
+ BUCKET_N - int(round(BUCKET_N * ECE_HOLDOUT_FRAC)) >= MIN_BUCKET_N,
+)
+
+
+# --------------------------------------------------------------- overconfident -> T > 1
+choice_k2 = overconfident_records(BUCKET_N, 2, QTYPES["choice"], seed=1, acc=0.6, mag=8.0)
+t_choice = fit_one_temperature([(z, t) for _, z, t, _ in choice_k2])
+check_true("overconfident/T>1", t_choice > 1.0, "T=%r" % t_choice)
+check_true("overconfident/T<=TEMP_MAX", t_choice <= TEMP_MAX, "T=%r" % t_choice)
+
+
+# --------------------------------------------------------------- bucket keys + omit n<MIN_BUCKET_N
+noul_k2 = overconfident_records(BUCKET_N, 2, QTYPES["noul"], seed=2, acc=0.6, mag=8.0)
+choice_k12 = overconfident_records(BUCKET_N, 12, QTYPES["choice"], seed=3, acc=0.6, mag=8.0)
+# Exactly the per-bucket floor: fitted, but too small to hold any rows out for ECE.
+choice_k4 = overconfident_records(MIN_BUCKET_N, 4, QTYPES["choice"], seed=6, acc=0.6, mag=8.0)
+tiny_score = overconfident_records(5, 4, QTYPES["score"], seed=4, acc=0.6, mag=8.0)
+mixed = choice_k2 + noul_k2 + choice_k12 + choice_k4 + tiny_score
+fitted = fit_temperature_map(mixed, compute_ece=True)
+keys = set(fitted["temperature_by_options"])
+check_true("keys/choice:2", "choice:2" in keys, keys)
+check_true("keys/noul:2", "noul:2" in keys, keys)
+check_true("keys/choice:11+", "choice:11+" in keys, keys)
+check_true("keys/choice:3-5 at floor", "choice:3-5" in keys, keys)
+check("keys/temp_bucket choice:2", temp_bucket(QTYPES["choice"], 2), "choice:2")
+check("keys/temp_bucket noul:2", temp_bucket(QTYPES["noul"], 2), "noul:2")
+check("keys/temp_bucket choice:11+", temp_bucket(QTYPES["choice"], 12), "choice:11+")
+check_true("omit/n<MIN_BUCKET_N score:3-5 not in map", "score:3-5" not in keys, keys)
+check("omit/n_by_bucket still counts tiny", fitted["n_by_bucket"].get("score:3-5"), 5)
+check("omit/n_by_bucket counts floor bucket", fitted["n_by_bucket"].get("choice:3-5"), MIN_BUCKET_N)
+check_true("omit/not NaN", all(np.isfinite(v) for v in fitted["temperature_by_options"].values()))
+check_true("alias/fit_temperatures is fit_temperature_map", fit_temperatures is fit_temperature_map)
+
+
+# --------------------------------------------------------------- type floor stays below the bucket floor
+# 50 is above MIN_TYPE_N and below MIN_BUCKET_N, so the scalar moves and the bucket is omitted.
+medium_score = overconfident_records(50, 4, QTYPES["score"], seed=5, acc=0.6, mag=8.0)
+med = fit_temperature_map(medium_score, compute_ece=False, seed=0)
+med_other_seed = fit_temperature_map(medium_score, compute_ece=False, seed=1)
+check_true(
+ "type-floor/fitted below bucket floor",
+ med["temperature"][QTYPES["score"]] > 1.0,
+ med["temperature"],
+)
+check_true("type-floor/bucket omitted", "score:3-5" not in med["temperature_by_options"])
+check_true("ece/off has no report", "report" not in med)
+check("ece/off ignores seed", med["temperature"], med_other_seed["temperature"])
+
+
+# --------------------------------------------------------------- clamp 0.5-5
+# 50% accurate, extreme logits: NLL wants a large T, clamped at TEMP_MAX.
+rng = np.random.RandomState(7)
+hi_pairs = []
+for i in range(MIN_BUCKET_N):
+ y = int(rng.randint(0, 2))
+ pred = y if i % 2 == 0 else 1 - y
+ z, t = peaked(2, pred, 80.0)
+ t = np.zeros(2, dtype=np.float32)
+ t[y] = 1.0
+ hi_pairs.append((z, t))
+t_hi = fit_one_temperature(hi_pairs)
+check_true("clamp/high in [0.5, 5]", TEMP_MIN <= t_hi <= TEMP_MAX, "T=%r" % t_hi)
+check_true("clamp/high at TEMP_MAX", abs(t_hi - TEMP_MAX) < 1e-4, "T=%r" % t_hi)
+
+# Always-correct, mild logits: NLL wants a small T, clamped at TEMP_MIN.
+lo_pairs = []
+for i in range(MIN_BUCKET_N):
+ z, t = peaked(2, i % 2, 0.3)
+ lo_pairs.append((z, t))
+t_lo = fit_one_temperature(lo_pairs)
+check_true("clamp/low in [0.5, 5]", TEMP_MIN <= t_lo <= TEMP_MAX, "T=%r" % t_lo)
+check_true("clamp/low at TEMP_MIN", abs(t_lo - TEMP_MIN) < 1e-4, "T=%r" % t_lo)
+
+src = inspect.getsource(fit_one_temperature)
+check_true("clamp/source uses clamp_temperature", "clamp_temperature" in src)
+check_true("clamp/source uses TEMP_MIN", "TEMP_MIN" in src)
+check_true("clamp/source uses TEMP_MAX", "TEMP_MAX" in src)
+check_true("clamp/n<MIN_BUCKET_N returns 1.0", fit_one_temperature(lo_pairs[:5]) == 1.0)
+check_true(
+ "clamp/n=MIN_BUCKET_N-1 returns 1.0",
+ fit_one_temperature(lo_pairs[: MIN_BUCKET_N - 1]) == 1.0,
+)
+
+
+# --------------------------------------------------------------- ECE on a held-out split, not the fit rows
+report = fitted["report"]
+check_true("ece/after < before", report["ece_after"] < report["ece_before"],
+ "before=%r after=%r" % (report["ece_before"], report["ece_after"]))
+check_true("ece/before finite", np.isfinite(report["ece_before"]))
+check_true("ece/after finite", np.isfinite(report["ece_after"]))
+check("ece/n is the full input", report["n"], float(len(mixed)))
+
+parsed = _iter_records(mixed)
+fit_recs, eval_recs, excluded = _ece_split(parsed, 0)
+fit_ids = [id(r) for r in fit_recs]
+eval_ids = [id(r) for r in eval_recs]
+check_true("split/disjoint", set(fit_ids).isdisjoint(eval_ids))
+check("split/partition", len(fit_ids) + len(eval_ids), len(parsed))
+again_fit, again_eval, again_ex = _ece_split(parsed, 0)
+check("split/deterministic eval", [id(r) for r in again_eval], eval_ids)
+check("split/deterministic excluded", again_ex, excluded)
+other_fit, other_eval, _other_ex = _ece_split(parsed, 1)
+check_true("split/seed changes holdout", [id(r) for r in other_eval] != eval_ids)
+check_true("split/floor bucket excluded", "choice:3-5" in excluded, excluded)
+check_true("split/tiny excluded", "score:3-5" in excluded, excluded)
+check_true("split/big bucket held out", "choice:2" not in excluded, excluded)
+check("ece/n_eval", report["n_eval"], float(len(eval_recs)))
+check_true("ece/n_eval < n", report["n_eval"] < report["n"])
+check("ece/excluded noted", report["buckets_excluded_from_eval"], excluded)
+# The map itself is the one fit on the non-held-out rows.
+refit = fit_temperature_map(fit_recs, compute_ece=False)
+check("split/fit ignores eval buckets", fitted["temperature_by_options"], refit["temperature_by_options"])
+check("split/fit ignores eval types", fitted["temperature"], refit["temperature"])
+holdout_ece = _ece_report(eval_recs, fitted["temperature"], fitted["temperature_by_options"])
+check("ece/after is the holdout", report["ece_after"], holdout_ece["ece_after"])
+check("ece/before is the holdout", report["ece_before"], holdout_ece["ece_before"])
+full_ece = _ece_report(parsed, fitted["temperature"], fitted["temperature_by_options"])
+check_true(
+ "ece/not scored on the fit pool",
+ full_ece["ece_after"] != report["ece_after"],
+ "full=%r holdout=%r" % (full_ece["ece_after"], report["ece_after"]),
+)
+again_fit_map = fit_temperature_map(mixed, compute_ece=True, seed=0)
+check("ece/fit deterministic", again_fit_map["report"], fitted["report"])
+
+
+# --------------------------------------------------------------- live Agent map + JSON round-trip
+_CFG = {
+ "encoder": "answerdotai/ModernBERT-large",
+ "head_layers": 2,
+ "max_len": 512,
+ "model_name": "laya",
+ "temperature": [9.0, 9.0, 9.0],
+ "temperature_by_options": {"choice:2": 9.0},
+}
+agent = Agent.__new__(Agent)
+agent.temperature = [1.0, 1.0, 1.0]
+agent.temperature_by_options = {}
+agent.model_id_or_path = "convaiinnovations/laya"
+agent.subfolder = None
+agent.cfg = dict(_CFG)
+live = agent.fit_temperatures(mixed, compute_ece=True)
+check_true("agent/live temperature_by_options updates",
+ "choice:2" in agent.temperature_by_options, agent.temperature_by_options)
+check("agent/live matches result", agent.temperature_by_options, live["temperature_by_options"])
+check_true("agent/type-level choice T>1", agent.temperature[0] > 1.0, agent.temperature)
+check_true("agent/forwards seed", "seed=seed" in inspect.getsource(Agent.fit_temperatures))
+
+td = tempfile.mkdtemp()
+calib_path = os.path.join(td, "calibration.json")
+agent.save_calibration(calib_path)
+check_true("save/no model.safetensors", not os.path.exists(os.path.join(td, "model.safetensors")))
+check_true("save/only the json file", os.listdir(td) == ["calibration.json"], os.listdir(td))
+with open(calib_path) as f:
+ payload = json.load(f)
+check(
+ "save/keys",
+ sorted(payload.keys()),
+ ["config", "model_id_or_path", "subfolder", "temperature", "temperature_by_options", "version"],
+)
+check("save/version", payload["version"], CALIBRATION_VERSION)
+check("save/model_id", payload["model_id_or_path"], "convaiinnovations/laya")
+check("save/subfolder", payload["subfolder"], None)
+check("save/config encoder", payload["config"]["encoder"], "answerdotai/ModernBERT-large")
+check("save/config model_name", payload["config"]["model_name"], "laya")
+check_true("save/config omits temperature", "temperature" not in payload["config"])
+check_true(
+ "save/config omits temperature_by_options",
+ "temperature_by_options" not in payload["config"],
+)
+check_true("save/no weights key", "model.safetensors" not in json.dumps(payload))
+
+other = Agent.__new__(Agent)
+other.model_id_or_path = agent.model_id_or_path
+other.subfolder = agent.subfolder
+other.cfg = dict(_CFG)
+with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ other.load_calibration(calib_path)
+check_true("load/matching identity is silent", not caught, [str(w.message) for w in caught])
+check("load/temperature", other.temperature, agent.temperature)
+check("load/by_options", other.temperature_by_options, agent.temperature_by_options)
+
+mismatch = Agent.__new__(Agent)
+mismatch.model_id_or_path = "convaiinnovations/laya-multilingual"
+mismatch.subfolder = "multilingual"
+mismatch.cfg = {"encoder": "jhu-clsp/mmBERT-base", "head_layers": 2}
+with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ mismatch.load_calibration(calib_path)
+check_true(
+ "load/mismatch warns",
+ any(issubclass(w.category, UserWarning) for w in caught),
+ [str(w.message) for w in caught],
+)
+check("load/mismatch still applies temperature", mismatch.temperature, agent.temperature)
+check("load/mismatch still applies by_options", mismatch.temperature_by_options, agent.temperature_by_options)
+
+cfg_only = Agent.__new__(Agent)
+cfg_only.model_id_or_path = agent.model_id_or_path
+cfg_only.subfolder = agent.subfolder
+cfg_only.cfg = {"encoder": "some-other-encoder", "head_layers": 4, "temperature": [1.0, 1.0, 1.0]}
+with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ cfg_only.load_calibration(calib_path)
+check_true(
+ "load/config mismatch warns",
+ any("config identity differs" in str(w.message) for w in caught),
+ [str(w.message) for w in caught],
+)
+check("load/config mismatch still applies", cfg_only.temperature, agent.temperature)
+
+# module helpers round-trip on a stub
+stub = type("Stub", (), {})()
+apply_calibration_payload(stub, calibration_payload([1.2, 1.1, 1.3], {"choice:2": 1.4}))
+check("stub/temperature", stub.temperature, [1.2, 1.1, 1.3])
+check("stub/by_options", stub.temperature_by_options, {"choice:2": 1.4})
+
+# A payload with no version is the original schema and must still load, even onto an
+# agent that has its own identity. No warning: there is no recorded checkpoint to disagree with.
+legacy = {"temperature": [1.4, 1.2, 1.1], "temperature_by_options": {"noul:2": 1.5}}
+legacy_agent = Agent.__new__(Agent)
+legacy_agent.model_id_or_path = "convaiinnovations/laya"
+legacy_agent.subfolder = None
+legacy_agent.cfg = {"encoder": "answerdotai/ModernBERT-large"}
+with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ apply_calibration_payload(legacy_agent, legacy)
+check("version/fallback temperature", legacy_agent.temperature, [1.4, 1.2, 1.1])
+check("version/fallback by_options", legacy_agent.temperature_by_options, {"noul:2": 1.5})
+check_true("version/missing is silent", not caught, [str(w.message) for w in caught])
+
+# Invalid entries clamp instead of crashing the load (#142, same guard as a checkpoint).
+clamped = Agent.__new__(Agent)
+clamped.model_id_or_path = "convaiinnovations/laya"
+clamped.subfolder = None
+clamped.cfg = {"encoder": "answerdotai/ModernBERT-large"}
+bad_payload = {
+ "version": CALIBRATION_VERSION,
+ "temperature": [0.1, "nope", 9],
+ "temperature_by_options": {"choice:2": 0, "noul:2": None},
+ "model_id_or_path": "convaiinnovations/laya",
+ "subfolder": None,
+ "config": {"encoder": "answerdotai/ModernBERT-large"},
+}
+with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ apply_calibration_payload(clamped, bad_payload)
+check("clamp-load/temperature", clamped.temperature, [TEMP_MIN, 1.0, TEMP_MAX])
+check("clamp-load/by_options", clamped.temperature_by_options, {"choice:2": TEMP_MIN, "noul:2": 1.0})
+check_true(
+ "clamp-load/warns",
+ any(issubclass(w.category, RuntimeWarning) and "choice:2" in str(w.message) for w in caught),
+ [str(w.message) for w in caught],
+)
+bad_path = os.path.join(td, "bad.json")
+Agent.save_calibration(clamped, bad_path)
+with open(bad_path) as f:
+ bad_saved = json.load(f)
+check("clamp-load/saved temperature is applied", bad_saved["temperature"], [TEMP_MIN, 1.0, TEMP_MAX])
+check(
+ "clamp-load/saved buckets are applied",
+ bad_saved["temperature_by_options"],
+ {"choice:2": TEMP_MIN, "noul:2": 1.0},
+)
+
+
+# --------------------------------------------------------------- constructor wiring (no Hub download)
+init_src = inspect.getsource(Agent.__init__)
+check_true("init/calibration kwarg", "calibration: Optional[str] = None" in init_src)
+check_true("init/load_calibration call", "self.load_calibration(calibration)" in init_src)
+check_true("init/does not write safetensors", "save_file" not in init_src)
+check_true("init/stores model_id_or_path", "self.model_id_or_path = model_id_or_path" in init_src)
+check_true("init/stores subfolder", "self.subfolder = subfolder" in init_src)
+load_src = inspect.getsource(load)
+check_true("load/calibration kwarg", "calibration" in load_src)
+check_true("load/forwards calibration", "calibration=calibration" in load_src)
+
+rec_src = inspect.getsource(records_from_labeled)
+check_true("records/_encode_state", "_encode_state" in rec_src)
+check_true("records/_decode_answers path", "_decode_answers" in rec_src)
+check_true("records/no temp_bucket divide", "temp_bucket" not in rec_src)
+check_true("records/_option_logits", "_option_logits" in rec_src)
+check_true("decode/uses _option_logits", "_option_logits" in inspect.getsource(Agent._decode_answers))
+sys_one = inspect.getsource(Agent.system_one)
+check_true("system_one/uses predict_batch", "predict_batch" in sys_one)
+check_true("system_one/does not fit", "fit_temperature" not in sys_one)
+batch_src = inspect.getsource(Agent.predict_batch)
+check_true("predict_batch/_encode_state", "_encode_state" in batch_src)
+check_true("predict_batch/_decode_answers", "_decode_answers" in batch_src)
+
+
+# --------------------------------------------------------------- labeled records use raw logits
+class _RecAgent:
+ """Stand-in encoder: real question checks, fake forward, temperatures that must not scale logits."""
+
+ temperature = [9.0, 9.0, 9.0]
+ temperature_by_options = {"choice:2": 9.0}
+ tok = type("Tok", (), {"pad_token_id": 0})()
+ calls = []
+
+ def _check_question(self, qid, qdef):
+ Agent._check_question(qid, qdef)
+
+ def _to_internal(self, qdef):
+ return Agent._to_internal(qdef)
+
+ def _encode_state(self, state, ids, internal):
+ self.calls.append(state)
+ items = []
+ for qid in ids:
+ q = internal[qid]
+ if q["t"] == "noul":
+ k = 2
+ else:
+ k = len(q["crit"])
+ items.append({"ids": [4, 5, 6], "markers": list(range(k)), "qtype": QTYPES[q["t"]]})
+ return items
+
+ def _forward(self, batch):
+ n = int(batch["input_ids"].shape[0])
+ kmax = int(batch["marker_pos"].shape[1])
+ logits = np.full((n, kmax), 4.0, dtype=np.float32)
+ for i in range(n):
+ logits[i, 0] = 1.0 + i
+ act = np.zeros((n, 2), dtype=np.float32)
+ return logits, act
+
+
+rec_agent = _RecAgent()
+questions = {
+ "c": {"type": "choice", "instructions": "pick", "criteria": ["a", "b"]},
+ "n": {"type": "noul", "instructions": "yes?"},
+}
+targets = {
+ "c": np.array([1.0, 0.0], dtype=np.float32),
+ "n": np.array([0.0, 1.0], dtype=np.float32),
+}
+labeled = records_from_labeled(rec_agent, [("hello", questions, targets)])
+check("records/state reaches _encode_state", rec_agent.calls, ["hello"])
+check("records/choice qtype", labeled[0][0], QTYPES["choice"])
+check("records/choice k", labeled[0][3], 2)
+check("records/choice logits unscaled", labeled[0][1].tolist(), [1.0, 4.0])
+check("records/choice target", labeled[0][2].tolist(), [1.0, 0.0])
+check("records/noul qtype", labeled[1][0], QTYPES["noul"])
+check("records/noul k", labeled[1][3], 2)
+check("records/noul logits unscaled", labeled[1][1].tolist(), [2.0, 4.0])
+
+
+# public exports stay lazy: importing the package must not import torch for these names
+import laya as _laya # noqa: E402
+for name in ("fit_temperatures", "fit_one_temperature", "fit_temperature_map"):
+ check_true("export/%s in __all__" % name, name in _laya.__all__)
+ check_true("export/%s attr" % name, hasattr(_laya, name))
+ check(
+ "export/%s lazy table" % name,
+ _laya._LAZY_ATTRS.get(name),
+ (".calibrate", name),
+ )
+
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+if not FAIL:
+ print("synthetic ECE before=%.4f after=%.4f choice:2 T=%.3f noul:2 T=%.3f choice:11+ T=%.3f" % (
+ report["ece_before"], report["ece_after"],
+ fitted["temperature_by_options"]["choice:2"],
+ fitted["temperature_by_options"]["noul:2"],
+ fitted["temperature_by_options"]["choice:11+"],
+ ))
+sys.exit(1 if FAIL else 0)