Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Fix: run the checkpoints on transformers 4.x and on MPS (rebased #51) (#273)
Aashish Mahato194afbe63523
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 1a370b30..52ae7c16 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -78,6 +78,8 @@ jobs:python tests/test_cli.pypython tests/test_mcp.pypython tests/test_langchain.py+ python tests/test_portability.py+ python tests/test_training.py- name: Email cleaning testsrun: python tests/test_email.pydiff --git a/.gitignore b/.gitignoreindex 941841d1..3d264b23 100644--- a/.gitignore+++ b/.gitignore@@ -69,3 +69,7 @@ credentials.json# Documentation site build output (zensical build)site/++# ruff's cache, and the JSON the research harnesses write on every run+.ruff_cache/+research/*_benchmark_results.jsondiff --git a/laya/agent.py b/laya/agent.pyindex 36abee8c..99df622f 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -4,6 +4,7 @@ import osimport tempfileimport threadingimport warnings+from contextlib import nullcontextfrom typing import Any, Dict, List, Optional, Unionimport numpy as np@@ -163,6 +164,19 @@ def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any:return tokenizer+def _amp_context(device, dtype):+ """Autocast context for the forward pass, or a no-op when mixed precision is not in use.++ Autocast is a CUDA-only win here. Entering `torch.autocast` on a device torch has no+ autocast backend for raises even with `enabled=False` ('User specified an unsupported+ autocast device_type mps'), which broke every `predict()` call on the MPS GPU that torch+ selects automatically on Apple/AMD machines. Only wrap the forward pass when we use it.+ """+ if device.type == "cuda":+ return torch.autocast(device_type="cuda", dtype=dtype)+ return nullcontext()++class Agent:"""System 1 decision model runtime: fast, non-autoregressive, calibrated decisions."""@@ -485,8 +499,7 @@ class Agent:def _forward(self, b: Dict):"""Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""def run():- use_amp = self.device.type == "cuda"- with torch.autocast(device_type=self.device.type, dtype=self.dtype, enabled=use_amp):+ with _amp_context(self.device, self.dtype):return self.model(b["input_ids"].to(self.device),b["attention_mask"].to(self.device),diff --git a/laya/common.py b/laya/common.pyindex d73e248b..6d0859bb 100644--- a/laya/common.py+++ b/laya/common.py@@ -163,11 +163,35 @@ class DecisionModel(nn.Module):return logits, act_logits+def _apply_rope_config(ecfg) -> None:+ """Carry transformers>=5 per-layer RoPE settings over to the attributes 4.x reads.++ A checkpoint re-saved by transformers 5 stores RoPE as+ `rope_parameters = {"full_attention": {"rope_theta": ...}, "sliding_attention": {...}}`.+ transformers 4.x does not know that key, so it keeps its own defaults (global 160000,+ local 10000) and any checkpoint whose sliding-attention theta differs silently runs the+ wrong RoPE base -- mmBERT is exactly that case, both of its thetas are 160000. Map the+ values onto `global_rope_theta` / `local_rope_theta`, which 4.x does read. On+ transformers 5 this is a no-op beyond re-setting the same numbers.+ """+ rope = getattr(ecfg, "rope_parameters", None)+ if not isinstance(rope, dict):+ return+ flat = rope.get("rope_theta")+ for layer_type, attr in (("full_attention", "global_rope_theta"),+ ("sliding_attention", "local_rope_theta")):+ params = rope.get(layer_type)+ theta = params.get("rope_theta") if isinstance(params, dict) else flat+ if theta is not None and hasattr(ecfg, attr):+ setattr(ecfg, attr, float(theta))++def build_model(cfg: Dict, encoder_dir: Optional[str] = None, pretrained: bool = True) -> DecisionModel:from transformers import AutoConfig, AutoModelif not pretrained or (encoder_dir and os.path.exists(encoder_dir)):ecfg = AutoConfig.from_pretrained(encoder_dir or cfg["encoder"])+ _apply_rope_config(ecfg)enc = AutoModel.from_config(ecfg, attn_implementation="sdpa")else:enc = AutoModel.from_pretrained(cfg["encoder"], attn_implementation="sdpa")@@ -291,6 +315,13 @@ def collate_items(batch, pad_id: int):mpos[i, :k] = torch.tensor(it["markers"])mmask[i, :k] = Trueif has_target and "target" in it:+ if len(it["target"]) > kmax:+ # Otherwise this lands as "The expanded size of the tensor (k) must match the+ # existing size (kmax)" from inside the assignment, which says nothing about the+ # actual mistake: a target with more entries than the item has options.+ raise ValueError(+ "collate_items: item %d has %d target entries but only %d marker positions; "+ "a target needs one entry per option" % (i, len(it["target"]), kmax))target[i, : len(it["target"])] = torch.tensor(it["target"], dtype=torch.float32)res = {diff --git a/tests/test_portability.py b/tests/test_portability.pynew file mode 100644index 00000000..fffe24b3--- /dev/null+++ b/tests/test_portability.py@@ -0,0 +1,214 @@+"""Portability regressions: checkpoints saved by transformers 5, and non-CUDA devices.++Two failures found while getting Laya to run on a macOS/Intel box, both silent or fatal depending+on the machine:++1. transformers 5 records ModernBERT's RoPE bases as+ `rope_parameters = {"full_attention": {...}, "sliding_attention": {...}}`. transformers 4.x+ (the only line that runs on macOS Intel, where PyTorch stops at 2.2.2) does not read that key+ and falls back to its own defaults -- global 160000, local 10000. English and typed-decisions+ happen to match those defaults; mmBERT does not (both of its bases are 160000), so it ran with+ the wrong RoPE base instead of failing loudly.++2. `torch.autocast(device_type=self.device.type, enabled=False)` raises on devices torch has no+ autocast backend for, even when disabled: 'User specified an unsupported autocast device_type+ mps'. Laya only ever enables autocast on CUDA, but it still entered the context on every call,+ so `predict()` died outright on the MPS device torch selects on Apple/AMD machines.+"""+import os+import sys+from contextlib import nullcontext+from types import SimpleNamespace+from unittest import mock++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++import torch # noqa: E402++from laya import agent as _agent # noqa: E402+from laya.common import _apply_rope_config # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))+++def check_true(name, cond, detail=""):+ if cond:+ PASS.append(name)+ else:+ FAIL.append("%s %s" % (name, detail))+++# ------------------------------------------------ 1. transformers-5 RoPE config -> transformers 4.x+def cfg_with(rope_parameters, **kwargs):+ """A stand-in for a ModernBertConfig carrying both the 5.x and the 4.x view of RoPE."""+ base = dict(global_rope_theta=160000.0, local_rope_theta=10000.0) # 4.x defaults+ base.update(kwargs)+ return SimpleNamespace(rope_parameters=rope_parameters, **base)+++# the mmBERT case: both bases are 160000, so the 4.x default local theta (10000) is wrong+mmbert = cfg_with({"full_attention": {"rope_theta": 160000, "rope_type": "default"},+ "sliding_attention": {"rope_theta": 160000, "rope_type": "default"}})+_apply_rope_config(mmbert)+check("rope/mmbert global theta", mmbert.global_rope_theta, 160000.0)+check("rope/mmbert local theta corrected", mmbert.local_rope_theta, 160000.0)++# ModernBERT-large / typed-decisions: 160000 / 10000 -- already the 4.x defaults, must not move+modernbert = cfg_with({"full_attention": {"rope_theta": 160000.0, "rope_type": "default"},+ "sliding_attention": {"rope_theta": 10000.0, "rope_type": "default"}})+_apply_rope_config(modernbert)+check("rope/modernbert global theta unchanged", modernbert.global_rope_theta, 160000.0)+check("rope/modernbert local theta unchanged", modernbert.local_rope_theta, 10000.0)++# a config that carries no rope_parameters at all (every 4.x-era checkpoint) must be left alone+native = cfg_with(None, global_rope_theta=1234.0, local_rope_theta=5678.0)+_apply_rope_config(native)+check("rope/absent rope_parameters is a no-op", (native.global_rope_theta, native.local_rope_theta),+ (1234.0, 5678.0))++# transformers 5 keeps the thetas only inside rope_parameters; the old attributes are gone, so+# there is nothing to write and the call must not raise+v5_only = SimpleNamespace(rope_parameters={"full_attention": {"rope_theta": 160000},+ "sliding_attention": {"rope_theta": 160000}})+try:+ _apply_rope_config(v5_only)+ check_true("rope/transformers-5 config is a no-op", not hasattr(v5_only, "global_rope_theta"))+except Exception as e:+ FAIL.append("rope/transformers-5 config raised: %r" % e)++# a flat rope_parameters mapping is also accepted+flat = cfg_with({"rope_theta": 10000.0, "rope_type": "default"})+_apply_rope_config(flat)+check("rope/flat rope_parameters", (flat.global_rope_theta, flat.local_rope_theta), (10000.0, 10000.0))+++# ------------------------------------------------ 2. autocast is never entered off CUDA+def _autocast_that_rejects_non_cuda(entered):+ def fake(*args, **kwargs):+ entered.append(kwargs.get("device_type"))+ device_type = kwargs.get("device_type")+ if device_type != "cuda":+ raise RuntimeError("User specified an unsupported autocast device_type %r" % device_type)+ return nullcontext()+ return fake+++for name in ("cpu", "mps", "xpu"):+ entered = []+ with mock.patch.object(_agent.torch, "autocast", _autocast_that_rejects_non_cuda(entered)):+ try:+ ctx = _agent._amp_context(SimpleNamespace(type=name), torch.float32)+ with ctx:+ pass+ check_true("amp/%s never enters autocast" % name, entered == [], "entered %s" % entered)+ except RuntimeError as e:+ FAIL.append("amp/%s raised %s" % (name, e))++# CUDA still gets mixed precision, with the dtype the model was configured for+entered = []+with mock.patch.object(_agent.torch, "autocast", _autocast_that_rejects_non_cuda(entered)):+ with _agent._amp_context(SimpleNamespace(type="cuda"), torch.float16):+ pass+check("amp/cuda uses autocast", entered, ["cuda"])++# the old unconditional call is what broke MPS -- make sure it cannot come back+import inspect # noqa: E402++_src = inspect.getsource(_agent.Agent._forward)+check_true("amp/forward pass goes through _amp_context", "with _amp_context(self.device, self.dtype):" in _src)+check_true("amp/no unconditional autocast in the forward pass",+ "torch.autocast(device_type=self.device.type" not in _src)+++# ------------------------------------------------ 3. a failed GPU placement falls back to CPU+# `system_one` promises to survive a device that cannot hold the work: on a memory error it moves+# to CPU and re-runs. That path is easy to break and never fires on a machine with no GPU, so it+# is driven here with a stand-in model that fails the first forward pass the way CUDA does.+class _FakeTok:+ cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0+ mask_token = "[M]"++ def __call__(self, text, add_special_tokens=False):+ return {"input_ids": [10 + (ord(c) % 40) for c in text]}+++class _FailsOnce(torch.nn.Module):+ """Raises a CUDA-style OOM on the first call, then answers."""++ def __init__(self, message):+ super().__init__()+ self.message, self.calls = message, 0+ self.dummy = torch.nn.Parameter(torch.zeros(1))++ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):+ self.calls += 1+ if self.calls == 1:+ raise RuntimeError(self.message)+ logits = torch.zeros((input_ids.shape[0], marker_mask.shape[1]))+ logits[:, 0] = 1.0+ return logits, torch.tensor([[1.0, 0.0]])+++def _bare_agent(model):+ agent = _agent.Agent.__new__(_agent.Agent) # no weights: exercise system_one only+ agent.device = torch.device("mps") # any non-CPU device enters the fallback branch+ agent.dtype = torch.float32+ agent.cfg = {"max_len": 64, "head_max_len": 32}+ agent.temperature = [1.0, 1.0, 1.0]+ agent.temperature_by_options = {}+ agent.tok = _FakeTok()+ agent.model = model+ return agent+++QUESTIONS = {"q": {"type": "choice", "instructions": "Pick one",+ "criteria": {"a": "first option", "b": "second option"}}}++# Placement on the stand-in device is faked so the scenario also runs where torch has no+# such backend compiled in (the CPU-only wheels CI installs): batch `.to(device)` calls+# targeting the fake device become no-ops and the stand-in model supplies the failure,+# exactly as the real device would.+_real_tensor_to = torch.Tensor.to+++def _to_that_ignores_mps(self, *args, **kwargs):+ target = args[0] if args else kwargs.get("device")+ if (isinstance(target, torch.device) and target.type == "mps") or target == "mps":+ return self+ return _real_tensor_to(self, *args, **kwargs)+++with mock.patch.object(torch.Tensor, "to", _to_that_ignores_mps):+ agent = _bare_agent(_FailsOnce("CUDA out of memory. Tried to allocate 2.00 GiB"))+ try:+ result = agent.predict({"body": "some state"}, QUESTIONS)+ check("fallback/answers after a memory failure", result["answers"]["q"]["choice"], "a")+ check("fallback/forward pass retried once", agent.model.calls, 2)+ check("fallback/device is cpu now", agent.device.type, "cpu")+ check("fallback/dtype downgraded to fp32", agent.dtype, torch.float32)+ except Exception as e: # noqa: BLE001+ FAIL.append("fallback/memory failure was not survived: %s: %s" % (type(e).__name__, e))++ # a non-memory RuntimeError must still propagate: silently swallowing real bugs is worse+ # than the crash it would hide+ agent = _bare_agent(_FailsOnce("shape mismatch in attention"))+ try:+ agent.predict({"body": "some state"}, QUESTIONS)+ FAIL.append("fallback/non-memory error propagates (nothing raised)")+ except RuntimeError:+ PASS.append("fallback/non-memory error propagates")+ except Exception as e: # noqa: BLE001+ FAIL.append("fallback/non-memory error raised %s instead of RuntimeError" % type(e).__name__)+++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL " + f)+sys.exit(1 if FAIL else 0)diff --git a/tests/test_training.py b/tests/test_training.pynew file mode 100644index 00000000..796e5259--- /dev/null+++ b/tests/test_training.py@@ -0,0 +1,294 @@+"""Training-side helpers: the scoring rules Laya is trained against, and the metrics around them.++`proper_reward` is the reward the model is optimised for, so its *properness* is a correctness+property, not a detail: if the reward were maximised by something other than the true+distribution, the calibration claims in the README would not hold. `td_lambda_targets` builds+bootstrapped targets for multi-turn episodes, and `ece_score` / `confidence_from_probs` are the+metrics used to report calibration.++No model weights are loaded, so this runs anywhere torch does.+"""+import math+import os+import sys++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++import numpy as np # noqa: E402+import torch # noqa: E402++from laya.common import ( # noqa: E402+ QTYPES,+ QTYPE_NAMES,+ amp_dtype,+ build_sequence,+ collate_items,+ confidence_from_probs,+ ece_score,+ proper_reward,+ temp_bucket,+ td_lambda_targets,+)++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))+++def check_true(name, cond, detail=""):+ if cond:+ PASS.append(name)+ else:+ FAIL.append("%s %s" % (name, detail))+++def close(name, got, want, tol=1e-6):+ check_true(name, abs(got - want) <= tol, "got %r want %r" % (got, want))+++def one_hot(i, k):+ row = [0.0] * k+ row[i] = 1.0+ return row+++# =============================================================== proper_reward+# The reward is a weighted sum of a log score and a spherical score, minus a ranked probability+# score for ordinal (`score`) questions only.+Q = torch.tensor([[0.7, 0.3]])+T = torch.tensor([[1.0, 0.0]])+M = torch.tensor([[1.0, 1.0]])+QT = torch.tensor([QTYPES["choice"]])++r_perfect = proper_reward(torch.tensor([[0.7, 0.3]]), T, QT, M).item()+r_wrong = proper_reward(torch.tensor([[0.3, 0.7]]), T, QT, M).item()+check_true("reward/perfect beats wrong", r_perfect > r_wrong, "%.4f vs %.4f" % (r_perfect, r_wrong))++# Expected reward under the target must be maximised at q == target (strict properness).+# E[r(q, y)] = sum_i p_i * r(q, one_hot(i)) -- the property that makes the reported+# probabilities meaningful.+def expected_reward(q_vec, target, qtype=QTYPES["choice"], w_sph=0.5, w_rps=1.0):+ q = torch.tensor([q_vec], dtype=torch.float32)+ qt = torch.tensor([qtype])+ mask = torch.ones_like(q)+ total = 0.0+ for i, p in enumerate(target):+ if p == 0:+ continue+ tgt = torch.tensor([one_hot(i, len(target))], dtype=torch.float32)+ total += p * proper_reward(q, tgt, qt, mask, w_sph=w_sph, w_rps=w_rps).item()+ return total+++target = [0.7, 0.3]+grid = [0.5, 0.6, 0.7, 0.8, 0.9]+rewards = [(q, expected_reward([q, 1 - q], target)) for q in grid]+best = max(rewards, key=lambda kv: kv[1])[0]+close("reward/strictly proper: argmax at q == target (choice)", best, 0.7)++# same property for the ordinal (score) rule, where the RPS term also applies+rewards_score = [(q, expected_reward([q, 1 - q], target, qtype=QTYPES["score"])) for q in grid]+close("reward/strictly proper: argmax at q == target (score)",+ max(rewards_score, key=lambda kv: kv[1])[0], 0.7)++# the RPS term is score-only: the same inputs must score differently for the two types+r_choice = proper_reward(Q, T, torch.tensor([QTYPES["choice"]]), M).item()+r_score = proper_reward(Q, T, torch.tensor([QTYPES["score"]]), M).item()+check_true("reward/rps applies to score but not choice", r_score < r_choice,+ "score %.4f vs choice %.4f" % (r_score, r_choice))++# a masked-out option cannot influence the reward+r_masked_a = proper_reward(torch.tensor([[0.9, 0.1]]), T, QT, torch.tensor([[1.0, 0.0]])).item()+r_masked_b = proper_reward(torch.tensor([[0.9, 0.9]]), T, QT, torch.tensor([[1.0, 0.0]])).item()+close("reward/masked option is ignored", r_masked_a, r_masked_b)++# zero probability is clamped by log_floor rather than producing -inf+r_zero = proper_reward(torch.tensor([[1.0, 0.0]]), T, QT, M).item()+check_true("reward/zero probability is finite", math.isfinite(r_zero), "%r" % r_zero)++# permuting options consistently leaves the reward unchanged+perm = proper_reward(torch.tensor([[0.3, 0.7]]), torch.tensor([[0.0, 1.0]]), QT, M).item()+close("reward/invariant to consistent permutation", perm, proper_reward(Q, T, QT, M).item())++# the spherical weight is a real knob+hi = proper_reward(Q, T, QT, M, w_sph=2.0).item()+lo = proper_reward(Q, T, QT, M, w_sph=0.0).item()+check_true("reward/w_sph changes the reward", hi > lo, "%.4f vs %.4f" % (hi, lo))++# multi-row batches are handled independently+batch_r = proper_reward(torch.tensor([[0.7, 0.3], [0.1, 0.9]]),+ torch.tensor([[1.0, 0.0], [0.0, 1.0]]),+ torch.tensor([QTYPES["choice"], QTYPES["choice"]]),+ torch.ones(2, 2))+check("reward/batch shape", tuple(batch_r.shape), (2,))+++# =============================================================== td_lambda_targets+p_true = torch.tensor([0.1, 0.4, 0.9])++# without episode grouping the targets are returned untouched+batch = {"target": torch.tensor([[1.0, 0.0], [1.0, 0.0], [1.0, 0.0]])}+out = td_lambda_targets(p_true, batch, lam=1.0)+check_true("td_lambda/no groups -> unchanged targets",+ torch.equal(out, batch["target"]))++# grouped: rows are ordered by ep_step, and the terminal row's truth bootstraps backwards+batch = {"target": torch.tensor([[1.0, 0.0], [1.0, 0.0], [1.0, 0.0]]),+ "ep_group": torch.tensor([0, 0, 0]),+ "ep_step": torch.tensor([0, 1, 2])}+lam1 = td_lambda_targets(p_true, {"target": batch["target"].clone(),+ "ep_group": batch["ep_group"],+ "ep_step": batch["ep_step"]}, lam=1.0)+close("td_lambda/lam=1 propagates terminal truth to every step", float(lam1[0, 1]), 0.0)+close("td_lambda/lam=1 terminal step unchanged", float(lam1[2, 1]), 0.0)+check_true("td_lambda/lam=1 all steps equal the terminal target",+ torch.allclose(lam1[:, 0], 1 - lam1[:, 1]))++lam0 = td_lambda_targets(p_true, {"target": batch["target"].clone(),+ "ep_group": batch["ep_group"],+ "ep_step": batch["ep_step"]}, lam=0.0)+close("td_lambda/lam=0 step0 uses the next step's p_true", float(lam0[0, 1]), 0.4)+close("td_lambda/lam=0 step1 uses the next step's p_true", float(lam0[1, 1]), 0.9)+close("td_lambda/lam=0 terminal keeps the terminal target", float(lam0[2, 1]), 0.0)++# row order must not matter: steps are sorted by ep_step inside each group, and `p_true` is+# indexed by row, so the same logical episode written out of step order must agree per step.+shuffled = td_lambda_targets(torch.tensor([0.9, 0.1, 0.4]), # rows are [step2, step0, step1]+ {"target": torch.tensor([[1.0, 0.0]] * 3),+ "ep_group": torch.tensor([0, 0, 0]),+ "ep_step": torch.tensor([2, 0, 1])}, lam=0.0)+ordered = td_lambda_targets(torch.tensor([0.1, 0.4, 0.9]), # the same episode in step order+ {"target": torch.tensor([[1.0, 0.0]] * 3),+ "ep_group": torch.tensor([0, 0, 0]),+ "ep_step": torch.tensor([0, 1, 2])}, lam=0.0)+check_true("td_lambda/rows are processed in ep_step order",+ float(ordered[0, 1]) == float(shuffled[1, 1]) and+ float(ordered[1, 1]) == float(shuffled[2, 1]) and+ float(ordered[2, 1]) == float(shuffled[0, 1]),+ "ordered=%s shuffled=%s" % (ordered[:, 1].tolist(), shuffled[:, 1].tolist()))++# groups are independent: each episode bootstraps from its own terminal row+two = td_lambda_targets(torch.tensor([0.1, 0.2, 0.8, 0.9]),+ {"target": torch.tensor([[1.0, 0.0], [0.2, 0.8], [1.0, 0.0], [0.6, 0.4]]),+ "ep_group": torch.tensor([0, 0, 1, 1]),+ "ep_step": torch.tensor([0, 1, 0, 1])}, lam=1.0)+check("td_lambda/group 0 uses its own terminal", [round(float(v), 4) for v in two[0:2, 1]], [0.8, 0.8])+check("td_lambda/group 1 uses its own terminal", [round(float(v), 4) for v in two[2:4, 1]], [0.4, 0.4])+++# =============================================================== ece_score+# perfectly calibrated: 90% confident, right 90% of the time+conf = np.full(100, 0.9)+correct = np.array([1] * 90 + [0] * 10)+close("ece/perfectly calibrated is ~0", ece_score(conf, correct), 0.0, tol=1e-9)++# maximally overconfident: certain and always wrong+close("ece/certain and wrong is 1.0", ece_score(np.ones(10), np.zeros(10, dtype=int)), 1.0, tol=1e-9)++# empty input is nan rather than an exception+check_true("ece/empty is nan", math.isnan(ece_score(np.array([]), np.array([]))))++# overconfidence costs: same accuracy, higher confidence -> higher ECE+base_correct = np.array([1] * 80 + [0] * 20)+conf_a = np.full(100, 0.8) # calibrated+conf_b = np.full(100, 0.99) # overconfident+check_true("ece/overconfidence raises ECE", ece_score(conf_b, base_correct) > ece_score(conf_a, base_correct),+ "%.4f vs %.4f" % (ece_score(conf_b, base_correct), ece_score(conf_a, base_correct)))++# bins argument changes granularity, not correctness of the perfect case+close("ece/bins=5 perfectly calibrated", ece_score(conf, correct, bins=5), 0.0, tol=1e-9)+++# =============================================================== confidence_from_probs+close("confidence/uniform over 2 is 0", confidence_from_probs(np.array([0.5, 0.5]), 2), 0.0)+close("confidence/uniform over 5 is 0", confidence_from_probs(np.full(5, 0.2), 5), 0.0)+close("confidence/one-hot is 1", confidence_from_probs(np.array([1.0, 0.0]), 2), 1.0)+close("confidence/single option is 1 by definition", confidence_from_probs(np.array([1.0]), 1), 1.0)+spread = confidence_from_probs(np.array([0.6, 0.3, 0.1]), 3)+peaked = confidence_from_probs(np.array([0.9, 0.05, 0.05]), 3)+check_true("confidence/decreases with entropy", peaked > spread, "%.4f vs %.4f" % (peaked, spread))+close("confidence/only the first k entries count", confidence_from_probs(np.array([0.5, 0.5, 0.0]), 2), 0.0)+++# =============================================================== small helpers+check("qtypes/choice", QTYPES["choice"], 0)+check("qtypes/score", QTYPES["score"], 1)+check("qtypes/noul", QTYPES["noul"], 2)+check("qtypes/names round-trip", QTYPE_NAMES[QTYPES["noul"]], "noul")++check("temp_bucket/2 options is noul:2", temp_bucket(QTYPES["noul"], 2), "noul:2")+check("temp_bucket/3-5", temp_bucket(QTYPES["choice"], 4), "choice:3-5")+check("temp_bucket/6-10", temp_bucket(QTYPES["choice"], 10), "choice:6-10")+check("temp_bucket/11+", temp_bucket(QTYPES["choice"], 77), "choice:11+")+check("temp_bucket/score bucket", temp_bucket(QTYPES["score"], 3), "score:3-5")++check("amp_dtype/bf16", amp_dtype("bf16"), torch.bfloat16)+check("amp_dtype/anything else is fp16", amp_dtype(None), torch.float16)+++# =============================================================== collate_items+items = [{"ids": [1, 2, 3], "markers": [1, 2], "qtype": 0, "label": 1},+ {"ids": [4, 5], "markers": [1], "qtype": 2, "label": -1}]+b = collate_items([items], pad_id=0)+check("collate/batch size", tuple(b["input_ids"].shape), (2, 3))+check("collate/marker columns = longest marker list", tuple(b["marker_pos"].shape), (2, 2))+check("collate/padding filled with pad_id", b["input_ids"][1].tolist(), [4, 5, 0])+check("collate/attention mask marks real tokens", b["attention_mask"][1].tolist(), [1, 1, 0])+check("collate/marker mask marks real markers", b["marker_mask"][1].tolist(), [True, False])+check("collate/qtype carried", b["qtype"].tolist(), [0, 2])+check("collate/no target unless requested", "target" in b, False)+b2 = collate_items([[{"ids": [1], "markers": [0, 1], "qtype": 1, "target": [0.0, 1.0]}]], pad_id=0)+check("collate/target included when present", tuple(b2["target"].shape), (1, 2))+check("collate/empty batch returns None", collate_items([], pad_id=0), None)++# a target with more entries than the item has options is a data bug; it must say so, not+# surface as a tensor shape error from inside the assignment+try:+ collate_items([[{"ids": [1], "markers": [0], "qtype": 1, "target": [0.0, 1.0]}]], pad_id=0)+ FAIL.append("collate/oversized target raises ValueError (nothing raised)")+except ValueError as e:+ check_true("collate/oversized target raises a clear ValueError",+ "one entry per option" in str(e), str(e)[:90])+except Exception as e: # noqa: BLE001+ FAIL.append("collate/oversized target raised %s instead of ValueError: %s" % (type(e).__name__, e))+++# =============================================================== build_sequence+class _Tok:+ """Minimal stand-in for a fast tokenizer: one id per character."""+ cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0+ mask_token = "[M]"++ def __call__(self, text, add_special_tokens=False):+ return {"input_ids": [10 + (ord(c) % 50) for c in text]}+++tok = _Tok()+q = {"t": "choice", "ins": "Which team?", "crit": {"a": "first", "b": "second", "c": "third"}}+seq, markers = build_sequence(tok, "some state text", q, max_len=128, head_max_len=64)+check("build_sequence/one marker per option", len(markers), 3)+check_true("build_sequence/markers point inside the sequence", all(0 <= m < len(seq) for m in markers))+check_true("build_sequence/starts with CLS", seq[0] == tok.cls_token_id)+check_true("build_sequence/ends with SEP", seq[-1] == tok.sep_token_id)+check_true("build_sequence/respects max_len", len(seq) <= 128)++long_seq, long_markers = build_sequence(tok, "x" * 5000, q, max_len=64, head_max_len=32)+check_true("build_sequence/truncates long state to max_len", len(long_seq) <= 64, "len=%d" % len(long_seq))++noul_seq, noul_markers = build_sequence(tok, "state", {"t": "noul", "ins": "Is it?", "crit": None}, 128, 64)+check("build_sequence/noul always offers two options", len(noul_markers), 2)++score_seq, score_markers = build_sequence(tok, "state", {"t": "score", "ins": "How bad?",+ "crit": ["low", "mid", "high"]}, 128, 64)+check("build_sequence/score has one marker per level", len(score_markers), 3)+++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL " + f)+sys.exit(1 if FAIL else 0)