Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Fix: run the checkpoints on transformers 4.x and on MPS (rebased #51) (#273)

Aashish Mahato194afbe63523
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1a370b30..52ae7c16 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -78,6 +78,8 @@ jobs:
python tests/test_cli.py
python tests/test_mcp.py
python tests/test_langchain.py
+ python tests/test_portability.py
+ python tests/test_training.py
- name: Email cleaning tests
run: python tests/test_email.py
diff --git a/.gitignore b/.gitignore
index 941841d1..3d264b23 100644
--- a/.gitignore
+++ b/.gitignore
@@ -69,3 +69,7 @@ credentials.json
# Documentation site build output (zensical build)
site/
+
+# ruff's cache, and the JSON the research harnesses write on every run
+.ruff_cache/
+research/*_benchmark_results.json
diff --git a/laya/agent.py b/laya/agent.py
index 36abee8c..99df622f 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -4,6 +4,7 @@ import os
import tempfile
import threading
import warnings
+from contextlib import nullcontext
from typing import Any, Dict, List, Optional, Union
import numpy as np
@@ -163,6 +164,19 @@ def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any:
return tokenizer
+def _amp_context(device, dtype):
+ """Autocast context for the forward pass, or a no-op when mixed precision is not in use.
+
+ Autocast is a CUDA-only win here. Entering `torch.autocast` on a device torch has no
+ autocast backend for raises even with `enabled=False` ('User specified an unsupported
+ autocast device_type mps'), which broke every `predict()` call on the MPS GPU that torch
+ selects automatically on Apple/AMD machines. Only wrap the forward pass when we use it.
+ """
+ if device.type == "cuda":
+ return torch.autocast(device_type="cuda", dtype=dtype)
+ return nullcontext()
+
+
class Agent:
"""System 1 decision model runtime: fast, non-autoregressive, calibrated decisions."""
@@ -485,8 +499,7 @@ class Agent:
def _forward(self, b: Dict):
"""Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""
def run():
- use_amp = self.device.type == "cuda"
- with torch.autocast(device_type=self.device.type, dtype=self.dtype, enabled=use_amp):
+ with _amp_context(self.device, self.dtype):
return self.model(
b["input_ids"].to(self.device),
b["attention_mask"].to(self.device),
diff --git a/laya/common.py b/laya/common.py
index d73e248b..6d0859bb 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -163,11 +163,35 @@ class DecisionModel(nn.Module):
return logits, act_logits
+def _apply_rope_config(ecfg) -> None:
+ """Carry transformers>=5 per-layer RoPE settings over to the attributes 4.x reads.
+
+ A checkpoint re-saved by transformers 5 stores RoPE as
+ `rope_parameters = {"full_attention": {"rope_theta": ...}, "sliding_attention": {...}}`.
+ transformers 4.x does not know that key, so it keeps its own defaults (global 160000,
+ local 10000) and any checkpoint whose sliding-attention theta differs silently runs the
+ wrong RoPE base -- mmBERT is exactly that case, both of its thetas are 160000. Map the
+ values onto `global_rope_theta` / `local_rope_theta`, which 4.x does read. On
+ transformers 5 this is a no-op beyond re-setting the same numbers.
+ """
+ rope = getattr(ecfg, "rope_parameters", None)
+ if not isinstance(rope, dict):
+ return
+ flat = rope.get("rope_theta")
+ for layer_type, attr in (("full_attention", "global_rope_theta"),
+ ("sliding_attention", "local_rope_theta")):
+ params = rope.get(layer_type)
+ theta = params.get("rope_theta") if isinstance(params, dict) else flat
+ if theta is not None and hasattr(ecfg, attr):
+ setattr(ecfg, attr, float(theta))
+
+
def build_model(cfg: Dict, encoder_dir: Optional[str] = None, pretrained: bool = True) -> DecisionModel:
from transformers import AutoConfig, AutoModel
if not pretrained or (encoder_dir and os.path.exists(encoder_dir)):
ecfg = AutoConfig.from_pretrained(encoder_dir or cfg["encoder"])
+ _apply_rope_config(ecfg)
enc = AutoModel.from_config(ecfg, attn_implementation="sdpa")
else:
enc = AutoModel.from_pretrained(cfg["encoder"], attn_implementation="sdpa")
@@ -291,6 +315,13 @@ def collate_items(batch, pad_id: int):
mpos[i, :k] = torch.tensor(it["markers"])
mmask[i, :k] = True
if has_target and "target" in it:
+ if len(it["target"]) > kmax:
+ # Otherwise this lands as "The expanded size of the tensor (k) must match the
+ # existing size (kmax)" from inside the assignment, which says nothing about the
+ # actual mistake: a target with more entries than the item has options.
+ raise ValueError(
+ "collate_items: item %d has %d target entries but only %d marker positions; "
+ "a target needs one entry per option" % (i, len(it["target"]), kmax))
target[i, : len(it["target"])] = torch.tensor(it["target"], dtype=torch.float32)
res = {
diff --git a/tests/test_portability.py b/tests/test_portability.py
new file mode 100644
index 00000000..fffe24b3
--- /dev/null
+++ b/tests/test_portability.py
@@ -0,0 +1,214 @@
+"""Portability regressions: checkpoints saved by transformers 5, and non-CUDA devices.
+
+Two failures found while getting Laya to run on a macOS/Intel box, both silent or fatal depending
+on the machine:
+
+1. transformers 5 records ModernBERT's RoPE bases as
+ `rope_parameters = {"full_attention": {...}, "sliding_attention": {...}}`. transformers 4.x
+ (the only line that runs on macOS Intel, where PyTorch stops at 2.2.2) does not read that key
+ and falls back to its own defaults -- global 160000, local 10000. English and typed-decisions
+ happen to match those defaults; mmBERT does not (both of its bases are 160000), so it ran with
+ the wrong RoPE base instead of failing loudly.
+
+2. `torch.autocast(device_type=self.device.type, enabled=False)` raises on devices torch has no
+ autocast backend for, even when disabled: 'User specified an unsupported autocast device_type
+ mps'. Laya only ever enables autocast on CUDA, but it still entered the context on every call,
+ so `predict()` died outright on the MPS device torch selects on Apple/AMD machines.
+"""
+import os
+import sys
+from contextlib import nullcontext
+from types import SimpleNamespace
+from unittest import mock
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import torch # noqa: E402
+
+from laya import agent as _agent # noqa: E402
+from laya.common import _apply_rope_config # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s %s" % (name, detail))
+
+
+# ------------------------------------------------ 1. transformers-5 RoPE config -> transformers 4.x
+def cfg_with(rope_parameters, **kwargs):
+ """A stand-in for a ModernBertConfig carrying both the 5.x and the 4.x view of RoPE."""
+ base = dict(global_rope_theta=160000.0, local_rope_theta=10000.0) # 4.x defaults
+ base.update(kwargs)
+ return SimpleNamespace(rope_parameters=rope_parameters, **base)
+
+
+# the mmBERT case: both bases are 160000, so the 4.x default local theta (10000) is wrong
+mmbert = cfg_with({"full_attention": {"rope_theta": 160000, "rope_type": "default"},
+ "sliding_attention": {"rope_theta": 160000, "rope_type": "default"}})
+_apply_rope_config(mmbert)
+check("rope/mmbert global theta", mmbert.global_rope_theta, 160000.0)
+check("rope/mmbert local theta corrected", mmbert.local_rope_theta, 160000.0)
+
+# ModernBERT-large / typed-decisions: 160000 / 10000 -- already the 4.x defaults, must not move
+modernbert = cfg_with({"full_attention": {"rope_theta": 160000.0, "rope_type": "default"},
+ "sliding_attention": {"rope_theta": 10000.0, "rope_type": "default"}})
+_apply_rope_config(modernbert)
+check("rope/modernbert global theta unchanged", modernbert.global_rope_theta, 160000.0)
+check("rope/modernbert local theta unchanged", modernbert.local_rope_theta, 10000.0)
+
+# a config that carries no rope_parameters at all (every 4.x-era checkpoint) must be left alone
+native = cfg_with(None, global_rope_theta=1234.0, local_rope_theta=5678.0)
+_apply_rope_config(native)
+check("rope/absent rope_parameters is a no-op", (native.global_rope_theta, native.local_rope_theta),
+ (1234.0, 5678.0))
+
+# transformers 5 keeps the thetas only inside rope_parameters; the old attributes are gone, so
+# there is nothing to write and the call must not raise
+v5_only = SimpleNamespace(rope_parameters={"full_attention": {"rope_theta": 160000},
+ "sliding_attention": {"rope_theta": 160000}})
+try:
+ _apply_rope_config(v5_only)
+ check_true("rope/transformers-5 config is a no-op", not hasattr(v5_only, "global_rope_theta"))
+except Exception as e:
+ FAIL.append("rope/transformers-5 config raised: %r" % e)
+
+# a flat rope_parameters mapping is also accepted
+flat = cfg_with({"rope_theta": 10000.0, "rope_type": "default"})
+_apply_rope_config(flat)
+check("rope/flat rope_parameters", (flat.global_rope_theta, flat.local_rope_theta), (10000.0, 10000.0))
+
+
+# ------------------------------------------------ 2. autocast is never entered off CUDA
+def _autocast_that_rejects_non_cuda(entered):
+ def fake(*args, **kwargs):
+ entered.append(kwargs.get("device_type"))
+ device_type = kwargs.get("device_type")
+ if device_type != "cuda":
+ raise RuntimeError("User specified an unsupported autocast device_type %r" % device_type)
+ return nullcontext()
+ return fake
+
+
+for name in ("cpu", "mps", "xpu"):
+ entered = []
+ with mock.patch.object(_agent.torch, "autocast", _autocast_that_rejects_non_cuda(entered)):
+ try:
+ ctx = _agent._amp_context(SimpleNamespace(type=name), torch.float32)
+ with ctx:
+ pass
+ check_true("amp/%s never enters autocast" % name, entered == [], "entered %s" % entered)
+ except RuntimeError as e:
+ FAIL.append("amp/%s raised %s" % (name, e))
+
+# CUDA still gets mixed precision, with the dtype the model was configured for
+entered = []
+with mock.patch.object(_agent.torch, "autocast", _autocast_that_rejects_non_cuda(entered)):
+ with _agent._amp_context(SimpleNamespace(type="cuda"), torch.float16):
+ pass
+check("amp/cuda uses autocast", entered, ["cuda"])
+
+# the old unconditional call is what broke MPS -- make sure it cannot come back
+import inspect # noqa: E402
+
+_src = inspect.getsource(_agent.Agent._forward)
+check_true("amp/forward pass goes through _amp_context", "with _amp_context(self.device, self.dtype):" in _src)
+check_true("amp/no unconditional autocast in the forward pass",
+ "torch.autocast(device_type=self.device.type" not in _src)
+
+
+# ------------------------------------------------ 3. a failed GPU placement falls back to CPU
+# `system_one` promises to survive a device that cannot hold the work: on a memory error it moves
+# to CPU and re-runs. That path is easy to break and never fires on a machine with no GPU, so it
+# is driven here with a stand-in model that fails the first forward pass the way CUDA does.
+class _FakeTok:
+ cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0
+ mask_token = "[M]"
+
+ def __call__(self, text, add_special_tokens=False):
+ return {"input_ids": [10 + (ord(c) % 40) for c in text]}
+
+
+class _FailsOnce(torch.nn.Module):
+ """Raises a CUDA-style OOM on the first call, then answers."""
+
+ def __init__(self, message):
+ super().__init__()
+ self.message, self.calls = message, 0
+ self.dummy = torch.nn.Parameter(torch.zeros(1))
+
+ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):
+ self.calls += 1
+ if self.calls == 1:
+ raise RuntimeError(self.message)
+ logits = torch.zeros((input_ids.shape[0], marker_mask.shape[1]))
+ logits[:, 0] = 1.0
+ return logits, torch.tensor([[1.0, 0.0]])
+
+
+def _bare_agent(model):
+ agent = _agent.Agent.__new__(_agent.Agent) # no weights: exercise system_one only
+ agent.device = torch.device("mps") # any non-CPU device enters the fallback branch
+ agent.dtype = torch.float32
+ agent.cfg = {"max_len": 64, "head_max_len": 32}
+ agent.temperature = [1.0, 1.0, 1.0]
+ agent.temperature_by_options = {}
+ agent.tok = _FakeTok()
+ agent.model = model
+ return agent
+
+
+QUESTIONS = {"q": {"type": "choice", "instructions": "Pick one",
+ "criteria": {"a": "first option", "b": "second option"}}}
+
+# Placement on the stand-in device is faked so the scenario also runs where torch has no
+# such backend compiled in (the CPU-only wheels CI installs): batch `.to(device)` calls
+# targeting the fake device become no-ops and the stand-in model supplies the failure,
+# exactly as the real device would.
+_real_tensor_to = torch.Tensor.to
+
+
+def _to_that_ignores_mps(self, *args, **kwargs):
+ target = args[0] if args else kwargs.get("device")
+ if (isinstance(target, torch.device) and target.type == "mps") or target == "mps":
+ return self
+ return _real_tensor_to(self, *args, **kwargs)
+
+
+with mock.patch.object(torch.Tensor, "to", _to_that_ignores_mps):
+ agent = _bare_agent(_FailsOnce("CUDA out of memory. Tried to allocate 2.00 GiB"))
+ try:
+ result = agent.predict({"body": "some state"}, QUESTIONS)
+ check("fallback/answers after a memory failure", result["answers"]["q"]["choice"], "a")
+ check("fallback/forward pass retried once", agent.model.calls, 2)
+ check("fallback/device is cpu now", agent.device.type, "cpu")
+ check("fallback/dtype downgraded to fp32", agent.dtype, torch.float32)
+ except Exception as e: # noqa: BLE001
+ FAIL.append("fallback/memory failure was not survived: %s: %s" % (type(e).__name__, e))
+
+ # a non-memory RuntimeError must still propagate: silently swallowing real bugs is worse
+ # than the crash it would hide
+ agent = _bare_agent(_FailsOnce("shape mismatch in attention"))
+ try:
+ agent.predict({"body": "some state"}, QUESTIONS)
+ FAIL.append("fallback/non-memory error propagates (nothing raised)")
+ except RuntimeError:
+ PASS.append("fallback/non-memory error propagates")
+ except Exception as e: # noqa: BLE001
+ FAIL.append("fallback/non-memory error raised %s instead of RuntimeError" % type(e).__name__)
+
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+sys.exit(1 if FAIL else 0)
diff --git a/tests/test_training.py b/tests/test_training.py
new file mode 100644
index 00000000..796e5259
--- /dev/null
+++ b/tests/test_training.py
@@ -0,0 +1,294 @@
+"""Training-side helpers: the scoring rules Laya is trained against, and the metrics around them.
+
+`proper_reward` is the reward the model is optimised for, so its *properness* is a correctness
+property, not a detail: if the reward were maximised by something other than the true
+distribution, the calibration claims in the README would not hold. `td_lambda_targets` builds
+bootstrapped targets for multi-turn episodes, and `ece_score` / `confidence_from_probs` are the
+metrics used to report calibration.
+
+No model weights are loaded, so this runs anywhere torch does.
+"""
+import math
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import numpy as np # noqa: E402
+import torch # noqa: E402
+
+from laya.common import ( # noqa: E402
+ QTYPES,
+ QTYPE_NAMES,
+ amp_dtype,
+ build_sequence,
+ collate_items,
+ confidence_from_probs,
+ ece_score,
+ proper_reward,
+ temp_bucket,
+ td_lambda_targets,
+)
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s %s" % (name, detail))
+
+
+def close(name, got, want, tol=1e-6):
+ check_true(name, abs(got - want) <= tol, "got %r want %r" % (got, want))
+
+
+def one_hot(i, k):
+ row = [0.0] * k
+ row[i] = 1.0
+ return row
+
+
+# =============================================================== proper_reward
+# The reward is a weighted sum of a log score and a spherical score, minus a ranked probability
+# score for ordinal (`score`) questions only.
+Q = torch.tensor([[0.7, 0.3]])
+T = torch.tensor([[1.0, 0.0]])
+M = torch.tensor([[1.0, 1.0]])
+QT = torch.tensor([QTYPES["choice"]])
+
+r_perfect = proper_reward(torch.tensor([[0.7, 0.3]]), T, QT, M).item()
+r_wrong = proper_reward(torch.tensor([[0.3, 0.7]]), T, QT, M).item()
+check_true("reward/perfect beats wrong", r_perfect > r_wrong, "%.4f vs %.4f" % (r_perfect, r_wrong))
+
+# Expected reward under the target must be maximised at q == target (strict properness).
+# E[r(q, y)] = sum_i p_i * r(q, one_hot(i)) -- the property that makes the reported
+# probabilities meaningful.
+def expected_reward(q_vec, target, qtype=QTYPES["choice"], w_sph=0.5, w_rps=1.0):
+ q = torch.tensor([q_vec], dtype=torch.float32)
+ qt = torch.tensor([qtype])
+ mask = torch.ones_like(q)
+ total = 0.0
+ for i, p in enumerate(target):
+ if p == 0:
+ continue
+ tgt = torch.tensor([one_hot(i, len(target))], dtype=torch.float32)
+ total += p * proper_reward(q, tgt, qt, mask, w_sph=w_sph, w_rps=w_rps).item()
+ return total
+
+
+target = [0.7, 0.3]
+grid = [0.5, 0.6, 0.7, 0.8, 0.9]
+rewards = [(q, expected_reward([q, 1 - q], target)) for q in grid]
+best = max(rewards, key=lambda kv: kv[1])[0]
+close("reward/strictly proper: argmax at q == target (choice)", best, 0.7)
+
+# same property for the ordinal (score) rule, where the RPS term also applies
+rewards_score = [(q, expected_reward([q, 1 - q], target, qtype=QTYPES["score"])) for q in grid]
+close("reward/strictly proper: argmax at q == target (score)",
+ max(rewards_score, key=lambda kv: kv[1])[0], 0.7)
+
+# the RPS term is score-only: the same inputs must score differently for the two types
+r_choice = proper_reward(Q, T, torch.tensor([QTYPES["choice"]]), M).item()
+r_score = proper_reward(Q, T, torch.tensor([QTYPES["score"]]), M).item()
+check_true("reward/rps applies to score but not choice", r_score < r_choice,
+ "score %.4f vs choice %.4f" % (r_score, r_choice))
+
+# a masked-out option cannot influence the reward
+r_masked_a = proper_reward(torch.tensor([[0.9, 0.1]]), T, QT, torch.tensor([[1.0, 0.0]])).item()
+r_masked_b = proper_reward(torch.tensor([[0.9, 0.9]]), T, QT, torch.tensor([[1.0, 0.0]])).item()
+close("reward/masked option is ignored", r_masked_a, r_masked_b)
+
+# zero probability is clamped by log_floor rather than producing -inf
+r_zero = proper_reward(torch.tensor([[1.0, 0.0]]), T, QT, M).item()
+check_true("reward/zero probability is finite", math.isfinite(r_zero), "%r" % r_zero)
+
+# permuting options consistently leaves the reward unchanged
+perm = proper_reward(torch.tensor([[0.3, 0.7]]), torch.tensor([[0.0, 1.0]]), QT, M).item()
+close("reward/invariant to consistent permutation", perm, proper_reward(Q, T, QT, M).item())
+
+# the spherical weight is a real knob
+hi = proper_reward(Q, T, QT, M, w_sph=2.0).item()
+lo = proper_reward(Q, T, QT, M, w_sph=0.0).item()
+check_true("reward/w_sph changes the reward", hi > lo, "%.4f vs %.4f" % (hi, lo))
+
+# multi-row batches are handled independently
+batch_r = proper_reward(torch.tensor([[0.7, 0.3], [0.1, 0.9]]),
+ torch.tensor([[1.0, 0.0], [0.0, 1.0]]),
+ torch.tensor([QTYPES["choice"], QTYPES["choice"]]),
+ torch.ones(2, 2))
+check("reward/batch shape", tuple(batch_r.shape), (2,))
+
+
+# =============================================================== td_lambda_targets
+p_true = torch.tensor([0.1, 0.4, 0.9])
+
+# without episode grouping the targets are returned untouched
+batch = {"target": torch.tensor([[1.0, 0.0], [1.0, 0.0], [1.0, 0.0]])}
+out = td_lambda_targets(p_true, batch, lam=1.0)
+check_true("td_lambda/no groups -> unchanged targets",
+ torch.equal(out, batch["target"]))
+
+# grouped: rows are ordered by ep_step, and the terminal row's truth bootstraps backwards
+batch = {"target": torch.tensor([[1.0, 0.0], [1.0, 0.0], [1.0, 0.0]]),
+ "ep_group": torch.tensor([0, 0, 0]),
+ "ep_step": torch.tensor([0, 1, 2])}
+lam1 = td_lambda_targets(p_true, {"target": batch["target"].clone(),
+ "ep_group": batch["ep_group"],
+ "ep_step": batch["ep_step"]}, lam=1.0)
+close("td_lambda/lam=1 propagates terminal truth to every step", float(lam1[0, 1]), 0.0)
+close("td_lambda/lam=1 terminal step unchanged", float(lam1[2, 1]), 0.0)
+check_true("td_lambda/lam=1 all steps equal the terminal target",
+ torch.allclose(lam1[:, 0], 1 - lam1[:, 1]))
+
+lam0 = td_lambda_targets(p_true, {"target": batch["target"].clone(),
+ "ep_group": batch["ep_group"],
+ "ep_step": batch["ep_step"]}, lam=0.0)
+close("td_lambda/lam=0 step0 uses the next step's p_true", float(lam0[0, 1]), 0.4)
+close("td_lambda/lam=0 step1 uses the next step's p_true", float(lam0[1, 1]), 0.9)
+close("td_lambda/lam=0 terminal keeps the terminal target", float(lam0[2, 1]), 0.0)
+
+# row order must not matter: steps are sorted by ep_step inside each group, and `p_true` is
+# indexed by row, so the same logical episode written out of step order must agree per step.
+shuffled = td_lambda_targets(torch.tensor([0.9, 0.1, 0.4]), # rows are [step2, step0, step1]
+ {"target": torch.tensor([[1.0, 0.0]] * 3),
+ "ep_group": torch.tensor([0, 0, 0]),
+ "ep_step": torch.tensor([2, 0, 1])}, lam=0.0)
+ordered = td_lambda_targets(torch.tensor([0.1, 0.4, 0.9]), # the same episode in step order
+ {"target": torch.tensor([[1.0, 0.0]] * 3),
+ "ep_group": torch.tensor([0, 0, 0]),
+ "ep_step": torch.tensor([0, 1, 2])}, lam=0.0)
+check_true("td_lambda/rows are processed in ep_step order",
+ float(ordered[0, 1]) == float(shuffled[1, 1]) and
+ float(ordered[1, 1]) == float(shuffled[2, 1]) and
+ float(ordered[2, 1]) == float(shuffled[0, 1]),
+ "ordered=%s shuffled=%s" % (ordered[:, 1].tolist(), shuffled[:, 1].tolist()))
+
+# groups are independent: each episode bootstraps from its own terminal row
+two = td_lambda_targets(torch.tensor([0.1, 0.2, 0.8, 0.9]),
+ {"target": torch.tensor([[1.0, 0.0], [0.2, 0.8], [1.0, 0.0], [0.6, 0.4]]),
+ "ep_group": torch.tensor([0, 0, 1, 1]),
+ "ep_step": torch.tensor([0, 1, 0, 1])}, lam=1.0)
+check("td_lambda/group 0 uses its own terminal", [round(float(v), 4) for v in two[0:2, 1]], [0.8, 0.8])
+check("td_lambda/group 1 uses its own terminal", [round(float(v), 4) for v in two[2:4, 1]], [0.4, 0.4])
+
+
+# =============================================================== ece_score
+# perfectly calibrated: 90% confident, right 90% of the time
+conf = np.full(100, 0.9)
+correct = np.array([1] * 90 + [0] * 10)
+close("ece/perfectly calibrated is ~0", ece_score(conf, correct), 0.0, tol=1e-9)
+
+# maximally overconfident: certain and always wrong
+close("ece/certain and wrong is 1.0", ece_score(np.ones(10), np.zeros(10, dtype=int)), 1.0, tol=1e-9)
+
+# empty input is nan rather than an exception
+check_true("ece/empty is nan", math.isnan(ece_score(np.array([]), np.array([]))))
+
+# overconfidence costs: same accuracy, higher confidence -> higher ECE
+base_correct = np.array([1] * 80 + [0] * 20)
+conf_a = np.full(100, 0.8) # calibrated
+conf_b = np.full(100, 0.99) # overconfident
+check_true("ece/overconfidence raises ECE", ece_score(conf_b, base_correct) > ece_score(conf_a, base_correct),
+ "%.4f vs %.4f" % (ece_score(conf_b, base_correct), ece_score(conf_a, base_correct)))
+
+# bins argument changes granularity, not correctness of the perfect case
+close("ece/bins=5 perfectly calibrated", ece_score(conf, correct, bins=5), 0.0, tol=1e-9)
+
+
+# =============================================================== confidence_from_probs
+close("confidence/uniform over 2 is 0", confidence_from_probs(np.array([0.5, 0.5]), 2), 0.0)
+close("confidence/uniform over 5 is 0", confidence_from_probs(np.full(5, 0.2), 5), 0.0)
+close("confidence/one-hot is 1", confidence_from_probs(np.array([1.0, 0.0]), 2), 1.0)
+close("confidence/single option is 1 by definition", confidence_from_probs(np.array([1.0]), 1), 1.0)
+spread = confidence_from_probs(np.array([0.6, 0.3, 0.1]), 3)
+peaked = confidence_from_probs(np.array([0.9, 0.05, 0.05]), 3)
+check_true("confidence/decreases with entropy", peaked > spread, "%.4f vs %.4f" % (peaked, spread))
+close("confidence/only the first k entries count", confidence_from_probs(np.array([0.5, 0.5, 0.0]), 2), 0.0)
+
+
+# =============================================================== small helpers
+check("qtypes/choice", QTYPES["choice"], 0)
+check("qtypes/score", QTYPES["score"], 1)
+check("qtypes/noul", QTYPES["noul"], 2)
+check("qtypes/names round-trip", QTYPE_NAMES[QTYPES["noul"]], "noul")
+
+check("temp_bucket/2 options is noul:2", temp_bucket(QTYPES["noul"], 2), "noul:2")
+check("temp_bucket/3-5", temp_bucket(QTYPES["choice"], 4), "choice:3-5")
+check("temp_bucket/6-10", temp_bucket(QTYPES["choice"], 10), "choice:6-10")
+check("temp_bucket/11+", temp_bucket(QTYPES["choice"], 77), "choice:11+")
+check("temp_bucket/score bucket", temp_bucket(QTYPES["score"], 3), "score:3-5")
+
+check("amp_dtype/bf16", amp_dtype("bf16"), torch.bfloat16)
+check("amp_dtype/anything else is fp16", amp_dtype(None), torch.float16)
+
+
+# =============================================================== collate_items
+items = [{"ids": [1, 2, 3], "markers": [1, 2], "qtype": 0, "label": 1},
+ {"ids": [4, 5], "markers": [1], "qtype": 2, "label": -1}]
+b = collate_items([items], pad_id=0)
+check("collate/batch size", tuple(b["input_ids"].shape), (2, 3))
+check("collate/marker columns = longest marker list", tuple(b["marker_pos"].shape), (2, 2))
+check("collate/padding filled with pad_id", b["input_ids"][1].tolist(), [4, 5, 0])
+check("collate/attention mask marks real tokens", b["attention_mask"][1].tolist(), [1, 1, 0])
+check("collate/marker mask marks real markers", b["marker_mask"][1].tolist(), [True, False])
+check("collate/qtype carried", b["qtype"].tolist(), [0, 2])
+check("collate/no target unless requested", "target" in b, False)
+b2 = collate_items([[{"ids": [1], "markers": [0, 1], "qtype": 1, "target": [0.0, 1.0]}]], pad_id=0)
+check("collate/target included when present", tuple(b2["target"].shape), (1, 2))
+check("collate/empty batch returns None", collate_items([], pad_id=0), None)
+
+# a target with more entries than the item has options is a data bug; it must say so, not
+# surface as a tensor shape error from inside the assignment
+try:
+ collate_items([[{"ids": [1], "markers": [0], "qtype": 1, "target": [0.0, 1.0]}]], pad_id=0)
+ FAIL.append("collate/oversized target raises ValueError (nothing raised)")
+except ValueError as e:
+ check_true("collate/oversized target raises a clear ValueError",
+ "one entry per option" in str(e), str(e)[:90])
+except Exception as e: # noqa: BLE001
+ FAIL.append("collate/oversized target raised %s instead of ValueError: %s" % (type(e).__name__, e))
+
+
+# =============================================================== build_sequence
+class _Tok:
+ """Minimal stand-in for a fast tokenizer: one id per character."""
+ cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0
+ mask_token = "[M]"
+
+ def __call__(self, text, add_special_tokens=False):
+ return {"input_ids": [10 + (ord(c) % 50) for c in text]}
+
+
+tok = _Tok()
+q = {"t": "choice", "ins": "Which team?", "crit": {"a": "first", "b": "second", "c": "third"}}
+seq, markers = build_sequence(tok, "some state text", q, max_len=128, head_max_len=64)
+check("build_sequence/one marker per option", len(markers), 3)
+check_true("build_sequence/markers point inside the sequence", all(0 <= m < len(seq) for m in markers))
+check_true("build_sequence/starts with CLS", seq[0] == tok.cls_token_id)
+check_true("build_sequence/ends with SEP", seq[-1] == tok.sep_token_id)
+check_true("build_sequence/respects max_len", len(seq) <= 128)
+
+long_seq, long_markers = build_sequence(tok, "x" * 5000, q, max_len=64, head_max_len=32)
+check_true("build_sequence/truncates long state to max_len", len(long_seq) <= 64, "len=%d" % len(long_seq))
+
+noul_seq, noul_markers = build_sequence(tok, "state", {"t": "noul", "ins": "Is it?", "crit": None}, 128, 64)
+check("build_sequence/noul always offers two options", len(noul_markers), 2)
+
+score_seq, score_markers = build_sequence(tok, "state", {"t": "score", "ins": "How bad?",
+ "crit": ["low", "mid", "high"]}, 128, 64)
+check("build_sequence/score has one marker per level", len(score_markers), 3)
+
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+sys.exit(1 if FAIL else 0)