Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(common): parallel option layout for order-invariant decisions

sharath-sms9aaf4af99eac
diff --git a/laya/agent.py b/laya/agent.py
index 9ef1b0f3..7714f1fc 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -45,6 +45,7 @@ from .common import (
serialize_state,
temp_bucket,
unpermute_probs,
+ uses_parallel_layout,
window_batch_cap,
state_room,
window_budget,
@@ -469,6 +470,8 @@ class Agent(HookRegistry):
# The stock forward is also used by lightweight runtimes built with __new__ in tests.
_fast = None
_backend = None
+ # The checkpoint's `option_layout` (see `uses_parallel_layout`); runtimes built with __new__ are sequential.
+ parallel_options = False
# Scoped CPU-fallback observability: how often _infer's per-request OOM fallback fired
# and what the last failure was, so an operator sees a slow lane in /health instead of
# discovering it by accident. Class defaults cover instances built without __init__.
@@ -615,6 +618,7 @@ class Agent(HookRegistry):
with open(cfg_path) as f:
self.cfg = json.load(f)
+ self.parallel_options = uses_parallel_layout(self.cfg)
weights_path = os.path.join(model_dir, "model.safetensors")
if not os.path.exists(weights_path):
@@ -1150,10 +1154,10 @@ class Agent(HookRegistry):
items = []
for qid in ids:
q = internal[qid]
- seq, markers, stats, state_stats = build_sequence(self.tok, state, q, max_len, head_max_len,
- option_order=q.get("option_order"),
- truncate_left=truncate_left, state_ids=state_ids,
- return_stats=True, return_truncation_stats=True)
+ seq, markers, stats, state_stats, *layout = build_sequence(
+ self.tok, state, q, max_len, head_max_len, option_order=q.get("option_order"),
+ truncate_left=truncate_left, state_ids=state_ids, return_stats=True,
+ return_truncation_stats=True, return_layout=self.parallel_options)
n_opts = len(render_options(q))
if len(markers) != n_opts:
# The markers are placed at absolute positions and `build_sequence` then drops the
@@ -1171,8 +1175,11 @@ class Agent(HookRegistry):
"head_max_len=%d spent on the question; lower head_max_len, raise max_len, "
"or use fewer options"
% (qid, len(markers), n_opts, max_len, head_max_len))
- items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats,
- "state_stats": state_stats})
+ item = {"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats,
+ "state_stats": state_stats}
+ if layout:
+ item["layout"] = layout[0]
+ items.append(item)
return items
def _amp_enabled_for(self, rows: int) -> bool:
@@ -1199,6 +1206,12 @@ class Agent(HookRegistry):
def _infer(self, b: Dict):
"""Run the forward pass under autocast, degrading gracefully on OOM or unsupported autocast."""
+ if "option_ids" in b and (self._fast is not None or self._compiled
+ or getattr(self._backend, "name", "eager") != "eager"):
+ # Those forwards take only the five sequential-layout inputs and would run this
+ # checkpoint's batch without its masks.
+ raise ValueError("this checkpoint uses option_layout='parallel', which only the eager "
+ "backend runs; load it without fast=/compile= or call set_backend('eager')")
if self._compiled and self._fast is None and self.device.type == "cuda":
b = _pad_cuda_compile_batch(b, self.tok.pad_token_id)
use_amp = self._amp_enabled_for(b["input_ids"].shape[0])
@@ -1230,6 +1243,8 @@ class Agent(HookRegistry):
b["marker_pos"].to(self.device),
b["marker_mask"].to(self.device),
b["qtype"].to(self.device),
+ **({"position_ids": b["position_ids"].to(self.device),
+ "option_ids": b["option_ids"].to(self.device)} if "option_ids" in b else {}),
)
# CUDA graph outputs belong to a reusable pool; callers can retain our copies.
return tuple(t.clone() for t in out) if graph_outputs else out
diff --git a/laya/backends/__init__.py b/laya/backends/__init__.py
index 08f1b116..5b16aa36 100644
--- a/laya/backends/__init__.py
+++ b/laya/backends/__init__.py
@@ -46,6 +46,8 @@ def is_modernbert(model) -> bool:
def auto_policy(agent) -> str:
"""The backend `auto` resolves to for this agent (device, dtype, encoder, installed extras)."""
+ if getattr(agent, "parallel_options", False):
+ return "eager" # the accelerated forwards do not take the parallel layout's masks
if agent.device.type == "cuda":
import torch
if tilelang_available() and is_modernbert(agent.model) and agent.dtype in (torch.bfloat16, torch.float16):
diff --git a/laya/common.py b/laya/common.py
index 1cf74f26..207cd396 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -144,6 +144,7 @@ def build_sequence(
state_ids: Optional[List[int]] = None,
return_stats: bool = False,
return_truncation_stats: bool = False,
+ return_layout: bool = False,
):
"""Format: [CLS] <type> instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP].
@@ -169,8 +170,12 @@ def build_sequence(
The question half is `build_head`; `state_room` reports how much of `max_len` is left for the
state after it, which is what a caller must size a window against.
+
+ `return_layout=True` adds one last return value, the parallel option layout from
+ `parallel_layout`: `{"position_ids": [...], "option_ids": [...]}`, one entry per token.
"""
ids, markers, stats = build_head(tok, q, head_max_len, option_order=option_order)
+ head_len = len(ids)
room = max(0, max_len - len(ids) - 1)
if state_ids is None:
state_ids = encode_text(tok, serialize_state(state).replace(tok.mask_token, " "),
@@ -178,6 +183,8 @@ def build_sequence(
# not state_ids[-room:]: with no room left, state_ids[-0:] is the whole state rather than none of it
st = state_ids[max(0, len(state_ids) - room):] if truncate_left else state_ids[:room]
ids = ids + st + [tok.sep_token_id]
+ # Laid out before the clamp: a dropped trailing option must not widen the last surviving span
+ layout = parallel_layout(markers, head_len, len(ids)) if return_layout else None
ids, markers = ids[:max_len], [m for m in markers if m < max_len]
extra = ()
if return_truncation_stats:
@@ -188,6 +195,8 @@ def build_sequence(
"state_tokens_dropped": len(state_ids) - len(st),
"truncated": len(st) < len(state_ids),
},)
+ if return_layout:
+ extra += ({k: v[:max_len] for k, v in layout.items()},)
if not return_stats:
return (ids, markers) + extra
return (ids, markers, stats) + extra
@@ -247,6 +256,77 @@ def build_head(tok, q: Dict, head_max_len: int = 192, option_order: Optional[Lis
}
+def parallel_layout(markers: List[int], head_len: int, length: int) -> Dict[str, List[int]]:
+ """Position ids and option ids that make the encoder blind to option order.
+
+ In the sequential layout option `s` sits at positions after option `s-1`, and every option
+ attends to every other, so where an option is listed changes its embedding: the slot logits
+ of five identical options spread by 3.59 (`tests/test_option_order.py`). Here every option
+ starts at the same position, the first one after the instruction's [SEP], and the head's
+ closing [SEP] and the state continue after the longest option. Together with
+ `parallel_option_masks`, which stops an option from attending to another, reordering the
+ options only reorders the marker embeddings.
+
+ `option_ids` is 0 for shared tokens (instruction, closing [SEP], state, padding) and `s + 1`
+ for the tokens of slot `s`, which runs from its [MASK] up to the next marker or, for the last
+ slot, up to the [SEP] at `head_len - 1`.
+ """
+ if not markers:
+ return {"position_ids": list(range(length)), "option_ids": [0] * length}
+ start = markers[0]
+ spans = list(zip(markers, markers[1:] + [head_len - 1]))
+ position_ids, option_ids = list(range(start)), [0] * start
+ for s, (a, b) in enumerate(spans):
+ position_ids += range(start, start + b - a)
+ option_ids += [s + 1] * (b - a)
+ after = start + max(b - a for a, b in spans)
+ position_ids += range(after, after + length - len(position_ids))
+ option_ids += [0] * (length - len(option_ids))
+ return {"position_ids": position_ids, "option_ids": option_ids}
+
+
+OPTION_LAYOUTS = ("sequential", "parallel")
+
+
+def uses_parallel_layout(cfg: Dict) -> bool:
+ """Whether a checkpoint config asks for the parallel option layout (`parallel_layout`).
+
+ The layout is a property of the weights -- a checkpoint trained on one layout reads the other
+ as a different input -- so it lives in the config, defaulting to the sequential layout every
+ published checkpoint was trained on. The parallel masks go through ModernBERT's per-layer-type
+ mask argument, which transformers 4.x does not have.
+ """
+ layout = cfg.get("option_layout", "sequential")
+ if layout not in OPTION_LAYOUTS:
+ raise ValueError("option_layout must be one of %s, got %r" % (", ".join(OPTION_LAYOUTS), layout))
+ if layout == "parallel":
+ import transformers
+ if int(transformers.__version__.split(".")[0]) < 5:
+ raise RuntimeError("option_layout='parallel' needs transformers>=5, found %s" % transformers.__version__)
+ return layout == "parallel"
+
+
+def parallel_option_masks(attention_mask: torch.Tensor, position_ids: torch.Tensor, option_ids: torch.Tensor,
+ sliding_window: Optional[int]) -> Dict[str, torch.Tensor]:
+ """Per-layer-type boolean attention masks for the parallel layout (`parallel_layout`).
+
+ A token attends to every real token except those of a *different* option. ModernBERT's local
+ layers keep their window, measured in position ids rather than sequence index: options share
+ positions, so a sequence-index window would let slot 0 see more of the state than slot 3.
+ Returned in the `{"full_attention": ..., "sliding_attention": ...}` form ModernBERT takes
+ in place of a padding mask; each mask is `[batch, 1, query, key]`, True where attention is allowed.
+ """
+ keys = attention_mask.bool()[:, None, :]
+ other_option = (option_ids[:, :, None] > 0) & (option_ids[:, None, :] > 0) \
+ & (option_ids[:, :, None] != option_ids[:, None, :])
+ full = keys & ~other_option
+ masks = {"full_attention": full[:, None]}
+ if sliding_window is not None:
+ near = (position_ids[:, :, None] - position_ids[:, None, :]).abs() <= sliding_window
+ masks["sliding_attention"] = (full & near)[:, None]
+ return masks
+
+
def state_room(tok, q: Dict, max_len: int = 512, head_max_len: int = 192) -> int:
"""How many state tokens `q` leaves inside `max_len`, which is what `build_sequence` keeps.
@@ -504,8 +584,16 @@ class DecisionModel(nn.Module):
self.temperature = torch.empty_like(self.temperature, device="cpu")
self.head_checkpointing = False
- def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder: bool = False):
- h = self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
+ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder: bool = False,
+ position_ids=None, option_ids=None):
+ if option_ids is None:
+ h = self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
+ else:
+ # Parallel option layout (`parallel_layout`). The head below has no positional
+ # encoding, so with order-blind marker embeddings the logits permute with the options.
+ masks = parallel_option_masks(attention_mask, position_ids, option_ids,
+ getattr(self.encoder.config, "sliding_window", None))
+ h = self.encoder(input_ids=input_ids, attention_mask=masks, position_ids=position_ids).last_hidden_state
if detach_encoder:
h = h.detach()
h = h + self.type_emb(qtype)[:, None, :]
@@ -815,8 +903,18 @@ def collate_items(batch, pad_id: int):
"marker_mask": mmask,
"qtype": torch.tensor([it["qtype"] for it in items]),
"label": torch.tensor([it.get("label", -1) for it in items]),
- "meta": [{k: it[k] for k in it if k not in ("ids", "markers", "target")} for it in items],
+ "meta": [{k: it[k] for k in it if k not in ("ids", "markers", "target", "layout")} for it in items],
}
if target is not None:
res["target"] = target
+ if any("layout" in it for it in items):
+ # All or none: a batch that mixes layouts would run half its rows through the wrong masks
+ if not all("layout" in it for it in items):
+ raise ValueError("collate_items: some items carry a parallel layout and some do not")
+ pos = torch.zeros((n, L), dtype=torch.long)
+ opt = torch.zeros((n, L), dtype=torch.long)
+ for i, it in enumerate(items):
+ pos[i, : len(it["ids"])] = torch.tensor(it["layout"]["position_ids"])
+ opt[i, : len(it["ids"])] = torch.tensor(it["layout"]["option_ids"])
+ res["position_ids"], res["option_ids"] = pos, opt
return res
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 5ca5fb93..1c9904e2 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -142,6 +142,11 @@ class ONNXAgent(HookRegistry):
with open(cfg_path) as f:
self.cfg = json.load(f)
+ if self.cfg.get("option_layout", "sequential") != "sequential":
+ # The exported graph takes the five sequential-layout inputs only, so it would run this
+ # checkpoint without the masks it was trained with.
+ raise ValueError("option_layout=%r is not supported by the ONNX runtime; load this "
+ "checkpoint with laya.Agent" % (self.cfg.get("option_layout"),))
if not os.path.exists(onnx_path):
raise FileNotFoundError(
diff --git a/research/eval/laya_eval.py b/research/eval/laya_eval.py
index 2900c1cd..bf50ab76 100644
--- a/research/eval/laya_eval.py
+++ b/research/eval/laya_eval.py
@@ -134,19 +134,26 @@ def score_cases(agent, cases) -> List[Any]:
here so the caller can score under more than one regime from one pass.
"""
import torch
- from laya.common import QTYPES, build_sequence, collate_items, render_options
+ from laya.common import QTYPES, build_sequence, collate_items, render_options, uses_parallel_layout
max_len = agent.cfg.get("max_len", 512)
head_max_len = agent.cfg.get("head_max_len", 192)
+ parallel = uses_parallel_layout(agent.cfg)
items = []
for state, questions in cases:
for _qid, qdef in questions.items():
q = internal_question(qdef)
- ids, markers = build_sequence(agent.tok, state, q, max_len, head_max_len)
+ ids, markers, *layout = build_sequence(agent.tok, state, q, max_len, head_max_len,
+ return_layout=parallel)
if len(markers) != len(render_options(q)):
raise ValueError("marker/option count mismatch; question exceeds head_max_len")
- items.append({"ids": ids, "markers": markers, "qtype": QTYPES[q["t"]]})
+ item = {"ids": ids, "markers": markers, "qtype": QTYPES[q["t"]]}
+ if layout:
+ item["layout"] = layout[0]
+ items.append(item)
batch = collate_items([items], agent.tok.pad_token_id)
+ extra = ({"position_ids": batch["position_ids"].to(agent.device),
+ "option_ids": batch["option_ids"].to(agent.device)} if parallel else {})
with torch.no_grad():
logits, _ = agent.model(
batch["input_ids"].to(agent.device),
@@ -154,6 +161,7 @@ def score_cases(agent, cases) -> List[Any]:
batch["marker_pos"].to(agent.device),
batch["marker_mask"].to(agent.device),
batch["qtype"].to(agent.device),
+ **extra,
)
logits = logits.float().cpu().numpy()
return [logits[i, :len(it["markers"])] for i, it in enumerate(items)]
diff --git a/scripts/test_suites.py b/scripts/test_suites.py
index b3cd5d3a..512ee675 100644
--- a/scripts/test_suites.py
+++ b/scripts/test_suites.py
@@ -31,6 +31,7 @@ SCRIPT_SUITES = [
"tests/test_predict_long.py",
"tests/test_attention_dynamic_shapes.py",
"tests/test_option_order.py",
+ "tests/test_parallel_options.py",
"tests/test_state_budget.py",
"tests/test_hooks.py",
"tests/test_hooks_api.py",
diff --git a/tests/test_parallel_options.py b/tests/test_parallel_options.py
new file mode 100644
index 00000000..34b6dc12
--- /dev/null
+++ b/tests/test_parallel_options.py
@@ -0,0 +1,182 @@
+"""`option_layout="parallel"` must make the decision logits permute with the options, exactly.
+
+In the sequential layout every option sits at later positions than the one before it and attends
+to all the others, so the slot an option is listed in changes its score: five identical options
+spread by 3.59 logits on the English checkpoint (`tests/test_option_order.py`). The parallel layout
+gives every option the same starting position and stops options attending to each other; the head
+on top has no positional encoding, so reordering the options can only reorder the logits.
+
+The checks run a tiny randomly initialised ModernBERT inside the real `DecisionModel`, so no
+weights are downloaded. They pin:
+
+- the layout itself (`parallel_layout`): shared option positions, the state after the longest option;
+- the masks: with no options marked, they reproduce the encoder's own padding/sliding-window masks;
+- equivariance: under any option order the parallel logits are the canonical ones permuted,
+ while the sequential layout on the same weights is not (so the check can fail);
+- the plumbing: `collate_items` carries the layout, refuses a mixed batch, and the config gate.
+
+Run: python tests/test_parallel_options.py
+"""
+import itertools
+import os
+import sys
+
+import torch
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import transformers # noqa: E402
+
+if int(transformers.__version__.split(".")[0]) < 5:
+ print("skipped: the parallel layout needs transformers>=5, found %s" % transformers.__version__)
+ sys.exit(0)
+
+from transformers import ModernBertConfig, ModernBertModel # noqa: E402
+
+from laya.common import ( # noqa: E402
+ DecisionModel,
+ build_sequence,
+ collate_items,
+ parallel_layout,
+ parallel_option_masks,
+ uses_parallel_layout,
+)
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s: got %r, want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s %s" % (name, detail))
+
+
+def check_raises(name, exc, fn):
+ try:
+ fn()
+ except exc:
+ PASS.append(name)
+ return
+ except Exception as e: # noqa: BLE001 - the point is the wrong type
+ FAIL.append("%s: raised %s" % (name, type(e).__name__))
+ return
+ FAIL.append("%s: did not raise" % name)
+
+
+class StubTok:
+ mask_token = "[MASK]"
+ mask_token_id, cls_token_id, sep_token_id, pad_token_id = 3, 1, 2, 0
+
+ def __init__(self):
+ self._vocab = {}
+
+ def __call__(self, text, add_special_tokens=True, truncation=False, max_length=None):
+ ids = [self._vocab.setdefault(w, len(self._vocab) + 10) for w in str(text).split()]
+ return {"input_ids": ids[:max_length] if (truncation and max_length) else ids}
+
+
+# ------------------------------------------------------------------ the layout
+# [CLS] a b [SEP] | [MASK] x | [MASK] y z | [MASK] | [SEP] s s [SEP]
+# 0 1 2 3 4 5 6 7 8 9 10 11 12 13
+lay = parallel_layout([4, 6, 9], head_len=11, length=14)
+check("layout/instruction keeps its positions", lay["position_ids"][:4], [0, 1, 2, 3])
+check("layout/every option starts at the same position",
+ [lay["position_ids"][m] for m in (4, 6, 9)], [4, 4, 4])
+check("layout/option tokens count up inside their span", lay["position_ids"][4:10], [4, 5, 4, 5, 6, 4])
+check("layout/closing [SEP] and state follow the longest option", lay["position_ids"][10:], [7, 8, 9, 10])
+check("layout/option ids mark each slot, 0 elsewhere", lay["option_ids"], [0] * 4 + [1, 1, 2, 2, 2, 3] + [0] * 4)
+check("layout/no options is the identity", parallel_layout([], 4, 6)["position_ids"], list(range(6)))
+
+tok = StubTok()
+QI = {"t": "choice", "ins": "which team handles this",
+ "crit": {"refund": "money back please", "tech": "a bug", "sales": "pricing", "other": "none of these fit"}}
+STATE = {"msg": " ".join("w%d" % i for i in range(40))}
+ids, markers, layout = build_sequence(tok, STATE, QI, 512, 192, return_layout=True)
+check("build/one layout entry per token", (len(layout["position_ids"]), len(layout["option_ids"])), (len(ids),) * 2)
+check("build/without the flag the return shape is unchanged", len(build_sequence(tok, STATE, QI, 512, 192)), 2)
+short_ids, short_markers, short_layout = build_sequence(tok, STATE, QI, 24, 192, return_layout=True)
+check("build/clamped to max_len with the ids", len(short_layout["option_ids"]), len(short_ids))
+check_true("build/a clamp does not widen the last surviving option",
+ short_layout["option_ids"][:len(short_ids)] == layout["option_ids"][:len(short_ids)])
+
+# ------------------------------------------------------------------ the model
+torch.manual_seed(0)
+enc_cfg = ModernBertConfig(vocab_size=512, hidden_size=64, intermediate_size=128, num_hidden_layers=4,
+ num_attention_heads=4, local_attention=8, global_attn_every_n_layers=2,
+ pad_token_id=0, cls_token_id=1, sep_token_id=2, bos_token_id=1, eos_token_id=2,
+ attn_implementation="sdpa")
+model = DecisionModel(ModernBertModel(enc_cfg), head_layers=2).eval()
+
+
+def logits_for(order, parallel):
+ seq, mk, *lay_ = build_sequence(tok, STATE, QI, 512, 192, option_order=order, return_layout=parallel)
+ item = {"ids": seq, "markers": mk, "qtype": 0}
+ if lay_:
+ item["layout"] = lay_[0]
+ b = collate_items([[item]], tok.pad_token_id)
+ extra = {"position_ids": b["position_ids"], "option_ids": b["option_ids"]} if parallel else {}
+ with torch.no_grad():
+ logits, act = model(b["input_ids"], b["attention_mask"], b["marker_pos"], b["marker_mask"], b["qtype"],
+ **extra)
+ return logits[0], act[0]
+
+
+# ------------------------------------------------------------------ masks match the stock ones
+# No options marked and plain positions: the hand-built masks must be the encoder's own, padding included.
+batch = collate_items([[{"ids": ids, "markers": markers, "qtype": 0},
+ {"ids": ids[:30], "markers": markers, "qtype": 0}]], tok.pad_token_id)
+plain_pos = torch.arange(batch["input_ids"].shape[1]).expand_as(batch["input_ids"])
+masks = parallel_option_masks(batch["attention_mask"], plain_pos, torch.zeros_like(plain_pos),
+ enc_cfg.sliding_window)
+with torch.no_grad():
+ stock = model.encoder(input_ids=batch["input_ids"], attention_mask=batch["attention_mask"]).last_hidden_state
+ ours = model.encoder(input_ids=batch["input_ids"], attention_mask=masks, position_ids=plain_pos).last_hidden_state
+real = batch["attention_mask"].bool()
+check_true("masks/reproduce the stock padding + sliding-window masks",
+ torch.allclose(stock[real], ours[real], atol=1e-5),
+ "max diff %.2e" % (stock[real] - ours[real]).abs().max().item())
+
+# ------------------------------------------------------------------ equivariance
+canon_par, canon_act = logits_for(None, parallel=True)
+worst_par, worst_seq, worst_act = 0.0, 0.0, 0.0
+canon_seq, _ = logits_for(None, parallel=False)
+for order in itertools.permutations(range(4)):
+ order = list(order)
+ par, act = logits_for(order, parallel=True)
+ seq_, _ = logits_for(order, parallel=False)
+ # slot s shows option order[s], so slot s must score what option order[s] scored canonically
+ worst_par = max(worst_par, (par[:4] - canon_par[order]).abs().max().item())
+ worst_seq = max(worst_seq, (seq_[:4] - canon_seq[order]).abs().max().item())
+ worst_act = max(worst_act, (act - canon_act).abs().max().item())
+check_true("equivariant/parallel logits permute with the options (all 24 orders)", worst_par < 1e-5,
+ "max diff %.2e" % worst_par)
+check_true("equivariant/the act head is order-blind too", worst_act < 1e-5, "max diff %.2e" % worst_act)
+check_true("equivariant/the sequential layout is not (the check can fail)", worst_seq > 1e-4,
+ "max diff %.2e" % worst_seq)
+
+# ------------------------------------------------------------------ plumbing
+with_layout = {"ids": ids, "markers": markers, "qtype": 0, "layout": layout}
+b = collate_items([[with_layout]], tok.pad_token_id)
+check("collate/carries position ids", b["position_ids"][0].tolist(), layout["position_ids"])
+check_true("collate/keeps the layout out of meta", "layout" not in b["meta"][0])
+check_true("collate/no layout, no layout tensors",
+ "option_ids" not in collate_items([[{"ids": ids, "markers": markers, "qtype": 0}]], 0))
+check_raises("collate/refuses a batch mixing layouts", ValueError,
+ lambda: collate_items([[with_layout, {"ids": ids, "markers": markers, "qtype": 0}]], 0))
+
+check("config/published checkpoints stay sequential", uses_parallel_layout({}), False)
+check("config/parallel is opt-in", uses_parallel_layout({"option_layout": "parallel"}), True)
+check_raises("config/rejects an unknown layout", ValueError, lambda: uses_parallel_layout({"option_layout": "set"}))
+
+if FAIL:
+ print("FAILED:\n " + "\n ".join(FAIL))
+ sys.exit(1)
+print("all parallel-option tests passed (%d checks)" % len(PASS))