Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO PrefixGrouper: build the shared-prefix layout from one host copy instead of per-token loops and per-segment device fills (#15835)
Daniel Han9d26414ca40d
diff --git a/tests/test_prefix_grouper_layout.py b/tests/test_prefix_grouper_layout.pynew file mode 100644index 00000000000..450e59d1978--- /dev/null+++ b/tests/test_prefix_grouper_layout.py@@ -0,0 +1,268 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""PrefixGrouper layout: the single-copy builder matches the token-by-token reference, and its+host / device traffic does not grow with the number of groups or segments."""++import importlib.util+import random+import sys+import types+from pathlib import Path++import pytest+import torch++_UTILS = Path(__file__).resolve().parents[1] / "unsloth" / "utils"+++def _load():+ pkg_name = "_pg_layout_test_pkg"+ if pkg_name not in sys.modules:+ pkg = types.ModuleType(pkg_name)+ pkg.__path__ = [str(_UTILS)]+ sys.modules[pkg_name] = pkg+ mods = {}+ for name in ("prefix_grouper_kernel", "prefix_grouper"):+ full = f"{pkg_name}.{name}"+ if full not in sys.modules:+ spec = importlib.util.spec_from_file_location(full, _UTILS / f"{name}.py")+ module = importlib.util.module_from_spec(spec)+ sys.modules[full] = module+ spec.loader.exec_module(module)+ mods[name] = sys.modules[full]+ return mods["prefix_grouper"], mods["prefix_grouper_kernel"]+++pg, pgk = _load()++_LAYOUT_TENSORS = ("flat_ids", "position_ids", "tgt_rows", "tgt_cols", "tgt_pred", "tgt_flat")+_SEG_TENSORS = ("group_of_kv", "is_prefix", "suffix_of_kv")+++def _assert_same(ref, got):+ if ref is None or got is None:+ assert ref is None and got is None+ return+ for name in _LAYOUT_TENSORS:+ a, b = getattr(ref, name), getattr(got, name)+ assert a.dtype == b.dtype and a.shape == b.shape and a.device == b.device, name+ assert torch.equal(a, b), name+ assert b.is_contiguous() and b.storage_offset() == 0, name+ for name in ("total_rows", "L", "W", "tok_r", "signature"):+ assert getattr(ref, name) == getattr(got, name), name+ for name in _SEG_TENSORS:+ a, b = getattr(ref.prefix_seg_info, name), getattr(got.prefix_seg_info, name)+ assert a.dtype == b.dtype and a.shape == b.shape and torch.equal(a, b), name+ assert b.is_contiguous() and b.storage_offset() == 0, name+ for name in ("signature", "T", "T_pad"):+ assert getattr(ref.prefix_seg_info, name) == getattr(got.prefix_seg_info, name), name+++def _batch(+ rng,+ num_groups,+ G,+ P,+ C,+ pad = 0,+ left_packed = True,+):+ """GRPO-shaped rows: G completions per prompt, left pad in the prompt, right pad after+ each completion; left_packed shifts every real run to column 0 like the de-padded input."""+ L = P + C+ ids = torch.randint(1, 50, (num_groups * G, L), generator = rng)+ ids[ids == pad] = pad + 1+ lp = torch.zeros(num_groups * G, dtype = torch.long)+ for g in range(num_groups):+ rows = range(g * G, (g + 1) * G)+ plen = int(torch.randint(1, P + 1, (1,), generator = rng))+ prompt = ids[rows[0], :P].clone()+ for r in rows:+ ids[r, :P] = prompt+ ids[r, : P - plen] = pad+ lp[r] = P - plen+ clen = int(torch.randint(0, C + 1, (1,), generator = rng))+ ids[r, P + clen :] = pad+ if left_packed:+ k = P - plen+ ids[r] = torch.cat([ids[r, k:], torch.full((k,), pad, dtype = ids.dtype)])+ return ids, lp+++def _cases():+ rng = torch.Generator().manual_seed(3407)+ pick = random.Random(3407)+ for i in range(400):+ G = pick.choice([2, 3, 4, 8])+ num_groups = pick.choice([1, 2, 3])+ P = pick.choice([1, 4, 17, 64])+ C = pick.choice([1, 5, 33, 96])+ pad = pick.choice([0, 7])+ ids, lp = _batch(rng, num_groups, G, P, C, pad, left_packed = pick.random() < 0.7)+ kind = pick.choice(["plain"] * 6 + ["hole", "other_prompt", "lp_shift", "int32", "bad_G"])+ if kind == "hole": # a pad token inside a real run: not one contiguous run+ ids[pick.randrange(ids.shape[0]), pick.randrange(ids.shape[1])] = pad+ elif kind == "other_prompt" and ids.shape[0] > 1:+ ids[1, 0] = (int(ids[1, 0]) % 40) + 3+ elif kind == "lp_shift":+ lp = lp + torch.randint(-2, 3, lp.shape, generator = rng)+ elif kind == "int32":+ ids, lp = ids.to(torch.int32), lp.to(torch.int32)+ num_gen = G+ if kind == "bad_G":+ num_gen = pick.choice([None, 1, G + 1])+ ltk = C + pick.choice([0, 0, 1])+ kwargs = dict(+ apply_tokr_gate = pick.random() < 0.3,+ max_segment_cap = pick.choice([None, None, P + C // 2]),+ )+ yield i, (ids, ltk, pad, num_gen, lp), kwargs+++def test_layout_matches_token_by_token_reference():+ built = 0+ for i, args, kwargs in _cases():+ ref = pg._build_group_layout_reference(*args, **kwargs)+ got = pg.build_group_layout(*args, **kwargs)+ _assert_same(ref, got)+ built += got is not None+ assert built >= 100+++def test_seg_info_matches_per_segment_build():+ specs = [(5, [3, 0, 7]), (1, [1, 1]), (130, [2, 9, 0, 4])]+ seg, meta = pgk.build_seg_info_multigroup(specs, torch.device("cpu"))+ group_of, is_prefix, suffix_of = [], [], []+ counter = 0+ for gid, (P, Rs) in enumerate(specs):+ group_of += [gid] * P+ is_prefix += [True] * P+ suffix_of += [-1] * P+ for r in Rs:+ group_of += [gid] * r+ is_prefix += [False] * r+ suffix_of += [counter] * r+ counter += 1+ T = len(group_of)+ T_pad = pgk._pad_len(T)+ pad = T_pad - T+ assert (seg.T, seg.T_pad) == (T, T_pad)+ assert torch.equal(seg.group_of_kv, torch.tensor(group_of + [pgk._PAD_GROUP] * pad))+ assert torch.equal(seg.is_prefix, torch.tensor(is_prefix + [False] * pad))+ assert torch.equal(seg.suffix_of_kv, torch.tensor(suffix_of + [pgk._PAD_GROUP] * pad))+ assert seg.signature == ("multi", T_pad, tuple((P, tuple(Rs)) for P, Rs in specs))+ assert meta[2]["prefix_last_index"] == meta[2]["base"] + 129+ assert meta[0]["suffix_slices"] == [(5, 8), (8, 8), (8, 15)]+++class _CountFactories(torch.overrides.TorchFunctionMode):+ """Counts tensor creations that name a device (each one a fill kernel or a host to+ device copy when that device is an accelerator)."""++ def __init__(self):+ super().__init__()+ self.calls = 0++ def __torch_function__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ kwargs = kwargs or {}+ if kwargs.get("device") is not None and func in (+ torch.tensor,+ torch.full,+ torch.ones,+ torch.zeros,+ ):+ self.calls += 1+ return func(*args, **kwargs)+++def _factory_calls(num_groups, G):+ rng = torch.Generator().manual_seed(0)+ ids, lp = _batch(rng, num_groups, G, 64, 16)+ with _CountFactories() as mode:+ layout = pg.build_group_layout(ids, 16, 0, G, lp, apply_tokr_gate = False)+ assert layout is not None+ return mode.calls+++def test_device_tensor_creations_do_not_grow_with_groups():+ # The per-segment build made three device tensors per prefix and per completion.+ assert _factory_calls(1, 8) == _factory_calls(6, 8) == _factory_calls(6, 2)++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_cuda_layout_reads_the_device_once_and_copies_once():+ import warnings++ rng = torch.Generator().manual_seed(0)+ ids, lp = _batch(rng, 4, 8, 256, 128)+ ids, lp = ids.cuda(), lp.cuda()+ ref = pg._build_group_layout_reference(ids, 128, 0, 8, lp)+ got = pg.build_group_layout(ids, 128, 0, 8, lp)+ _assert_same(ref, got)++ def syncs(fn):+ torch.cuda.synchronize()+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ torch.cuda.set_sync_debug_mode("warn")+ try:+ fn(ids, 128, 0, 8, lp)+ finally:+ torch.cuda.set_sync_debug_mode(0)+ # Count the per-operation reports, not set_sync_debug_mode's "prototype feature" notice.+ return [str(w.message) for w in caught if "called a synchronizing" in str(w.message)]++ # The counter sees the token-by-token build's reads (cstart, ids, contiguity, ...).+ assert len(syncs(pg._build_group_layout_reference)) > 2+ # One device to host read of the ids + one host to device copy of the index tables.+ got_syncs = syncs(pg.build_group_layout)+ assert len(got_syncs) <= 2, got_syncs+++def _outcome(fn, *args, **kwargs):+ try:+ return ("ok", fn(*args, **kwargs))+ except Exception as exc:+ return ("err", type(exc), str(exc))+++def test_unusual_arguments_take_the_reference_decision():+ # A float num_generations must raise the reference's error, a numpy one must keep a+ # plain int signature, and a pad id outside the ids dtype must compare like the reference.+ np = pytest.importorskip("numpy")+ rng = torch.Generator().manual_seed(0)+ ids, lp = _batch(rng, 2, 4, 32, 16)+ cases = [+ ((ids, 16, 0, 4.0, lp), {}),+ ((ids, 16, 0, np.int64(4), lp), dict(apply_tokr_gate = False)),+ ((ids.to(torch.int32), 16, 2**32, 4, lp), dict(apply_tokr_gate = False)),+ ]+ for args, kwargs in cases:+ ref = _outcome(pg._build_group_layout_reference, *args, **kwargs)+ got = _outcome(pg.build_group_layout, *args, **kwargs)+ assert ref[0] == got[0], (ref, got)+ if ref[0] == "err":+ assert ref[1:] == got[1:]+ else:+ _assert_same(ref[1], got[1])+ if ref[1] is not None:+ assert type(got[1].signature[0]) is type(ref[1].signature[0])+++def test_host_tables_ignore_the_default_device():+ rng = torch.Generator().manual_seed(0)+ ids, lp = _batch(rng, 3, 4, 32, 16)+ ref = pg._build_group_layout_reference(ids, 16, 0, 4, lp, apply_tokr_gate = False)+ torch.set_default_device("meta")+ try:+ got = pg.build_group_layout(ids, 16, 0, 4, lp, apply_tokr_gate = False)+ finally:+ torch.set_default_device(None)+ _assert_same(ref, got)diff --git a/unsloth/utils/prefix_grouper.py b/unsloth/utils/prefix_grouper.pyindex 744dc652d3e..1716784f2c7 100644--- a/unsloth/utils/prefix_grouper.py+++ b/unsloth/utils/prefix_grouper.py@@ -225,7 +225,7 @@ def _tok_r(groups) -> float:return (tok_full / tok_sp) if tok_sp else 1.0-def build_group_layout(+def _build_group_layout_reference(input_ids,logits_to_keep,pad_id,@@ -236,6 +236,7 @@ def build_group_layout(max_segment_cap = None,):"""Build the shared-prefix GroupLayout, or return None to fall back to the packed path.+ Token-by-token builder; build_group_layout's fallback for inputs its fast path skips.input_ids : [B, L]. GRPO's layout is left-padded in the prompt and right-padded inthe completion. Real tokens of a row are a contiguous run not necessarily@@ -358,6 +359,184 @@ def build_group_layout()+_INDEX_DTYPES = (torch.int64, torch.int32)+++def _is_int(x) -> bool:+ return isinstance(x, int) and not isinstance(x, bool)+++def build_group_layout(+ input_ids,+ logits_to_keep,+ pad_id,+ num_generations,+ left_pad_tokens_per_prompt,+ *,+ apply_tokr_gate = True,+ max_segment_cap = None,+):+ """Build the shared-prefix GroupLayout, or return None to fall back to the packed path.++ input_ids : [B, L]. GRPO's layout is left-padded in the prompt and right-padded in+ the completion. Real tokens of a row are a contiguous run not necessarily+ starting at column 0.+ logits_to_keep : int+ left_pad_tokens_per_prompt : [B] long tensor (per-row left-pad count in the prompt).++ Same result as _build_group_layout_reference, with one device->host and one+ host->device copy instead of per-token loops.+ """+ args = (input_ids, logits_to_keep, pad_id, num_generations, left_pad_tokens_per_prompt)+ kwargs = dict(apply_tokr_gate = apply_tokr_gate, max_segment_cap = max_segment_cap)+ lp = left_pad_tokens_per_prompt+ if not (+ isinstance(input_ids, torch.Tensor)+ and input_ids.dim() == 2+ and input_ids.shape[0] > 0+ and input_ids.dtype in _INDEX_DTYPES+ and isinstance(lp, torch.Tensor)+ and lp.dim() == 1+ and lp.shape[0] == input_ids.shape[0]+ and lp.dtype in _INDEX_DTYPES+ and lp.device == input_ids.device+ and _is_int(logits_to_keep)+ and _is_int(pad_id)+ and (num_generations is None or _is_int(num_generations))+ and torch.iinfo(input_ids.dtype).min <= pad_id <= torch.iinfo(input_ids.dtype).max+ ):+ return _build_group_layout_reference(*args, **kwargs)+ device = input_ids.device+ total_rows, L = input_ids.shape+ G = num_generations+ # Completion start column per row, matching create_completion_attention_mask / _pk_cstart.+ cstart = ((L - logits_to_keep) - lp).to(torch.long)+ host = torch.cat(+ [input_ids.to(torch.long), cstart.unsqueeze(1), lp.to(torch.long).unsqueeze(1)], dim = 1+ ).cpu()+ ids = host[:, :L]+ cs = host[:, L]+ keep = ids != pad_id+ n_real = keep.sum(dim = 1)+ first = torch.argmax(keep.to(torch.int8), dim = 1)+ ar = torch.arange(L, device = "cpu")+ end = first + n_real+ if not bool((keep == ((ar >= first.unsqueeze(1)) & (ar < end.unsqueeze(1)))).all()):+ return _build_group_layout_reference(*args, **kwargs)+ if G is None or G < 2 or total_rows % G != 0:+ return None+ # Prompt real columns [first, pe), completion [pe, end).+ pe = torch.minimum(torch.maximum(cs, first), end)+ P_row = pe - first+ R_row = end - pe+ num_groups = total_rows // G+ P_g = P_row.view(num_groups, G)+ if bool((P_g == 0).any()) or bool((P_g != P_g[:, :1]).any()):+ return None+ R_g = R_row.view(num_groups, G)+ if bool((R_g.sum(dim = 1) == 0).any()):+ return None+ first_g = first.view(num_groups, G)+ P_list = P_g[:, 0].tolist()+ for gi, P in enumerate(P_list):+ cols = first_g[gi].unsqueeze(1) + torch.arange(P, device = "cpu")+ prompts = ids[gi * G : (gi + 1) * G].gather(1, cols)+ if not bool((prompts == prompts[:1]).all()):+ return None+ R_lists = R_g.tolist()+ groups = [dict(P = P, R_list = R_list) for P, R_list in zip(P_list, R_lists)]++ # Sliding-window guard: a group's PG span is P + max(R), so fall back if it exceeds the window.+ if max_segment_cap is not None:+ for gm in groups:+ if gm["P"] + max(gm["R_list"]) > max_segment_cap:+ return None++ tok_r = _tok_r(groups)+ if apply_tokr_gate and tok_r < tokr_threshold():+ return None # low reuse -> not worth it; use the full-row packed path++ from .prefix_grouper_kernel import _labels_to_device, _seg_labels_host++ group_specs = [(gm["P"], gm["R_list"]) for gm in groups]+ labels, T, T_pad, _, sig_parts = _seg_labels_host(group_specs)++ # Flat order per group: shared prefix (from its first row), then one suffix per row.+ rows_g = torch.arange(total_rows, device = "cpu").view(num_groups, G)+ P_t = P_g[:, :1]+ seg_len = torch.cat([P_t, R_g], dim = 1).reshape(-1)+ seg_row = torch.cat([rows_g[:, :1], rows_g], dim = 1).reshape(-1)+ seg_col = torch.cat([first_g[:, :1], pe.view(num_groups, G)], dim = 1).reshape(-1)+ seg_pos = torch.cat([torch.zeros_like(P_t), P_t.expand(num_groups, G)], dim = 1).reshape(-1)+ seg_is_suffix = torch.ones((num_groups, G + 1), dtype = torch.bool, device = "cpu")+ seg_is_suffix[:, 0] = False+ seg_is_suffix = seg_is_suffix.reshape(-1)+ seg_start = torch.cumsum(seg_len, dim = 0) - seg_len+ group_base = seg_start.view(num_groups, G + 1)[:, :1]+ seg_plast = (group_base + P_t - 1).expand(num_groups, G + 1).reshape(-1)++ flat = torch.arange(T, device = "cpu")+ j = flat - torch.repeat_interleave(seg_start, seg_len, output_size = T)+ flat_src_rows = torch.repeat_interleave(seg_row, seg_len, output_size = T)+ flat_src_cols = torch.repeat_interleave(seg_col, seg_len, output_size = T) + j+ pos = torch.repeat_interleave(seg_pos, seg_len, output_size = T) + j+ is_tgt = torch.repeat_interleave(seg_is_suffix, seg_len, output_size = T)+ tgt_flat = flat[is_tgt]+ tgt_j = j[is_tgt]+ # Position 0 is predicted from the prefix's last token; j >= 1 from the previous suffix token.+ tgt_pred = torch.where(+ tgt_j == 0,+ torch.repeat_interleave(seg_plast, seg_len, output_size = T)[is_tgt],+ tgt_flat - 1,+ )+ N = int(tgt_flat.numel())+ table = torch.cat(+ [+ flat_src_rows,+ flat_src_cols,+ pos,+ flat_src_rows[is_tgt],+ flat_src_cols[is_tgt],+ tgt_pred,+ tgt_flat,+ labels.reshape(-1),+ ]+ ).to(device)+ pieces = torch.split(table, [T, T, T, N, N, N, N, 3 * T_pad])+ fr, fc, position_ids, tgt_rows, tgt_cols, tgt_pred_d, tgt_flat_d, labels_d = pieces+ group_of_kv, is_prefix, suffix_of_kv = _labels_to_device(labels_d.view(3, T_pad), T_pad, device)+ seg = PrefixSegInfo(+ group_of_kv = group_of_kv,+ is_prefix = is_prefix,+ suffix_of_kv = suffix_of_kv,+ signature = ("multi", T_pad, tuple(sig_parts)),+ T = T,+ T_pad = T_pad,+ )+ flat_ids = input_ids[fr, fc].unsqueeze(0) # [1, T] (grad-safe gather)++ max_left_pad = int(host[:, L + 1].max().item())+ W = logits_to_keep + max_left_pad++ # The logic is structural, so key on (num_groups, group_sizes), not exact lengths.+ sig = (num_groups, tuple(sorted(len(gm["R_list"]) for gm in groups)))++ return GroupLayout(+ flat_ids = flat_ids,+ position_ids = position_ids.clone().unsqueeze(0),+ prefix_seg_info = seg,+ tgt_rows = tgt_rows.clone(),+ tgt_cols = tgt_cols.clone(),+ tgt_pred = tgt_pred_d.clone(),+ tgt_flat = tgt_flat_d.clone(),+ total_rows = total_rows,+ L = L,+ W = W,+ tok_r = tok_r,+ signature = sig,+ )++__all__ = ["GroupLayout","build_group_layout",diff --git a/unsloth/utils/prefix_grouper_kernel.py b/unsloth/utils/prefix_grouper_kernel.pyindex ae5a873d4d8..e76560d3767 100644--- a/unsloth/utils/prefix_grouper_kernel.py+++ b/unsloth/utils/prefix_grouper_kernel.py@@ -181,26 +181,45 @@ def build_seg_info_multigroup(group_meta : list of dicts with 'base', 'P', 'prefix_last_index', 'suffix_slices'(flat offsets), enough to build the completion index map."""- group_of_list = []- is_prefix_list = []- suffix_of_list = []- group_meta = []+ labels, T, T_pad, group_meta, sig_parts = _seg_labels_host(group_specs)+ group_of_kv, is_prefix, suffix_of_kv = _labels_to_device(labels, T_pad, device)+ sig = ("multi", T_pad, tuple(sig_parts))+ seg = PrefixSegInfo(+ group_of_kv = group_of_kv,+ is_prefix = is_prefix,+ suffix_of_kv = suffix_of_kv,+ signature = sig,+ T = T,+ T_pad = T_pad,+ )+ return seg, group_meta++def _seg_labels_host(group_specs):+ """CPU int64 [3, T_pad] rows (group_of_kv, is_prefix, suffix_of_kv), pad sentinels+ included, plus T, T_pad, group_meta and signature parts."""+ group_meta = []+ sig_parts = []+ seg_lens = []+ seg_group = []+ seg_prefix = []+ seg_suffix = []base = 0suffix_counter = 0- sig_parts = []+ if not group_specs:+ torch.cat([]) # the empty-list error the per-segment build raisedfor gid, (P, R_list) in enumerate(group_specs):- # Prefix.- group_of_list.append(torch.full((P,), gid, dtype = torch.long, device = device))- is_prefix_list.append(torch.ones(P, dtype = torch.bool, device = device))- suffix_of_list.append(torch.full((P,), -1, dtype = torch.long, device = device))- prefix_last_index = base + P - 1+ seg_lens.append(P)+ seg_group.append(gid)+ seg_prefix.append(1)+ seg_suffix.append(-1)suffix_slices = []cursor = base + Pfor r in R_list:- group_of_list.append(torch.full((r,), gid, dtype = torch.long, device = device))- is_prefix_list.append(torch.zeros(r, dtype = torch.bool, device = device))- suffix_of_list.append(torch.full((r,), suffix_counter, dtype = torch.long, device = device))+ seg_lens.append(r)+ seg_group.append(gid)+ seg_prefix.append(0)+ seg_suffix.append(suffix_counter)suffix_slices.append((cursor, cursor + r))cursor += rsuffix_counter += 1@@ -208,29 +227,28 @@ def build_seg_info_multigroup({"base": base,"P": P,- "prefix_last_index": prefix_last_index,+ "prefix_last_index": base + P - 1,"suffix_slices": suffix_slices,})sig_parts.append((P, tuple(R_list)))base = cursor-- group_of_kv = torch.cat(group_of_list)- is_prefix = torch.cat(is_prefix_list)- suffix_of_kv = torch.cat(suffix_of_list)- group_of_kv, is_prefix, suffix_of_kv, T, T_pad = _pad_labels(- group_of_kv, is_prefix, suffix_of_kv, device- )- sig = ("multi", T_pad, tuple(sig_parts))- seg = PrefixSegInfo(- group_of_kv = group_of_kv,- is_prefix = is_prefix,- suffix_of_kv = suffix_of_kv,- signature = sig,- T = T,- T_pad = T_pad,- )- return seg, group_meta+ T = base+ T_pad = _pad_len(T)+ labels = torch.empty((3, T_pad), dtype = torch.long, device = "cpu")+ lens = torch.tensor(seg_lens, dtype = torch.long, device = "cpu")+ per_seg = torch.tensor([seg_group, seg_prefix, seg_suffix], dtype = torch.long, device = "cpu")+ labels[:, :T] = torch.repeat_interleave(per_seg, lens, dim = 1, output_size = T)+ labels[0, T:] = _PAD_GROUP+ labels[1, T:] = 0+ labels[2, T:] = _PAD_GROUP+ return labels, T, T_pad, group_meta, sig_parts+++def _labels_to_device(labels, T_pad, device):+ """One host->device copy; each returned tensor gets its own storage."""+ moved = labels.to(device)+ return moved[0].clone(), moved[1].to(torch.bool), moved[2].clone()# Block-mask cache, keyed on (signature, device): the mask depends only on per-token labels and T.