Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO PrefixGrouper: build the shared-prefix layout from one host copy instead of per-token loops and per-segment device fills (#15835)

Daniel Han9d26414ca40d
diff --git a/tests/test_prefix_grouper_layout.py b/tests/test_prefix_grouper_layout.py
new file mode 100644
index 00000000000..450e59d1978
--- /dev/null
+++ b/tests/test_prefix_grouper_layout.py
@@ -0,0 +1,268 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""PrefixGrouper layout: the single-copy builder matches the token-by-token reference, and its
+host / device traffic does not grow with the number of groups or segments."""
+
+import importlib.util
+import random
+import sys
+import types
+from pathlib import Path
+
+import pytest
+import torch
+
+_UTILS = Path(__file__).resolve().parents[1] / "unsloth" / "utils"
+
+
+def _load():
+ pkg_name = "_pg_layout_test_pkg"
+ if pkg_name not in sys.modules:
+ pkg = types.ModuleType(pkg_name)
+ pkg.__path__ = [str(_UTILS)]
+ sys.modules[pkg_name] = pkg
+ mods = {}
+ for name in ("prefix_grouper_kernel", "prefix_grouper"):
+ full = f"{pkg_name}.{name}"
+ if full not in sys.modules:
+ spec = importlib.util.spec_from_file_location(full, _UTILS / f"{name}.py")
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[full] = module
+ spec.loader.exec_module(module)
+ mods[name] = sys.modules[full]
+ return mods["prefix_grouper"], mods["prefix_grouper_kernel"]
+
+
+pg, pgk = _load()
+
+_LAYOUT_TENSORS = ("flat_ids", "position_ids", "tgt_rows", "tgt_cols", "tgt_pred", "tgt_flat")
+_SEG_TENSORS = ("group_of_kv", "is_prefix", "suffix_of_kv")
+
+
+def _assert_same(ref, got):
+ if ref is None or got is None:
+ assert ref is None and got is None
+ return
+ for name in _LAYOUT_TENSORS:
+ a, b = getattr(ref, name), getattr(got, name)
+ assert a.dtype == b.dtype and a.shape == b.shape and a.device == b.device, name
+ assert torch.equal(a, b), name
+ assert b.is_contiguous() and b.storage_offset() == 0, name
+ for name in ("total_rows", "L", "W", "tok_r", "signature"):
+ assert getattr(ref, name) == getattr(got, name), name
+ for name in _SEG_TENSORS:
+ a, b = getattr(ref.prefix_seg_info, name), getattr(got.prefix_seg_info, name)
+ assert a.dtype == b.dtype and a.shape == b.shape and torch.equal(a, b), name
+ assert b.is_contiguous() and b.storage_offset() == 0, name
+ for name in ("signature", "T", "T_pad"):
+ assert getattr(ref.prefix_seg_info, name) == getattr(got.prefix_seg_info, name), name
+
+
+def _batch(
+ rng,
+ num_groups,
+ G,
+ P,
+ C,
+ pad = 0,
+ left_packed = True,
+):
+ """GRPO-shaped rows: G completions per prompt, left pad in the prompt, right pad after
+ each completion; left_packed shifts every real run to column 0 like the de-padded input."""
+ L = P + C
+ ids = torch.randint(1, 50, (num_groups * G, L), generator = rng)
+ ids[ids == pad] = pad + 1
+ lp = torch.zeros(num_groups * G, dtype = torch.long)
+ for g in range(num_groups):
+ rows = range(g * G, (g + 1) * G)
+ plen = int(torch.randint(1, P + 1, (1,), generator = rng))
+ prompt = ids[rows[0], :P].clone()
+ for r in rows:
+ ids[r, :P] = prompt
+ ids[r, : P - plen] = pad
+ lp[r] = P - plen
+ clen = int(torch.randint(0, C + 1, (1,), generator = rng))
+ ids[r, P + clen :] = pad
+ if left_packed:
+ k = P - plen
+ ids[r] = torch.cat([ids[r, k:], torch.full((k,), pad, dtype = ids.dtype)])
+ return ids, lp
+
+
+def _cases():
+ rng = torch.Generator().manual_seed(3407)
+ pick = random.Random(3407)
+ for i in range(400):
+ G = pick.choice([2, 3, 4, 8])
+ num_groups = pick.choice([1, 2, 3])
+ P = pick.choice([1, 4, 17, 64])
+ C = pick.choice([1, 5, 33, 96])
+ pad = pick.choice([0, 7])
+ ids, lp = _batch(rng, num_groups, G, P, C, pad, left_packed = pick.random() < 0.7)
+ kind = pick.choice(["plain"] * 6 + ["hole", "other_prompt", "lp_shift", "int32", "bad_G"])
+ if kind == "hole": # a pad token inside a real run: not one contiguous run
+ ids[pick.randrange(ids.shape[0]), pick.randrange(ids.shape[1])] = pad
+ elif kind == "other_prompt" and ids.shape[0] > 1:
+ ids[1, 0] = (int(ids[1, 0]) % 40) + 3
+ elif kind == "lp_shift":
+ lp = lp + torch.randint(-2, 3, lp.shape, generator = rng)
+ elif kind == "int32":
+ ids, lp = ids.to(torch.int32), lp.to(torch.int32)
+ num_gen = G
+ if kind == "bad_G":
+ num_gen = pick.choice([None, 1, G + 1])
+ ltk = C + pick.choice([0, 0, 1])
+ kwargs = dict(
+ apply_tokr_gate = pick.random() < 0.3,
+ max_segment_cap = pick.choice([None, None, P + C // 2]),
+ )
+ yield i, (ids, ltk, pad, num_gen, lp), kwargs
+
+
+def test_layout_matches_token_by_token_reference():
+ built = 0
+ for i, args, kwargs in _cases():
+ ref = pg._build_group_layout_reference(*args, **kwargs)
+ got = pg.build_group_layout(*args, **kwargs)
+ _assert_same(ref, got)
+ built += got is not None
+ assert built >= 100
+
+
+def test_seg_info_matches_per_segment_build():
+ specs = [(5, [3, 0, 7]), (1, [1, 1]), (130, [2, 9, 0, 4])]
+ seg, meta = pgk.build_seg_info_multigroup(specs, torch.device("cpu"))
+ group_of, is_prefix, suffix_of = [], [], []
+ counter = 0
+ for gid, (P, Rs) in enumerate(specs):
+ group_of += [gid] * P
+ is_prefix += [True] * P
+ suffix_of += [-1] * P
+ for r in Rs:
+ group_of += [gid] * r
+ is_prefix += [False] * r
+ suffix_of += [counter] * r
+ counter += 1
+ T = len(group_of)
+ T_pad = pgk._pad_len(T)
+ pad = T_pad - T
+ assert (seg.T, seg.T_pad) == (T, T_pad)
+ assert torch.equal(seg.group_of_kv, torch.tensor(group_of + [pgk._PAD_GROUP] * pad))
+ assert torch.equal(seg.is_prefix, torch.tensor(is_prefix + [False] * pad))
+ assert torch.equal(seg.suffix_of_kv, torch.tensor(suffix_of + [pgk._PAD_GROUP] * pad))
+ assert seg.signature == ("multi", T_pad, tuple((P, tuple(Rs)) for P, Rs in specs))
+ assert meta[2]["prefix_last_index"] == meta[2]["base"] + 129
+ assert meta[0]["suffix_slices"] == [(5, 8), (8, 8), (8, 15)]
+
+
+class _CountFactories(torch.overrides.TorchFunctionMode):
+ """Counts tensor creations that name a device (each one a fill kernel or a host to
+ device copy when that device is an accelerator)."""
+
+ def __init__(self):
+ super().__init__()
+ self.calls = 0
+
+ def __torch_function__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ kwargs = kwargs or {}
+ if kwargs.get("device") is not None and func in (
+ torch.tensor,
+ torch.full,
+ torch.ones,
+ torch.zeros,
+ ):
+ self.calls += 1
+ return func(*args, **kwargs)
+
+
+def _factory_calls(num_groups, G):
+ rng = torch.Generator().manual_seed(0)
+ ids, lp = _batch(rng, num_groups, G, 64, 16)
+ with _CountFactories() as mode:
+ layout = pg.build_group_layout(ids, 16, 0, G, lp, apply_tokr_gate = False)
+ assert layout is not None
+ return mode.calls
+
+
+def test_device_tensor_creations_do_not_grow_with_groups():
+ # The per-segment build made three device tensors per prefix and per completion.
+ assert _factory_calls(1, 8) == _factory_calls(6, 8) == _factory_calls(6, 2)
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_cuda_layout_reads_the_device_once_and_copies_once():
+ import warnings
+
+ rng = torch.Generator().manual_seed(0)
+ ids, lp = _batch(rng, 4, 8, 256, 128)
+ ids, lp = ids.cuda(), lp.cuda()
+ ref = pg._build_group_layout_reference(ids, 128, 0, 8, lp)
+ got = pg.build_group_layout(ids, 128, 0, 8, lp)
+ _assert_same(ref, got)
+
+ def syncs(fn):
+ torch.cuda.synchronize()
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ torch.cuda.set_sync_debug_mode("warn")
+ try:
+ fn(ids, 128, 0, 8, lp)
+ finally:
+ torch.cuda.set_sync_debug_mode(0)
+ # Count the per-operation reports, not set_sync_debug_mode's "prototype feature" notice.
+ return [str(w.message) for w in caught if "called a synchronizing" in str(w.message)]
+
+ # The counter sees the token-by-token build's reads (cstart, ids, contiguity, ...).
+ assert len(syncs(pg._build_group_layout_reference)) > 2
+ # One device to host read of the ids + one host to device copy of the index tables.
+ got_syncs = syncs(pg.build_group_layout)
+ assert len(got_syncs) <= 2, got_syncs
+
+
+def _outcome(fn, *args, **kwargs):
+ try:
+ return ("ok", fn(*args, **kwargs))
+ except Exception as exc:
+ return ("err", type(exc), str(exc))
+
+
+def test_unusual_arguments_take_the_reference_decision():
+ # A float num_generations must raise the reference's error, a numpy one must keep a
+ # plain int signature, and a pad id outside the ids dtype must compare like the reference.
+ np = pytest.importorskip("numpy")
+ rng = torch.Generator().manual_seed(0)
+ ids, lp = _batch(rng, 2, 4, 32, 16)
+ cases = [
+ ((ids, 16, 0, 4.0, lp), {}),
+ ((ids, 16, 0, np.int64(4), lp), dict(apply_tokr_gate = False)),
+ ((ids.to(torch.int32), 16, 2**32, 4, lp), dict(apply_tokr_gate = False)),
+ ]
+ for args, kwargs in cases:
+ ref = _outcome(pg._build_group_layout_reference, *args, **kwargs)
+ got = _outcome(pg.build_group_layout, *args, **kwargs)
+ assert ref[0] == got[0], (ref, got)
+ if ref[0] == "err":
+ assert ref[1:] == got[1:]
+ else:
+ _assert_same(ref[1], got[1])
+ if ref[1] is not None:
+ assert type(got[1].signature[0]) is type(ref[1].signature[0])
+
+
+def test_host_tables_ignore_the_default_device():
+ rng = torch.Generator().manual_seed(0)
+ ids, lp = _batch(rng, 3, 4, 32, 16)
+ ref = pg._build_group_layout_reference(ids, 16, 0, 4, lp, apply_tokr_gate = False)
+ torch.set_default_device("meta")
+ try:
+ got = pg.build_group_layout(ids, 16, 0, 4, lp, apply_tokr_gate = False)
+ finally:
+ torch.set_default_device(None)
+ _assert_same(ref, got)
diff --git a/unsloth/utils/prefix_grouper.py b/unsloth/utils/prefix_grouper.py
index 744dc652d3e..1716784f2c7 100644
--- a/unsloth/utils/prefix_grouper.py
+++ b/unsloth/utils/prefix_grouper.py
@@ -225,7 +225,7 @@ def _tok_r(groups) -> float:
return (tok_full / tok_sp) if tok_sp else 1.0
-def build_group_layout(
+def _build_group_layout_reference(
input_ids,
logits_to_keep,
pad_id,
@@ -236,6 +236,7 @@ def build_group_layout(
max_segment_cap = None,
):
"""Build the shared-prefix GroupLayout, or return None to fall back to the packed path.
+ Token-by-token builder; build_group_layout's fallback for inputs its fast path skips.
input_ids : [B, L]. GRPO's layout is left-padded in the prompt and right-padded in
the completion. Real tokens of a row are a contiguous run not necessarily
@@ -358,6 +359,184 @@ def build_group_layout(
)
+_INDEX_DTYPES = (torch.int64, torch.int32)
+
+
+def _is_int(x) -> bool:
+ return isinstance(x, int) and not isinstance(x, bool)
+
+
+def build_group_layout(
+ input_ids,
+ logits_to_keep,
+ pad_id,
+ num_generations,
+ left_pad_tokens_per_prompt,
+ *,
+ apply_tokr_gate = True,
+ max_segment_cap = None,
+):
+ """Build the shared-prefix GroupLayout, or return None to fall back to the packed path.
+
+ input_ids : [B, L]. GRPO's layout is left-padded in the prompt and right-padded in
+ the completion. Real tokens of a row are a contiguous run not necessarily
+ starting at column 0.
+ logits_to_keep : int
+ left_pad_tokens_per_prompt : [B] long tensor (per-row left-pad count in the prompt).
+
+ Same result as _build_group_layout_reference, with one device->host and one
+ host->device copy instead of per-token loops.
+ """
+ args = (input_ids, logits_to_keep, pad_id, num_generations, left_pad_tokens_per_prompt)
+ kwargs = dict(apply_tokr_gate = apply_tokr_gate, max_segment_cap = max_segment_cap)
+ lp = left_pad_tokens_per_prompt
+ if not (
+ isinstance(input_ids, torch.Tensor)
+ and input_ids.dim() == 2
+ and input_ids.shape[0] > 0
+ and input_ids.dtype in _INDEX_DTYPES
+ and isinstance(lp, torch.Tensor)
+ and lp.dim() == 1
+ and lp.shape[0] == input_ids.shape[0]
+ and lp.dtype in _INDEX_DTYPES
+ and lp.device == input_ids.device
+ and _is_int(logits_to_keep)
+ and _is_int(pad_id)
+ and (num_generations is None or _is_int(num_generations))
+ and torch.iinfo(input_ids.dtype).min <= pad_id <= torch.iinfo(input_ids.dtype).max
+ ):
+ return _build_group_layout_reference(*args, **kwargs)
+ device = input_ids.device
+ total_rows, L = input_ids.shape
+ G = num_generations
+ # Completion start column per row, matching create_completion_attention_mask / _pk_cstart.
+ cstart = ((L - logits_to_keep) - lp).to(torch.long)
+ host = torch.cat(
+ [input_ids.to(torch.long), cstart.unsqueeze(1), lp.to(torch.long).unsqueeze(1)], dim = 1
+ ).cpu()
+ ids = host[:, :L]
+ cs = host[:, L]
+ keep = ids != pad_id
+ n_real = keep.sum(dim = 1)
+ first = torch.argmax(keep.to(torch.int8), dim = 1)
+ ar = torch.arange(L, device = "cpu")
+ end = first + n_real
+ if not bool((keep == ((ar >= first.unsqueeze(1)) & (ar < end.unsqueeze(1)))).all()):
+ return _build_group_layout_reference(*args, **kwargs)
+ if G is None or G < 2 or total_rows % G != 0:
+ return None
+ # Prompt real columns [first, pe), completion [pe, end).
+ pe = torch.minimum(torch.maximum(cs, first), end)
+ P_row = pe - first
+ R_row = end - pe
+ num_groups = total_rows // G
+ P_g = P_row.view(num_groups, G)
+ if bool((P_g == 0).any()) or bool((P_g != P_g[:, :1]).any()):
+ return None
+ R_g = R_row.view(num_groups, G)
+ if bool((R_g.sum(dim = 1) == 0).any()):
+ return None
+ first_g = first.view(num_groups, G)
+ P_list = P_g[:, 0].tolist()
+ for gi, P in enumerate(P_list):
+ cols = first_g[gi].unsqueeze(1) + torch.arange(P, device = "cpu")
+ prompts = ids[gi * G : (gi + 1) * G].gather(1, cols)
+ if not bool((prompts == prompts[:1]).all()):
+ return None
+ R_lists = R_g.tolist()
+ groups = [dict(P = P, R_list = R_list) for P, R_list in zip(P_list, R_lists)]
+
+ # Sliding-window guard: a group's PG span is P + max(R), so fall back if it exceeds the window.
+ if max_segment_cap is not None:
+ for gm in groups:
+ if gm["P"] + max(gm["R_list"]) > max_segment_cap:
+ return None
+
+ tok_r = _tok_r(groups)
+ if apply_tokr_gate and tok_r < tokr_threshold():
+ return None # low reuse -> not worth it; use the full-row packed path
+
+ from .prefix_grouper_kernel import _labels_to_device, _seg_labels_host
+
+ group_specs = [(gm["P"], gm["R_list"]) for gm in groups]
+ labels, T, T_pad, _, sig_parts = _seg_labels_host(group_specs)
+
+ # Flat order per group: shared prefix (from its first row), then one suffix per row.
+ rows_g = torch.arange(total_rows, device = "cpu").view(num_groups, G)
+ P_t = P_g[:, :1]
+ seg_len = torch.cat([P_t, R_g], dim = 1).reshape(-1)
+ seg_row = torch.cat([rows_g[:, :1], rows_g], dim = 1).reshape(-1)
+ seg_col = torch.cat([first_g[:, :1], pe.view(num_groups, G)], dim = 1).reshape(-1)
+ seg_pos = torch.cat([torch.zeros_like(P_t), P_t.expand(num_groups, G)], dim = 1).reshape(-1)
+ seg_is_suffix = torch.ones((num_groups, G + 1), dtype = torch.bool, device = "cpu")
+ seg_is_suffix[:, 0] = False
+ seg_is_suffix = seg_is_suffix.reshape(-1)
+ seg_start = torch.cumsum(seg_len, dim = 0) - seg_len
+ group_base = seg_start.view(num_groups, G + 1)[:, :1]
+ seg_plast = (group_base + P_t - 1).expand(num_groups, G + 1).reshape(-1)
+
+ flat = torch.arange(T, device = "cpu")
+ j = flat - torch.repeat_interleave(seg_start, seg_len, output_size = T)
+ flat_src_rows = torch.repeat_interleave(seg_row, seg_len, output_size = T)
+ flat_src_cols = torch.repeat_interleave(seg_col, seg_len, output_size = T) + j
+ pos = torch.repeat_interleave(seg_pos, seg_len, output_size = T) + j
+ is_tgt = torch.repeat_interleave(seg_is_suffix, seg_len, output_size = T)
+ tgt_flat = flat[is_tgt]
+ tgt_j = j[is_tgt]
+ # Position 0 is predicted from the prefix's last token; j >= 1 from the previous suffix token.
+ tgt_pred = torch.where(
+ tgt_j == 0,
+ torch.repeat_interleave(seg_plast, seg_len, output_size = T)[is_tgt],
+ tgt_flat - 1,
+ )
+ N = int(tgt_flat.numel())
+ table = torch.cat(
+ [
+ flat_src_rows,
+ flat_src_cols,
+ pos,
+ flat_src_rows[is_tgt],
+ flat_src_cols[is_tgt],
+ tgt_pred,
+ tgt_flat,
+ labels.reshape(-1),
+ ]
+ ).to(device)
+ pieces = torch.split(table, [T, T, T, N, N, N, N, 3 * T_pad])
+ fr, fc, position_ids, tgt_rows, tgt_cols, tgt_pred_d, tgt_flat_d, labels_d = pieces
+ group_of_kv, is_prefix, suffix_of_kv = _labels_to_device(labels_d.view(3, T_pad), T_pad, device)
+ seg = PrefixSegInfo(
+ group_of_kv = group_of_kv,
+ is_prefix = is_prefix,
+ suffix_of_kv = suffix_of_kv,
+ signature = ("multi", T_pad, tuple(sig_parts)),
+ T = T,
+ T_pad = T_pad,
+ )
+ flat_ids = input_ids[fr, fc].unsqueeze(0) # [1, T] (grad-safe gather)
+
+ max_left_pad = int(host[:, L + 1].max().item())
+ W = logits_to_keep + max_left_pad
+
+ # The logic is structural, so key on (num_groups, group_sizes), not exact lengths.
+ sig = (num_groups, tuple(sorted(len(gm["R_list"]) for gm in groups)))
+
+ return GroupLayout(
+ flat_ids = flat_ids,
+ position_ids = position_ids.clone().unsqueeze(0),
+ prefix_seg_info = seg,
+ tgt_rows = tgt_rows.clone(),
+ tgt_cols = tgt_cols.clone(),
+ tgt_pred = tgt_pred_d.clone(),
+ tgt_flat = tgt_flat_d.clone(),
+ total_rows = total_rows,
+ L = L,
+ W = W,
+ tok_r = tok_r,
+ signature = sig,
+ )
+
+
__all__ = [
"GroupLayout",
"build_group_layout",
diff --git a/unsloth/utils/prefix_grouper_kernel.py b/unsloth/utils/prefix_grouper_kernel.py
index ae5a873d4d8..e76560d3767 100644
--- a/unsloth/utils/prefix_grouper_kernel.py
+++ b/unsloth/utils/prefix_grouper_kernel.py
@@ -181,26 +181,45 @@ def build_seg_info_multigroup(
group_meta : list of dicts with 'base', 'P', 'prefix_last_index', 'suffix_slices'
(flat offsets), enough to build the completion index map.
"""
- group_of_list = []
- is_prefix_list = []
- suffix_of_list = []
- group_meta = []
+ labels, T, T_pad, group_meta, sig_parts = _seg_labels_host(group_specs)
+ group_of_kv, is_prefix, suffix_of_kv = _labels_to_device(labels, T_pad, device)
+ sig = ("multi", T_pad, tuple(sig_parts))
+ seg = PrefixSegInfo(
+ group_of_kv = group_of_kv,
+ is_prefix = is_prefix,
+ suffix_of_kv = suffix_of_kv,
+ signature = sig,
+ T = T,
+ T_pad = T_pad,
+ )
+ return seg, group_meta
+
+def _seg_labels_host(group_specs):
+ """CPU int64 [3, T_pad] rows (group_of_kv, is_prefix, suffix_of_kv), pad sentinels
+ included, plus T, T_pad, group_meta and signature parts."""
+ group_meta = []
+ sig_parts = []
+ seg_lens = []
+ seg_group = []
+ seg_prefix = []
+ seg_suffix = []
base = 0
suffix_counter = 0
- sig_parts = []
+ if not group_specs:
+ torch.cat([]) # the empty-list error the per-segment build raised
for gid, (P, R_list) in enumerate(group_specs):
- # Prefix.
- group_of_list.append(torch.full((P,), gid, dtype = torch.long, device = device))
- is_prefix_list.append(torch.ones(P, dtype = torch.bool, device = device))
- suffix_of_list.append(torch.full((P,), -1, dtype = torch.long, device = device))
- prefix_last_index = base + P - 1
+ seg_lens.append(P)
+ seg_group.append(gid)
+ seg_prefix.append(1)
+ seg_suffix.append(-1)
suffix_slices = []
cursor = base + P
for r in R_list:
- group_of_list.append(torch.full((r,), gid, dtype = torch.long, device = device))
- is_prefix_list.append(torch.zeros(r, dtype = torch.bool, device = device))
- suffix_of_list.append(torch.full((r,), suffix_counter, dtype = torch.long, device = device))
+ seg_lens.append(r)
+ seg_group.append(gid)
+ seg_prefix.append(0)
+ seg_suffix.append(suffix_counter)
suffix_slices.append((cursor, cursor + r))
cursor += r
suffix_counter += 1
@@ -208,29 +227,28 @@ def build_seg_info_multigroup(
{
"base": base,
"P": P,
- "prefix_last_index": prefix_last_index,
+ "prefix_last_index": base + P - 1,
"suffix_slices": suffix_slices,
}
)
sig_parts.append((P, tuple(R_list)))
base = cursor
-
- group_of_kv = torch.cat(group_of_list)
- is_prefix = torch.cat(is_prefix_list)
- suffix_of_kv = torch.cat(suffix_of_list)
- group_of_kv, is_prefix, suffix_of_kv, T, T_pad = _pad_labels(
- group_of_kv, is_prefix, suffix_of_kv, device
- )
- sig = ("multi", T_pad, tuple(sig_parts))
- seg = PrefixSegInfo(
- group_of_kv = group_of_kv,
- is_prefix = is_prefix,
- suffix_of_kv = suffix_of_kv,
- signature = sig,
- T = T,
- T_pad = T_pad,
- )
- return seg, group_meta
+ T = base
+ T_pad = _pad_len(T)
+ labels = torch.empty((3, T_pad), dtype = torch.long, device = "cpu")
+ lens = torch.tensor(seg_lens, dtype = torch.long, device = "cpu")
+ per_seg = torch.tensor([seg_group, seg_prefix, seg_suffix], dtype = torch.long, device = "cpu")
+ labels[:, :T] = torch.repeat_interleave(per_seg, lens, dim = 1, output_size = T)
+ labels[0, T:] = _PAD_GROUP
+ labels[1, T:] = 0
+ labels[2, T:] = _PAD_GROUP
+ return labels, T, T_pad, group_meta, sig_parts
+
+
+def _labels_to_device(labels, T_pad, device):
+ """One host->device copy; each returned tensor gets its own storage."""
+ moved = labels.to(device)
+ return moved[0].clone(), moved[1].to(torch.bool), moved[2].clone()
# Block-mask cache, keyed on (signature, device): the mask depends only on per-token labels and T.