Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Host-built packed metadata must match the old device reads bit for bit, without per-step syncs."""
import math
import warnings
import pytest
import torch
import unsloth.utils.packing as packing
from real_accelerator import has_real_cuda
def _old_packed_info(seq_lengths, device):
lengths = seq_lengths.to(device = device, dtype = torch.int32)
cu_seqlens = torch.zeros(lengths.numel() + 1, dtype = torch.int32, device = device)
torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])
return lengths, cu_seqlens, int(lengths.max().item())
def _old_cover(seq_info, total_tokens):
_, cu_seqlens, max_seqlen = seq_info
padding = total_tokens - int(cu_seqlens[-1].item())
if padding > 0:
tail = torch.tensor([total_tokens], dtype = cu_seqlens.dtype, device = cu_seqlens.device)
return torch.cat([cu_seqlens, tail]), max(max_seqlen, padding)
return cu_seqlens, max_seqlen
def _random_batches(n = 200, seed = 0):
g = torch.Generator().manual_seed(seed)
for i in range(n):
k = int(torch.randint(1, 33, (1,), generator = g))
high = [2, 9, 300, 5000][i % 4]
dtype = [torch.int32, torch.int64][i % 2]
yield torch.randint(1, high, (k,), generator = g, dtype = dtype)
def _same(a, b):
assert a.dtype == b.dtype and a.shape == b.shape and a.device == b.device
assert torch.equal(a, b)
@pytest.fixture(autouse = True)
def _clear():
packing.clear_packed_caches()
yield
packing.clear_packed_caches()
def test_stage_packed_metadata_matches_device_cumsum():
for seq in _random_batches():
staged, host = packing._stage_packed_metadata(seq)
lengths, cu, max_seqlen = _old_packed_info(seq, "cpu")
n = seq.numel()
assert staged.dtype == torch.int32 and staged.numel() == 2 * n + 1
_same(staged[:n], lengths)
_same(staged[n:], cu)
assert host == tuple(lengths.tolist()) and max(host) == max_seqlen
assert all(type(x) is int for x in host)
def test_stage_packed_metadata_stays_on_host_under_a_default_device():
# torch.set_default_device("cuda") must not move the host staging buffer (it is pinned next).
seq = torch.tensor([3, 1, 4], dtype = torch.int32)
torch.set_default_device("meta")
try:
staged, host = packing._stage_packed_metadata(seq)
finally:
torch.set_default_device(None)
assert staged.device.type == "cpu" and host == (3, 1, 4)
assert staged.tolist() == [3, 1, 4, 0, 3, 4, 8]
def test_packed_info_and_consumers_match_old_values():
cpu = torch.device("cpu")
for i, seq in enumerate(_random_batches()):
packing.clear_packed_caches()
lengths, cu, max_seqlen = packing.get_packed_info_from_kwargs(
{"packed_seq_lengths": seq}, cpu
)
ref = _old_packed_info(seq, cpu)
_same(lengths, ref[0])
_same(cu, ref[1])
assert max_seqlen == ref[2] and type(max_seqlen) is int
info = (lengths, cu, max_seqlen)
total = int(seq.sum()) + [0, 1, 7, 64][i % 4]
new_cu, new_max = packing.cover_padded_cu_seqlens(info, total)
old_cu, old_max = _old_cover(ref, total)
_same(new_cu, old_cu)
assert new_max == old_max
segs = packing.packed_segment_lengths(info, total)
assert segs == packing._with_padding_segment(tuple(int(x) for x in ref[0].tolist()), total)
if seq.numel() <= 8 and int(seq.sum()) <= 64:
mask = packing.build_sdpa_packed_attention_mask(
info, dtype = torch.float32, device = cpu, total_tokens = total
)
packing.clear_packed_caches()
# a seq_info not built by get_packed_info_from_kwargs takes the old .tolist() route
old_mask = packing.build_sdpa_packed_attention_mask(
(ref[0].clone(), ref[1].clone(), ref[2]),
dtype = torch.float32,
device = cpu,
total_tokens = total,
)
_same(mask, old_mask)
def test_host_lengths_only_reused_for_the_cached_tensors():
cpu = torch.device("cpu")
seq = torch.tensor([3, 5, 2], dtype = torch.int32)
lengths, cu, _ = packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, cpu)
assert packing._cached_host_lengths(lengths) == (3, 5, 2)
assert packing._cached_host_lengths(lengths, cu) == (3, 5, 2)
assert packing._cached_host_lengths(lengths.clone()) is None
assert packing._cached_host_lengths(lengths, cu.clone()) is None
# a foreign seq_info whose cu_seqlens is not the cumsum of these lengths keeps reading cu_seqlens[-1]
foreign_cu = torch.tensor([0, 4, 6, 9], dtype = torch.int32)
new_cu, new_max = packing.cover_padded_cu_seqlens((lengths, foreign_cu, 5), 12)
assert new_cu.tolist() == [0, 4, 6, 9, 12] and new_max == 5
packing.clear_packed_caches()
assert packing._cached_host_lengths(lengths) is None
def test_position_id_inferred_lengths_match():
cpu = torch.device("cpu")
pos = torch.cat([torch.arange(n) for n in (4, 1, 9, 3)])[None]
lengths, cu, max_seqlen = packing.get_packed_info_from_kwargs({}, cpu, pos)
assert lengths.tolist() == [4, 1, 9, 3] and cu.tolist() == [0, 4, 5, 14, 17] and max_seqlen == 9
def test_fsdp2_clip_empty_shards_match_reference():
from unsloth.models.fsdp2_native import native_clip_grad_norm_
g = torch.Generator().manual_seed(0)
for p_norm in (2.0, 1.0, math.inf, -math.inf, 3.0):
for dtype in (torch.float32, torch.float16, torch.bfloat16):
shapes = [(3, 4), (0,), (5,), (0, 4), (7, 2), (0,)]
params = []
for s in shapes:
p = torch.nn.Parameter(torch.zeros(s, dtype = dtype))
p.grad = (torch.randn(s, generator = g) * 3).to(dtype)
params.append(p)
# the previous implementation: identity-filled vector, live norms scattered in by index
grads = [p.grad for p in params]
live = [k for k, t in enumerate(grads) if t.numel()]
values = torch.full((len(grads),), math.inf if p_norm < 0 else 0.0, dtype = dtype)
values[torch.tensor(live)] = torch.stack(
list(torch._foreach_norm([grads[k] for k in live], p_norm))
)
expect = torch.linalg.vector_norm(torch.cat([values]), p_norm)
got = native_clip_grad_norm_([p for p in params], max_norm = 1e9, norm_type = p_norm)
assert torch.equal(got.to(expect.dtype), expect), (p_norm, dtype, got, expect)
def test_fused_ce_int_divisor_is_bitwise_torch_tensor():
for v in (0, 1, 7, 2**24 + 1, 2**31 + 12345, 2**40 + 999, 3.5, True):
a = torch.tensor(v, dtype = torch.float32)
b = torch.full((), v, dtype = torch.float32)
assert a.shape == b.shape and torch.equal(a.view(torch.int32), b.view(torch.int32))
_needs_cuda = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
def _count_syncs(fn):
torch.cuda.synchronize()
with warnings.catch_warnings(record = True) as w:
warnings.simplefilter("always")
torch.cuda.set_sync_debug_mode("warn")
try:
out = fn()
finally:
torch.cuda.set_sync_debug_mode("default")
return out, sum("synchronizing CUDA operation" in str(x.message) for x in w)
@_needs_cuda
def test_cuda_host_lengths_zero_syncs_and_one_read_for_device_lengths():
dev = torch.device("cuda")
seq = torch.tensor([5, 1, 70, 7], dtype = torch.int32)
def per_step(s):
info = packing.get_packed_info_from_kwargs({"packed_seq_lengths": s}, dev)
packing.cover_padded_cu_seqlens(info, 96)
packing.packed_segment_lengths(info, 96)
packing.build_sdpa_packed_attention_mask(
info, dtype = torch.float16, device = dev, total_tokens = 96
)
for _ in range(4): # later layers hit the caches
packing.get_packed_info_from_kwargs({"packed_seq_lengths": s}, dev)
packing.cover_padded_cu_seqlens(info, 96)
return info
info, n = _count_syncs(lambda: per_step(seq))
assert n == 0, f"{n} syncs for host lengths"
ref = _old_packed_info(seq, dev)
_same(info[0], ref[0])
_same(info[1], ref[1])
assert info[2] == ref[2]
packing.clear_packed_caches()
dev_seq = seq.to(dev)
_, n = _count_syncs(lambda: per_step(dev_seq))
assert n == 1, f"{n} syncs for device lengths (expected the one lengths read)"
@_needs_cuda
def test_cuda_fsdp2_clip_and_ce_divisor_no_sync():
from unsloth.models.fsdp2_native import native_clip_grad_norm_
params = []
for s in [(3, 4), (0,), (5,)]:
p = torch.nn.Parameter(torch.zeros(s, device = "cuda"))
p.grad = torch.randn(s, device = "cuda")
params.append(p)
_, n = _count_syncs(lambda: native_clip_grad_norm_(params, max_norm = 1.0))
assert n == 0, f"{n} syncs in native_clip_grad_norm_"
_, n = _count_syncs(lambda: torch.full((), 1234, dtype = torch.float32, device = "cuda"))
assert n == 0
def test_non_blocking_batches_only_for_pinned_cuda_loaders(monkeypatch):
from types import SimpleNamespace
from unsloth.models._utils import _unsloth_non_blocking_batches
def trainer(
pin = True,
device = "cuda",
non_blocking = False,
):
config = SimpleNamespace(non_blocking = non_blocking)
return SimpleNamespace(
args = SimpleNamespace(dataloader_pin_memory = pin),
accelerator = SimpleNamespace(dataloader_config = config, device = torch.device(device)),
), config
t, config = trainer()
assert _unsloth_non_blocking_batches(t) is True and config.non_blocking is True
for kwargs in ({"pin": False}, {"device": "cpu"}, {"non_blocking": True}):
t, config = trainer(**kwargs)
before = config.non_blocking
assert _unsloth_non_blocking_batches(t) is False and config.non_blocking == before
monkeypatch.setenv("UNSLOTH_NON_BLOCKING_BATCHES", "0")
t, config = trainer()
assert _unsloth_non_blocking_batches(t) is False and config.non_blocking is False
assert _unsloth_non_blocking_batches(SimpleNamespace()) is False
def _compile_over_segment_counts(dynamic):
import torch._dynamo as dynamo
from torch._dynamo.utils import counters
def step(x, lengths):
packing.clear_packed_caches()
_, cu_seqlens, max_seqlen = packing.get_packed_info_from_kwargs(
{"packed_seq_lengths": lengths}, x.device
)
return x * max_seqlen + cu_seqlens[-1]
dynamo.reset()
counters.clear()
compiled = torch.compile(step, backend = "eager", dynamic = dynamic)
for n in range(3, 15):
lengths = torch.full((n,), 4, dtype = torch.int32)
out = compiled(torch.ones(4), lengths)
assert torch.equal(out, torch.ones(4) * 4 + 4 * n)
graphs = counters["stats"]["unique_graphs"]
dynamo.reset()
return graphs
def test_compiled_dynamic_does_not_specialize_on_segment_count():
# 12 segment counts: the tensor path traces a handful of graphs; a host tuple of lengths
# would trace one per count and hit the recompile limit.
assert _compile_over_segment_counts(dynamic = True) <= 4
def test_compiling_takes_tensor_path(monkeypatch):
monkeypatch.setattr(packing, "_is_compiling", lambda: True)
seq = torch.tensor([3, 5, 2], dtype = torch.int64)
lengths, cu, max_seqlen = packing.get_packed_info_from_kwargs(
{"packed_seq_lengths": seq}, "cpu"
)
old = _old_packed_info(seq, "cpu")
_same(lengths, old[0])
_same(cu, old[1])
assert max_seqlen == old[2]
# No host tuple recorded: downstream helpers take their own (traceable) route.
assert packing._cached_host_lengths(lengths) is None
def test_pinned_to_device_falls_back_to_pageable(monkeypatch):
t = torch.arange(5, dtype = torch.int32)
def refuse(self, *args, **kwargs):
raise RuntimeError("cudaHostAlloc failed: out of pinned memory")
monkeypatch.setattr(torch.Tensor, "pin_memory", refuse)
_same(packing._pinned_to_device(t, "cpu"), t)
class _SaveForBackward(torch.autograd.Function):
@staticmethod
def forward(ctx, x, saved):
ctx.save_for_backward(saved)
return x * 2
@staticmethod
def backward(ctx, grad):
return grad * 2, None
def _survives_backward(saved):
x = torch.randn(4, requires_grad = True)
_SaveForBackward.apply(x, saved).sum().backward()
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64])
def test_inference_mode_metadata_is_not_served_to_a_training_step(dtype):
"""inference_mode forward then a training forward on the same batch: no inference tensors."""
cpu = torch.device("cpu")
kwargs = {"packed_seq_lengths": torch.tensor([3, 5], dtype = dtype)}
with torch.inference_mode():
info = packing.get_packed_info_from_kwargs(kwargs, cpu)
padded = packing.cover_padded_cu_seqlens(info, 10)
mask = packing.build_sdpa_packed_attention_mask(info, dtype = torch.float32, device = cpu)
assert torch.is_inference(info[1])
train = packing.get_packed_info_from_kwargs(kwargs, cpu)
assert train is not info and not torch.is_inference(train[1])
assert train[1].tolist() == [0, 3, 8] and train[2] == 5
_survives_backward(train[1])
train_padded = packing.cover_padded_cu_seqlens(train, 10)
assert not torch.is_inference(train_padded[0]) and train_padded[0].tolist() == [0, 3, 8, 10]
_survives_backward(train_padded[0])
train_mask = packing.build_sdpa_packed_attention_mask(train, dtype = torch.float32, device = cpu)
assert not torch.is_inference(train_mask)
torch.testing.assert_close(train_mask, mask.clone())
_survives_backward(train_mask)
assert packing.get_packed_info_from_kwargs(kwargs, cpu) is train
assert packing.cover_padded_cu_seqlens(train, 10) is train_padded
assert (
packing.build_sdpa_packed_attention_mask(train, dtype = torch.float32, device = cpu)
is train_mask
)
@pytest.mark.parametrize("grad_mode", ["inference_mode", "no_grad"])
def test_compiled_inference_mode_metadata_is_not_served_to_a_training_step(grad_mode):
from unsloth.utils import attention_dispatch
from unsloth.utils.attention_dispatch import _windowed_causal_mask
packing._PACKED_INFO_CACHE.clear()
attention_dispatch._WINDOW_MASK_CACHE.clear()
cpu = torch.device("cpu")
seq = torch.tensor([3, 5], dtype = torch.int32)
def build(seq):
info = packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, cpu)
return info, _windowed_causal_mask(8, 8, 2, cpu)
torch._dynamo.reset()
compiled = torch.compile(build, backend = "eager", fullgraph = True, dynamic = True)
context = torch.inference_mode() if grad_mode == "inference_mode" else torch.no_grad()
with torch._dynamo.config.patch(capture_scalar_outputs = True), context:
built, built_mask = compiled(seq)
assert torch.is_inference(built[1]) == (grad_mode == "inference_mode")
train, train_mask = build(seq)
assert not torch.is_inference(train[1]) and not torch.is_inference(train_mask)
assert train[1].tolist() == [0, 3, 8] and train[2] == 5
_survives_backward(train[1])
_survives_backward(train_mask.float())
assert (train is built) == (grad_mode == "no_grad")
assert (train_mask is built_mask) == (grad_mode == "no_grad")
@pytest.mark.parametrize("backend", ["eager", "aot_eager", "inductor"])
@pytest.mark.parametrize("grad_mode", ["inference_mode", "no_grad"])
@pytest.mark.parametrize("kind", ["metadata", "window_mask"])
def test_compiled_training_rejects_inference_mode_cache(backend, grad_mode, kind):
from unsloth.utils import attention_dispatch
cpu = torch.device("cpu")
seq = torch.tensor([3, 5], dtype = torch.int32)
attention_dispatch._WINDOW_MASK_CACHE.clear()
def build(seq):
if kind == "metadata":
return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, cpu)[1]
return attention_dispatch._windowed_causal_mask(8, 8, 2, cpu)
def train(x, seq):
saved = build(seq)
# Multiply by the original int/bool tensor: casting would hide the inference-tensor bug.
return (x * saved).sum(), saved
torch._dynamo.reset()
try:
compiled_build = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)
compiled_train = torch.compile(train, backend = backend, fullgraph = True, dynamic = True)
with torch._dynamo.config.patch(capture_scalar_outputs = True):
with getattr(torch, grad_mode)():
built = compiled_build(seq)
assert torch.is_inference(built) == (grad_mode == "inference_mode")
expected = built.clone()
if kind == "metadata":
assert expected.tolist() == [0, 3, 8]
for _ in range(3):
x = torch.ones(built.shape, requires_grad = True)
loss, saved = compiled_train(x, seq)
loss.backward()
assert not torch.is_inference(saved)
torch.testing.assert_close(saved, expected)
torch.testing.assert_close(x.grad, expected.float())
assert build(seq) is saved
finally:
attention_dispatch._WINDOW_MASK_CACHE.clear()
torch._dynamo.reset()
@pytest.mark.parametrize("backend", ["eager", "aot_eager", "inductor"])
@pytest.mark.parametrize("mode", [torch.no_grad, torch.inference_mode])
@pytest.mark.parametrize("warm_inside", [False, True])
@pytest.mark.parametrize("total", [8, 10])
def test_compiled_mode_transition_preserves_padded_metadata(backend, mode, warm_inside, total):
packing.clear_packed_caches()
torch._dynamo.reset()
device = torch.device("cpu")
seq = torch.tensor([3, 5], dtype = torch.int32)
def build(seq):
return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)
def train(x, seq):
padded, _ = packing.cover_padded_cu_seqlens(build(seq), total)
return (x * padded).sum()
try:
with torch._dynamo.config.patch(capture_scalar_outputs = True):
with mode():
info = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)(seq)
if warm_inside:
padded, _ = packing.cover_padded_cu_seqlens(info, total)
if not warm_inside:
padded, _ = packing.cover_padded_cu_seqlens(info, total)
expected = torch.tensor([0.0, 3.0, 8.0] + ([10.0] if total == 10 else []))
x = torch.ones(expected.shape, requires_grad = True)
torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(x, seq).backward()
torch.testing.assert_close(x.grad, expected)
finally:
packing.clear_packed_caches()
torch._dynamo.reset()
@pytest.mark.parametrize("backend", ["eager", "aot_eager", "inductor"])
def test_compiled_cache_checks_replaced_tensor_provenance(backend):
device = torch.device("cpu")
seq = torch.tensor([3, 5], dtype = torch.int32)
def build(seq):
return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)[1]
def train(x, seq):
return (x * build(seq)).sum()
torch._dynamo.reset()
try:
producer = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)
consumer = torch.compile(train, backend = backend, fullgraph = True, dynamic = True)
with torch._dynamo.config.patch(capture_scalar_outputs = True):
for mode in (torch.no_grad, torch.inference_mode, torch.no_grad, torch.inference_mode):
packing.clear_packed_caches()
with mode():
producer(seq)
x = torch.ones(3, requires_grad = True)
consumer(x, seq).backward()
torch.testing.assert_close(x.grad, torch.tensor([0.0, 3.0, 8.0]))
finally:
torch._dynamo.reset()
@pytest.mark.parametrize("backend", ["eager", "aot_eager", "inductor"])
@pytest.mark.parametrize("mode", [torch.no_grad, torch.inference_mode])
def test_compiled_cache_created_and_consumed_in_one_graph(backend, mode):
device = torch.device("cpu")
def train(x, seq):
with mode():
packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)
saved = packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)[1]
return (x * saved).sum()
torch._dynamo.reset()
try:
x = torch.ones(3, requires_grad = True)
seq = torch.tensor([3, 5], dtype = torch.int32)
with torch._dynamo.config.patch(capture_scalar_outputs = True):
torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(x, seq).backward()
torch.testing.assert_close(x.grad, torch.tensor([0.0, 3.0, 8.0]))
finally:
torch._dynamo.reset()
@pytest.mark.parametrize("backend", ["eager", "aot_eager", "inductor"])
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64])
@pytest.mark.parametrize("mode", [torch.no_grad, torch.inference_mode])
@pytest.mark.parametrize("warm_inside", [False, True])
@pytest.mark.parametrize("kind", ["mask", "segments"])
def test_compiled_mode_transition_preserves_derived_cache(backend, dtype, mode, warm_inside, kind):
device = torch.device("cpu")
seq = torch.tensor([3, 5], dtype = dtype)
def build(seq):
return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)
def consume(info):
if kind == "mask":
mask = packing.build_sdpa_packed_attention_mask(
info, dtype = torch.float32, device = device, total_tokens = 10
)
return mask.isfinite().float(), mask
coeff = torch.tensor(packing.packed_segment_lengths(info, 10), dtype = torch.float32)
return coeff, coeff
def train(x, seq):
coeff, saved = consume(build(seq))
return (x * coeff).sum(), saved
torch._dynamo.reset()
try:
with torch._dynamo.config.patch(capture_scalar_outputs = True):
with mode():
info = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)(seq)
if warm_inside:
expected = consume(info)[0]
if not warm_inside:
expected = consume(info)[0]
expected = expected.clone()
x = torch.ones(expected.shape, requires_grad = True)
loss, saved = torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(
x, seq
)
loss.backward()
assert not torch.is_inference(saved)
torch.testing.assert_close(x.grad, expected)
finally:
torch._dynamo.reset()
@_needs_cuda
@pytest.mark.parametrize("backend", ["eager", "aot_eager"])
def test_compiled_inference_metadata_preserves_cuda_device_alias(backend):
device = torch.device("cuda")
seq = torch.tensor([3, 5], dtype = torch.int32, device = "cpu")
def build(seq):
return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)
def train(x, seq):
padded, _ = packing.cover_padded_cu_seqlens(build(seq), 10)
return (x * padded).sum()
torch._dynamo.reset()
try:
with torch._dynamo.config.patch(capture_scalar_outputs = True):
with torch.inference_mode():
info = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)(seq)
packing.cover_padded_cu_seqlens(info, 10)
x = torch.ones(4, device = info[1].device, requires_grad = True)
torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(x, seq).backward()
torch.testing.assert_close(x.grad, x.new_tensor([0.0, 3.0, 8.0, 10.0]))
finally:
torch._dynamo.reset()
def test_new_lengths_do_not_inherit_derived_caches():
device = torch.device("cpu")
with torch.inference_mode():
old = packing.get_packed_info_from_kwargs(
{"packed_seq_lengths": torch.tensor([3, 5], dtype = torch.int64)}, device
)
packing.cover_padded_cu_seqlens(old, 10)
packing.build_sdpa_packed_attention_mask(old, dtype = torch.float32, device = device)
packing.packed_segment_lengths(old, 10)
new = packing.get_packed_info_from_kwargs(
{"packed_seq_lengths": torch.tensor([2, 6], dtype = torch.int64)}, device
)
assert packing.cover_padded_cu_seqlens(new, 10)[0].tolist() == [0, 2, 8, 10]
assert packing.packed_segment_lengths(new, 10) == (2, 6, 2)
mask = packing.build_sdpa_packed_attention_mask(new, dtype = torch.float32, device = device)
assert mask.isfinite().sum().item() == 24
def test_xformers_mask_lru_keeps_inference_masks_out_of_training(monkeypatch):
class _Mask:
@classmethod
def from_seqlens(cls, lengths):
return cls()
monkeypatch.setattr(packing, "_XFormersBlockMask", _Mask)
monkeypatch.setattr(packing, "move_xformers_attention_bias", lambda mask, device: mask)
packing._XFORMERS_MASK_CACHE.clear()
cpu = torch.device("cpu")
with torch.inference_mode():
evaluated = packing._get_cached_block_mask((3, 5), None, cpu)
assert packing._get_cached_block_mask((3, 5), None, cpu) is evaluated
trained = packing._get_cached_block_mask((3, 5), None, cpu)
assert trained is not evaluated
assert packing._get_cached_block_mask((3, 5), None, cpu) is trained
# A training-built mask serves inference_mode too: no second entry per lengths tuple.
packing._XFORMERS_MASK_CACHE.clear()
trained = packing._get_cached_block_mask((2, 2), None, cpu)
with torch.inference_mode():
assert packing._get_cached_block_mask((2, 2), None, cpu) is trained
assert len(packing._XFORMERS_MASK_CACHE) == 1
packing._XFORMERS_MASK_CACHE.clear()
def test_training_metadata_is_reused_under_inference_mode():
cpu = torch.device("cpu")
kwargs = {"packed_seq_lengths": torch.tensor([3, 5], dtype = torch.int32)}
train = packing.get_packed_info_from_kwargs(kwargs, cpu)
with torch.inference_mode():
assert packing.get_packed_info_from_kwargs(kwargs, cpu) is train
other = {"packed_seq_lengths": torch.tensor([2, 2], dtype = torch.int32)}
info = packing.get_packed_info_from_kwargs(other, cpu)
assert packing.get_packed_info_from_kwargs(other, cpu) is info
assert packing.get_packed_info_from_kwargs(kwargs, cpu) is not train
_survives_backward(packing.get_packed_info_from_kwargs(kwargs, cpu)[1])