Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Packing: do not serve compiled inference_mode metadata to a training step (#16183)
Daniel Hana32528bb8a80
diff --git a/tests/utils/test_packing_host_sync.py b/tests/utils/test_packing_host_sync.pyindex c4bea33e376..b6b0c789e6a 100644--- a/tests/utils/test_packing_host_sync.py+++ b/tests/utils/test_packing_host_sync.py@@ -10,6 +10,7 @@ import pytestimport torchimport unsloth.utils.packing as packing+from real_accelerator import has_real_cudadef _old_packed_info(seq_lengths, device):@@ -160,7 +161,7 @@ def test_fused_ce_int_divisor_is_bitwise_torch_tensor():assert a.shape == b.shape and torch.equal(a.view(torch.int32), b.view(torch.int32))-_needs_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")+_needs_cuda = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")def _count_syncs(fn):@@ -345,6 +346,252 @@ def test_inference_mode_metadata_is_not_served_to_a_training_step(dtype):)[email protected]("grad_mode", ["inference_mode", "no_grad"])+def test_compiled_inference_mode_metadata_is_not_served_to_a_training_step(grad_mode):+ from unsloth.utils import attention_dispatch+ from unsloth.utils.attention_dispatch import _windowed_causal_mask++ packing._PACKED_INFO_CACHE.clear()+ attention_dispatch._WINDOW_MASK_CACHE.clear()+ cpu = torch.device("cpu")+ seq = torch.tensor([3, 5], dtype = torch.int32)++ def build(seq):+ info = packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, cpu)+ return info, _windowed_causal_mask(8, 8, 2, cpu)++ torch._dynamo.reset()+ compiled = torch.compile(build, backend = "eager", fullgraph = True, dynamic = True)+ context = torch.inference_mode() if grad_mode == "inference_mode" else torch.no_grad()+ with torch._dynamo.config.patch(capture_scalar_outputs = True), context:+ built, built_mask = compiled(seq)+ assert torch.is_inference(built[1]) == (grad_mode == "inference_mode")+ train, train_mask = build(seq)+ assert not torch.is_inference(train[1]) and not torch.is_inference(train_mask)+ assert train[1].tolist() == [0, 3, 8] and train[2] == 5+ _survives_backward(train[1])+ _survives_backward(train_mask.float())+ assert (train is built) == (grad_mode == "no_grad")+ assert (train_mask is built_mask) == (grad_mode == "no_grad")++[email protected]("backend", ["eager", "aot_eager", "inductor"])[email protected]("grad_mode", ["inference_mode", "no_grad"])[email protected]("kind", ["metadata", "window_mask"])+def test_compiled_training_rejects_inference_mode_cache(backend, grad_mode, kind):+ from unsloth.utils import attention_dispatch++ cpu = torch.device("cpu")+ seq = torch.tensor([3, 5], dtype = torch.int32)+ attention_dispatch._WINDOW_MASK_CACHE.clear()++ def build(seq):+ if kind == "metadata":+ return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, cpu)[1]+ return attention_dispatch._windowed_causal_mask(8, 8, 2, cpu)++ def train(x, seq):+ saved = build(seq)+ # Multiply by the original int/bool tensor: casting would hide the inference-tensor bug.+ return (x * saved).sum(), saved++ torch._dynamo.reset()+ try:+ compiled_build = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)+ compiled_train = torch.compile(train, backend = backend, fullgraph = True, dynamic = True)+ with torch._dynamo.config.patch(capture_scalar_outputs = True):+ with getattr(torch, grad_mode)():+ built = compiled_build(seq)+ assert torch.is_inference(built) == (grad_mode == "inference_mode")+ expected = built.clone()+ if kind == "metadata":+ assert expected.tolist() == [0, 3, 8]+ for _ in range(3):+ x = torch.ones(built.shape, requires_grad = True)+ loss, saved = compiled_train(x, seq)+ loss.backward()+ assert not torch.is_inference(saved)+ torch.testing.assert_close(saved, expected)+ torch.testing.assert_close(x.grad, expected.float())+ assert build(seq) is saved+ finally:+ attention_dispatch._WINDOW_MASK_CACHE.clear()+ torch._dynamo.reset()++[email protected]("backend", ["eager", "aot_eager", "inductor"])[email protected]("mode", [torch.no_grad, torch.inference_mode])[email protected]("warm_inside", [False, True])[email protected]("total", [8, 10])+def test_compiled_mode_transition_preserves_padded_metadata(backend, mode, warm_inside, total):+ packing.clear_packed_caches()+ torch._dynamo.reset()+ device = torch.device("cpu")+ seq = torch.tensor([3, 5], dtype = torch.int32)++ def build(seq):+ return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)++ def train(x, seq):+ padded, _ = packing.cover_padded_cu_seqlens(build(seq), total)+ return (x * padded).sum()++ try:+ with torch._dynamo.config.patch(capture_scalar_outputs = True):+ with mode():+ info = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)(seq)+ if warm_inside:+ padded, _ = packing.cover_padded_cu_seqlens(info, total)+ if not warm_inside:+ padded, _ = packing.cover_padded_cu_seqlens(info, total)+ expected = torch.tensor([0.0, 3.0, 8.0] + ([10.0] if total == 10 else []))+ x = torch.ones(expected.shape, requires_grad = True)+ torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(x, seq).backward()+ torch.testing.assert_close(x.grad, expected)+ finally:+ packing.clear_packed_caches()+ torch._dynamo.reset()++[email protected]("backend", ["eager", "aot_eager", "inductor"])+def test_compiled_cache_checks_replaced_tensor_provenance(backend):+ device = torch.device("cpu")+ seq = torch.tensor([3, 5], dtype = torch.int32)++ def build(seq):+ return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)[1]++ def train(x, seq):+ return (x * build(seq)).sum()++ torch._dynamo.reset()+ try:+ producer = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)+ consumer = torch.compile(train, backend = backend, fullgraph = True, dynamic = True)+ with torch._dynamo.config.patch(capture_scalar_outputs = True):+ for mode in (torch.no_grad, torch.inference_mode, torch.no_grad, torch.inference_mode):+ packing.clear_packed_caches()+ with mode():+ producer(seq)+ x = torch.ones(3, requires_grad = True)+ consumer(x, seq).backward()+ torch.testing.assert_close(x.grad, torch.tensor([0.0, 3.0, 8.0]))+ finally:+ torch._dynamo.reset()++[email protected]("backend", ["eager", "aot_eager", "inductor"])[email protected]("mode", [torch.no_grad, torch.inference_mode])+def test_compiled_cache_created_and_consumed_in_one_graph(backend, mode):+ device = torch.device("cpu")++ def train(x, seq):+ with mode():+ packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)+ saved = packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)[1]+ return (x * saved).sum()++ torch._dynamo.reset()+ try:+ x = torch.ones(3, requires_grad = True)+ seq = torch.tensor([3, 5], dtype = torch.int32)+ with torch._dynamo.config.patch(capture_scalar_outputs = True):+ torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(x, seq).backward()+ torch.testing.assert_close(x.grad, torch.tensor([0.0, 3.0, 8.0]))+ finally:+ torch._dynamo.reset()++[email protected]("backend", ["eager", "aot_eager", "inductor"])[email protected]("dtype", [torch.int32, torch.int64])[email protected]("mode", [torch.no_grad, torch.inference_mode])[email protected]("warm_inside", [False, True])[email protected]("kind", ["mask", "segments"])+def test_compiled_mode_transition_preserves_derived_cache(backend, dtype, mode, warm_inside, kind):+ device = torch.device("cpu")+ seq = torch.tensor([3, 5], dtype = dtype)++ def build(seq):+ return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)++ def consume(info):+ if kind == "mask":+ mask = packing.build_sdpa_packed_attention_mask(+ info, dtype = torch.float32, device = device, total_tokens = 10+ )+ return mask.isfinite().float(), mask+ coeff = torch.tensor(packing.packed_segment_lengths(info, 10), dtype = torch.float32)+ return coeff, coeff++ def train(x, seq):+ coeff, saved = consume(build(seq))+ return (x * coeff).sum(), saved++ torch._dynamo.reset()+ try:+ with torch._dynamo.config.patch(capture_scalar_outputs = True):+ with mode():+ info = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)(seq)+ if warm_inside:+ expected = consume(info)[0]+ if not warm_inside:+ expected = consume(info)[0]+ expected = expected.clone()+ x = torch.ones(expected.shape, requires_grad = True)+ loss, saved = torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(+ x, seq+ )+ loss.backward()+ assert not torch.is_inference(saved)+ torch.testing.assert_close(x.grad, expected)+ finally:+ torch._dynamo.reset()+++@_needs_cuda[email protected]("backend", ["eager", "aot_eager"])+def test_compiled_inference_metadata_preserves_cuda_device_alias(backend):+ device = torch.device("cuda")+ seq = torch.tensor([3, 5], dtype = torch.int32, device = "cpu")++ def build(seq):+ return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)++ def train(x, seq):+ padded, _ = packing.cover_padded_cu_seqlens(build(seq), 10)+ return (x * padded).sum()++ torch._dynamo.reset()+ try:+ with torch._dynamo.config.patch(capture_scalar_outputs = True):+ with torch.inference_mode():+ info = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)(seq)+ packing.cover_padded_cu_seqlens(info, 10)+ x = torch.ones(4, device = info[1].device, requires_grad = True)+ torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(x, seq).backward()+ torch.testing.assert_close(x.grad, x.new_tensor([0.0, 3.0, 8.0, 10.0]))+ finally:+ torch._dynamo.reset()+++def test_new_lengths_do_not_inherit_derived_caches():+ device = torch.device("cpu")+ with torch.inference_mode():+ old = packing.get_packed_info_from_kwargs(+ {"packed_seq_lengths": torch.tensor([3, 5], dtype = torch.int64)}, device+ )+ packing.cover_padded_cu_seqlens(old, 10)+ packing.build_sdpa_packed_attention_mask(old, dtype = torch.float32, device = device)+ packing.packed_segment_lengths(old, 10)+ new = packing.get_packed_info_from_kwargs(+ {"packed_seq_lengths": torch.tensor([2, 6], dtype = torch.int64)}, device+ )+ assert packing.cover_padded_cu_seqlens(new, 10)[0].tolist() == [0, 2, 8, 10]+ assert packing.packed_segment_lengths(new, 10) == (2, 6, 2)+ mask = packing.build_sdpa_packed_attention_mask(new, dtype = torch.float32, device = device)+ assert mask.isfinite().sum().item() == 24++def test_xformers_mask_lru_keeps_inference_masks_out_of_training(monkeypatch):class _Mask:@classmethoddiff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex a3d0b1c6ba5..11d431b03a5 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -30,7 +30,8 @@ from ..models._utils import *from ..context_parallel import get_cp_managerfrom .ulysses import ulysses_attentionfrom ..utils.packing import (- _inference_mode,+ _built_in_inference_mode,+ _cache_hit,build_sdpa_packed_attention_mask,build_xformers_block_causal_mask,cover_padded_cu_seqlens,@@ -1312,12 +1313,7 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -"""Causal band mask of shape (1, 1, q_len, k_len). Read-only: callers must not mutate it."""params = (q_len, k_len, sliding_window)entry = _WINDOW_MASK_CACHE.get(device)- if (- entry is not None- and entry["params"] == params- # A mask built under inference_mode cannot be saved for a training step's backward.- and (not entry["inference"] or _inference_mode())- ):+ if entry is not None and entry["params"] == params and _cache_hit(entry):return entry["mask"]# Drop the outgoing mask first so a shape change does not peak a whole mask higher._WINDOW_MASK_CACHE.pop(device, None)@@ -1328,7 +1324,11 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -(k_pos[None, :] <= q_pos[:, None])& (k_pos[None, :] >= (q_pos[:, None] - (sliding_window - 1))))[None, None, :, :]- _WINDOW_MASK_CACHE[device] = {"params": params, "mask": mask, "inference": _inference_mode()}+ _WINDOW_MASK_CACHE[device] = {+ "params": params,+ "mask": mask,+ "inference": _built_in_inference_mode(),+ }return maskdiff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex 9db793ab0b7..feaf354a834 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -1760,9 +1760,49 @@ def _inference_mode() -> bool:return not _is_compiling() and torch.is_inference_mode_enabled()+def _built_in_inference_mode():+ """None marks a traced grad-off build whose tensor provenance is not yet known."""+ if _is_compiling():+ return None if not torch.is_grad_enabled() else False+ return torch.is_inference_mode_enabled()++[email protected]_constant_result+def _cached_tensor_is_unsafe(tensor) -> bool:+ # Constant evaluation loses inference context for intermediates created in this trace.+ if torch.is_inference(tensor):+ return True+ for cache in (_PACKED_INFO_CACHE, _PADDED_CU_SEQLENS_CACHE, _SDPA_MASK_CACHE):+ for entry in cache.values():+ value = entry.get("result", entry.get("mask"))+ values = value if isinstance(value, (tuple, list)) else (value,)+ if any(tensor is item for item in values):+ return False+ return True+++def _holds_inference_tensor(value) -> bool:+ if isinstance(value, torch.Tensor):+ return _cached_tensor_is_unsafe(value) if _is_compiling() else torch.is_inference(value)+ if isinstance(value, (tuple, list)):+ return any(_holds_inference_tensor(item) for item in value)+ return not isinstance(value, (int, float, bool, type(None)))++def _cache_hit(entry) -> bool:- # Inference tensors may serve inference mode only; normal tensors serve both.- return not entry["inference"] or _inference_mode()+ inference = entry["inference"]+ if inference is None:+ if _is_compiling():+ return not torch.is_grad_enabled() or not _holds_inference_tensor(+ entry.get("result", entry.get("mask"))+ )+ if torch.is_inference_mode_enabled():+ return True+ if _holds_inference_tensor(entry.get("result", entry.get("mask"))):+ return False+ entry["inference"] = False+ return True+ return not inference or _inference_mode()def _pinned_to_device(tensor: torch.Tensor, device) -> torch.Tensor:@@ -1775,6 +1815,38 @@ def _pinned_to_device(tensor: torch.Tensor, device) -> torch.Tensor:return pinned.to(device = device, non_blocking = True)+def _refresh_derived_packed_caches(previous, seq_lengths, result, device):+ if previous is None or previous["seq_lengths"] is not seq_lengths:+ return+ segments = _SEGMENT_LENGTHS_CACHE.get(result[0].device)+ if segments is not None and segments[0] is previous["result"][0]:+ _SEGMENT_LENGTHS_CACHE[result[0].device] = (result[0], segments[1], segments[2])+ padded_device = result[1].device+ padded = _PADDED_CU_SEQLENS_CACHE.get(padded_device)+ if padded is not None and padded["cu_seqlens"] is previous["result"][1]:+ value = padded["result"]+ if _holds_inference_tensor(value):+ value = (value[0].clone(), value[1])+ _PADDED_CU_SEQLENS_CACHE[padded_device] = {+ "cu_seqlens": result[1],+ "total": padded["total"],+ "result": value,+ "inference": _built_in_inference_mode(),+ }+ cached = _SDPA_MASK_CACHE.get(device)+ if cached is None or cached["seq_lengths"] is not previous["result"][0]:+ return+ mask = cached["mask"]+ if _holds_inference_tensor(mask):+ mask = mask.clone()+ _SDPA_MASK_CACHE[device] = {+ "seq_lengths": result[0],+ "params": cached["params"],+ "mask": mask,+ "inference": _built_in_inference_mode(),+ }++def get_packed_info_from_kwargs(kwargs: dict,device: torch.device,@@ -1805,10 +1877,11 @@ def get_packed_info_from_kwargs(torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])max_seqlen = int(lengths.max().item())result = (lengths, cu_seqlens, max_seqlen)+ _refresh_derived_packed_caches(entry, seq_lengths, result, device)_PACKED_INFO_CACHE[device] = {"seq_lengths": seq_lengths,"result": result,- "inference": False,+ "inference": _built_in_inference_mode(),}return result@@ -1832,10 +1905,11 @@ def get_packed_info_from_kwargs(max_seqlen = max(host)result = (lengths, cu_seqlens, max_seqlen)+ _refresh_derived_packed_caches(entry, seq_lengths, result, device)_PACKED_INFO_CACHE[device] = {"seq_lengths": seq_lengths,"result": result,- "inference": _inference_mode(),+ "inference": _built_in_inference_mode(),}_PACKED_HOST_LENGTHS_CACHE[lengths.device] = (lengths, cu_seqlens, host)return result@@ -1908,7 +1982,7 @@ def cover_padded_cu_seqlens("cu_seqlens": cu_seqlens,"total": total_tokens,"result": result,- "inference": _inference_mode(),+ "inference": _built_in_inference_mode(),}return result@@ -1951,7 +2025,7 @@ def build_xformers_block_causal_mask("seq_lengths": seq_lengths,"params": params,"mask": mask,- "inference": _inference_mode(),+ "inference": _built_in_inference_mode(),}else:mask = base_mask@@ -2046,7 +2120,7 @@ def build_sdpa_packed_attention_mask("seq_lengths": seq_lengths,"params": params,"mask": result,- "inference": _inference_mode(),+ "inference": _built_in_inference_mode(),}return result