Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Do not serve inference_mode packed metadata to a training forward (#15347)
Daniel Hanfa46d9620e11
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.pyindex e3d4e1c5a22..b3b4b5510e7 100644--- a/tests/test_hf_packed_attention.py+++ b/tests/test_hf_packed_attention.py@@ -1341,6 +1341,32 @@ def test_block_varlen_kwargs_rechecks_a_mask_edited_in_place(monkeypatch):assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs) is None+def test_block_varlen_kwargs_does_not_serve_inference_metadata_to_training(monkeypatch):+ """inference_mode forward then a training forward on the same batch: fresh cu_seqlens."""+ hpa._ORIG_SDPA[0] = _plain_sdpa+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())+ monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: False)+ monkeypatch.setattr(hpa, "_BLOCK_PREP", {})+ lengths = [3, 5]+ monkeypatch.setattr(+ "unsloth.utils.packing.get_packed_info_from_kwargs",+ lambda kwargs, device: (torch.tensor(lengths), _cu(lengths), 5),+ )+ hidden = torch.randn(1, 8, 8)+ kwargs = {"packed_seq_lengths": torch.tensor(lengths), "attention_mask": _block_causal(lengths)}+ with torch.inference_mode():+ cu, _ = hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)+ assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)[0] is cu+ assert torch.is_inference(cu)+ train, _ = hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)+ assert not torch.is_inference(train) and train.tolist() == [0, 3, 8]+ assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)[0] is train+ with torch.inference_mode():+ assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)[0] is train++def test_block_varlen_kwargs_survives_a_collected_mask_then_none(monkeypatch):"""Same lengths, the cached mask collected, then a call without a mask: refused, not a crash."""hpa._ORIG_SDPA[0] = _plain_sdpadiff --git a/tests/utils/test_attention_masks.py b/tests/utils/test_attention_masks.pyindex b3fb6e01008..c90baf31ab0 100644--- a/tests/utils/test_attention_masks.py+++ b/tests/utils/test_attention_masks.py@@ -797,6 +797,23 @@ def test_the_window_mask_is_built_once_per_shape(monkeypatch):attention_dispatch._WINDOW_MASK_CACHE.clear()+def test_an_inference_mode_window_mask_is_not_served_to_training():+ """Eval under inference_mode then a training step of the same shape: the cached mask must not+ be an inference tensor, which SDPA backends that save the mask cannot keep for backward."""+ attention_dispatch._WINDOW_MASK_CACHE.clear()+ cpu = torch.device("cpu")+ with torch.inference_mode():+ evaluated = attention_dispatch._windowed_causal_mask(6, 6, 3, cpu)+ assert attention_dispatch._windowed_causal_mask(6, 6, 3, cpu) is evaluated+ trained = attention_dispatch._windowed_causal_mask(6, 6, 3, cpu)+ assert not torch.is_inference(trained)+ assert torch.equal(trained, evaluated)+ assert attention_dispatch._windowed_causal_mask(6, 6, 3, cpu) is trained+ with torch.inference_mode():+ assert attention_dispatch._windowed_causal_mask(6, 6, 3, cpu) is trained+ attention_dispatch._WINDOW_MASK_CACHE.clear()++def test_the_outgoing_window_mask_is_freed_before_its_replacement(monkeypatch):"""A shape change must not hold two dense masks at once. Dynamic-length training walksthrough shapes, and at 32K each mask is 1 GiB on top of the construction temporaries."""diff --git a/tests/utils/test_packing_host_sync.py b/tests/utils/test_packing_host_sync.pyindex 4e4adb1fa15..c4bea33e376 100644--- a/tests/utils/test_packing_host_sync.py+++ b/tests/utils/test_packing_host_sync.py@@ -298,3 +298,86 @@ def test_pinned_to_device_falls_back_to_pageable(monkeypatch):monkeypatch.setattr(torch.Tensor, "pin_memory", refuse)_same(packing._pinned_to_device(t, "cpu"), t)+++class _SaveForBackward(torch.autograd.Function):+ @staticmethod+ def forward(ctx, x, saved):+ ctx.save_for_backward(saved)+ return x * 2++ @staticmethod+ def backward(ctx, grad):+ return grad * 2, None+++def _survives_backward(saved):+ x = torch.randn(4, requires_grad = True)+ _SaveForBackward.apply(x, saved).sum().backward()++[email protected]("dtype", [torch.int32, torch.int64])+def test_inference_mode_metadata_is_not_served_to_a_training_step(dtype):+ """inference_mode forward then a training forward on the same batch: no inference tensors."""+ cpu = torch.device("cpu")+ kwargs = {"packed_seq_lengths": torch.tensor([3, 5], dtype = dtype)}+ with torch.inference_mode():+ info = packing.get_packed_info_from_kwargs(kwargs, cpu)+ padded = packing.cover_padded_cu_seqlens(info, 10)+ mask = packing.build_sdpa_packed_attention_mask(info, dtype = torch.float32, device = cpu)+ assert torch.is_inference(info[1])+ train = packing.get_packed_info_from_kwargs(kwargs, cpu)+ assert train is not info and not torch.is_inference(train[1])+ assert train[1].tolist() == [0, 3, 8] and train[2] == 5+ _survives_backward(train[1])+ train_padded = packing.cover_padded_cu_seqlens(train, 10)+ assert not torch.is_inference(train_padded[0]) and train_padded[0].tolist() == [0, 3, 8, 10]+ _survives_backward(train_padded[0])+ train_mask = packing.build_sdpa_packed_attention_mask(train, dtype = torch.float32, device = cpu)+ assert not torch.is_inference(train_mask)+ torch.testing.assert_close(train_mask, mask.clone())+ _survives_backward(train_mask)+ assert packing.get_packed_info_from_kwargs(kwargs, cpu) is train+ assert packing.cover_padded_cu_seqlens(train, 10) is train_padded+ assert (+ packing.build_sdpa_packed_attention_mask(train, dtype = torch.float32, device = cpu)+ is train_mask+ )+++def test_xformers_mask_lru_keeps_inference_masks_out_of_training(monkeypatch):+ class _Mask:+ @classmethod+ def from_seqlens(cls, lengths):+ return cls()++ monkeypatch.setattr(packing, "_XFormersBlockMask", _Mask)+ monkeypatch.setattr(packing, "move_xformers_attention_bias", lambda mask, device: mask)+ packing._XFORMERS_MASK_CACHE.clear()+ cpu = torch.device("cpu")+ with torch.inference_mode():+ evaluated = packing._get_cached_block_mask((3, 5), None, cpu)+ assert packing._get_cached_block_mask((3, 5), None, cpu) is evaluated+ trained = packing._get_cached_block_mask((3, 5), None, cpu)+ assert trained is not evaluated+ assert packing._get_cached_block_mask((3, 5), None, cpu) is trained+ # A training-built mask serves inference_mode too: no second entry per lengths tuple.+ packing._XFORMERS_MASK_CACHE.clear()+ trained = packing._get_cached_block_mask((2, 2), None, cpu)+ with torch.inference_mode():+ assert packing._get_cached_block_mask((2, 2), None, cpu) is trained+ assert len(packing._XFORMERS_MASK_CACHE) == 1+ packing._XFORMERS_MASK_CACHE.clear()+++def test_training_metadata_is_reused_under_inference_mode():+ cpu = torch.device("cpu")+ kwargs = {"packed_seq_lengths": torch.tensor([3, 5], dtype = torch.int32)}+ train = packing.get_packed_info_from_kwargs(kwargs, cpu)+ with torch.inference_mode():+ assert packing.get_packed_info_from_kwargs(kwargs, cpu) is train+ other = {"packed_seq_lengths": torch.tensor([2, 2], dtype = torch.int32)}+ info = packing.get_packed_info_from_kwargs(other, cpu)+ assert packing.get_packed_info_from_kwargs(other, cpu) is info+ assert packing.get_packed_info_from_kwargs(kwargs, cpu) is not train+ _survives_backward(packing.get_packed_info_from_kwargs(kwargs, cpu)[1])diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 963bcb7f724..e77d471e950 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -29,6 +29,7 @@ from ..models._utils import *from ..context_parallel import get_cp_managerfrom .ulysses import ulysses_attentionfrom ..utils.packing import (+ _inference_mode,build_sdpa_packed_attention_mask,build_xformers_block_causal_mask,cover_padded_cu_seqlens,@@ -1222,7 +1223,12 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -"""Causal band mask of shape (1, 1, q_len, k_len). Read-only: callers must not mutate it."""params = (q_len, k_len, sliding_window)entry = _WINDOW_MASK_CACHE.get(device)- if entry is not None and entry["params"] == params:+ if (+ entry is not None+ and entry["params"] == params+ # A mask built under inference_mode cannot be saved for a training step's backward.+ and (not entry["inference"] or _inference_mode())+ ):return entry["mask"]# Drop the outgoing mask first so a shape change does not peak a whole mask higher._WINDOW_MASK_CACHE.pop(device, None)@@ -1233,7 +1239,7 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -(k_pos[None, :] <= q_pos[:, None])& (k_pos[None, :] >= (q_pos[:, None] - (sliding_window - 1))))[None, None, :, :]- _WINDOW_MASK_CACHE[device] = {"params": params, "mask": mask}+ _WINDOW_MASK_CACHE[device] = {"params": params, "mask": mask, "inference": _inference_mode()}return maskdiff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.pyindex 5ee47ce6ad4..7fb9b8aae6c 100644--- a/unsloth/utils/hf_packed_attention.py+++ b/unsloth/utils/hf_packed_attention.py@@ -734,6 +734,8 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):and entry[0]() is seq_lengthsand entry[1]() is maskand entry[2] == (hidden_states.device, seq_lengths._version, mask._version)+ # cu_seqlens built under inference_mode cannot be saved for a training step's backward.+ and (not entry[4] or torch.is_inference_mode_enabled())):if entry[3] is not None:HF_PACKED_ATTENTION_STATS["compiled"] += 1@@ -752,6 +754,7 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):weakref.ref(mask),(hidden_states.device, seq_lengths._version, mask._version),result,+ torch.is_inference_mode_enabled(),)return resultdiff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex e63c75d5f7b..ae83c2dd991 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -46,7 +46,7 @@ except Exception:_XFormersBidirectionalMask = None_XFORMERS_MASK_CACHE_MAXSIZE = 32-_XFORMERS_MASK_CACHE: OrderedDict[Tuple[torch.device, Tuple[int, ...], int, bool], Any] = (+_XFORMERS_MASK_CACHE: OrderedDict[Tuple[torch.device, Tuple[int, ...], int, bool, bool], Any] = (OrderedDict())@@ -131,11 +131,15 @@ def _get_cached_block_mask(device = torch.device(device)window_key = _window_cache_key(sliding_window)- cache_key = (device, lengths, window_key, is_causal)- cached = _XFORMERS_MASK_CACHE.get(cache_key)- if cached is not None:- _XFORMERS_MASK_CACHE.move_to_end(cache_key)- return cached+ # Outlives the batch: inference_mode-built masks must never reach a training step.+ inference = _inference_mode()+ for cache_key in ((device, lengths, window_key, is_causal, False),) + (+ ((device, lengths, window_key, is_causal, True),) if inference else ()+ ):+ cached = _XFORMERS_MASK_CACHE.get(cache_key)+ if cached is not None:+ _XFORMERS_MASK_CACHE.move_to_end(cache_key)+ return cachedmask = mask_class.from_seqlens(list(lengths))if window_key and mask is not None and hasattr(mask, "make_local_attention"):@@ -1697,6 +1701,16 @@ def _is_compiling() -> bool:return False+def _inference_mode() -> bool:+ """Traced builds count as normal: torch.is_inference_mode_enabled graph-breaks dynamo."""+ return not _is_compiling() and torch.is_inference_mode_enabled()+++def _cache_hit(entry) -> bool:+ # Inference tensors may serve inference mode only; normal tensors serve both.+ return not entry["inference"] or _inference_mode()++def _pinned_to_device(tensor: torch.Tensor, device) -> torch.Tensor:"""non_blocking copy from pinned memory; a plain (blocking) copy when pinning fails."""try:@@ -1726,7 +1740,7 @@ def get_packed_info_from_kwargs(seq_lengths = cu_seqlens[1:] - cu_seqlens[:-1]entry = _PACKED_INFO_CACHE.get(device)- if entry is not None and entry["seq_lengths"] is seq_lengths:+ if entry is not None and entry["seq_lengths"] is seq_lengths and _cache_hit(entry):return entry["result"]if _is_compiling():@@ -1737,7 +1751,11 @@ def get_packed_info_from_kwargs(torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])max_seqlen = int(lengths.max().item())result = (lengths, cu_seqlens, max_seqlen)- _PACKED_INFO_CACHE[device] = {"seq_lengths": seq_lengths, "result": result}+ _PACKED_INFO_CACHE[device] = {+ "seq_lengths": seq_lengths,+ "result": result,+ "inference": False,+ }return resultif (@@ -1759,7 +1777,11 @@ def get_packed_info_from_kwargs(max_seqlen = max(host)result = (lengths, cu_seqlens, max_seqlen)- _PACKED_INFO_CACHE[device] = {"seq_lengths": seq_lengths, "result": result}+ _PACKED_INFO_CACHE[device] = {+ "seq_lengths": seq_lengths,+ "result": result,+ "inference": _inference_mode(),+ }_PACKED_HOST_LENGTHS_CACHE[lengths.device] = (lengths, cu_seqlens, host)return result@@ -1799,7 +1821,12 @@ def cover_padded_cu_seqlens(_, cu_seqlens, max_seqlen = seq_infodevice = cu_seqlens.deviceentry = _PADDED_CU_SEQLENS_CACHE.get(device)- if entry is not None and entry["cu_seqlens"] is cu_seqlens and entry["total"] == total_tokens:+ if (+ entry is not None+ and entry["cu_seqlens"] is cu_seqlens+ and entry["total"] == total_tokens+ and _cache_hit(entry)+ ):return entry["result"]host = _cached_host_lengths(seq_info[0], cu_seqlens)@@ -1816,6 +1843,7 @@ def cover_padded_cu_seqlens("cu_seqlens": cu_seqlens,"total": total_tokens,"result": result,+ "inference": _inference_mode(),}return result@@ -1837,7 +1865,12 @@ def build_xformers_block_causal_mask(device = seq_lengths.deviceparams = (sliding_window, total_tokens, is_causal)entry = _XFORMERS_BLOCK_MASK_CACHE.get(device)- if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:+ if (+ entry is not None+ and entry["seq_lengths"] is seq_lengths+ and entry["params"] == params+ and _cache_hit(entry)+ ):return entry["mask"]lengths = _cached_host_lengths(seq_lengths)@@ -1853,6 +1886,7 @@ def build_xformers_block_causal_mask("seq_lengths": seq_lengths,"params": params,"mask": mask,+ "inference": _inference_mode(),}else:mask = base_mask@@ -1913,7 +1947,12 @@ def build_sdpa_packed_attention_mask(params = (dtype, sliding_window, total_tokens, is_causal)entry = _SDPA_MASK_CACHE.get(device)- if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:+ if (+ entry is not None+ and entry["seq_lengths"] is seq_lengths+ and entry["params"] == params+ and _cache_hit(entry)+ ):return entry["mask"]lengths = _with_padding_segment(_host_segment_lengths(seq_lengths), total_tokens)@@ -1942,6 +1981,7 @@ def build_sdpa_packed_attention_mask("seq_lengths": seq_lengths,"params": params,"mask": result,+ "inference": _inference_mode(),}return result