Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Do not serve inference_mode packed metadata to a training forward (#15347)

Daniel Hanfa46d9620e11
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.py
index e3d4e1c5a22..b3b4b5510e7 100644
--- a/tests/test_hf_packed_attention.py
+++ b/tests/test_hf_packed_attention.py
@@ -1341,6 +1341,32 @@ def test_block_varlen_kwargs_rechecks_a_mask_edited_in_place(monkeypatch):
assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs) is None
+def test_block_varlen_kwargs_does_not_serve_inference_metadata_to_training(monkeypatch):
+ """inference_mode forward then a training forward on the same batch: fresh cu_seqlens."""
+ hpa._ORIG_SDPA[0] = _plain_sdpa
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
+ monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: False)
+ monkeypatch.setattr(hpa, "_BLOCK_PREP", {})
+ lengths = [3, 5]
+ monkeypatch.setattr(
+ "unsloth.utils.packing.get_packed_info_from_kwargs",
+ lambda kwargs, device: (torch.tensor(lengths), _cu(lengths), 5),
+ )
+ hidden = torch.randn(1, 8, 8)
+ kwargs = {"packed_seq_lengths": torch.tensor(lengths), "attention_mask": _block_causal(lengths)}
+ with torch.inference_mode():
+ cu, _ = hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)
+ assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)[0] is cu
+ assert torch.is_inference(cu)
+ train, _ = hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)
+ assert not torch.is_inference(train) and train.tolist() == [0, 3, 8]
+ assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)[0] is train
+ with torch.inference_mode():
+ assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)[0] is train
+
+
def test_block_varlen_kwargs_survives_a_collected_mask_then_none(monkeypatch):
"""Same lengths, the cached mask collected, then a call without a mask: refused, not a crash."""
hpa._ORIG_SDPA[0] = _plain_sdpa
diff --git a/tests/utils/test_attention_masks.py b/tests/utils/test_attention_masks.py
index b3fb6e01008..c90baf31ab0 100644
--- a/tests/utils/test_attention_masks.py
+++ b/tests/utils/test_attention_masks.py
@@ -797,6 +797,23 @@ def test_the_window_mask_is_built_once_per_shape(monkeypatch):
attention_dispatch._WINDOW_MASK_CACHE.clear()
+def test_an_inference_mode_window_mask_is_not_served_to_training():
+ """Eval under inference_mode then a training step of the same shape: the cached mask must not
+ be an inference tensor, which SDPA backends that save the mask cannot keep for backward."""
+ attention_dispatch._WINDOW_MASK_CACHE.clear()
+ cpu = torch.device("cpu")
+ with torch.inference_mode():
+ evaluated = attention_dispatch._windowed_causal_mask(6, 6, 3, cpu)
+ assert attention_dispatch._windowed_causal_mask(6, 6, 3, cpu) is evaluated
+ trained = attention_dispatch._windowed_causal_mask(6, 6, 3, cpu)
+ assert not torch.is_inference(trained)
+ assert torch.equal(trained, evaluated)
+ assert attention_dispatch._windowed_causal_mask(6, 6, 3, cpu) is trained
+ with torch.inference_mode():
+ assert attention_dispatch._windowed_causal_mask(6, 6, 3, cpu) is trained
+ attention_dispatch._WINDOW_MASK_CACHE.clear()
+
+
def test_the_outgoing_window_mask_is_freed_before_its_replacement(monkeypatch):
"""A shape change must not hold two dense masks at once. Dynamic-length training walks
through shapes, and at 32K each mask is 1 GiB on top of the construction temporaries."""
diff --git a/tests/utils/test_packing_host_sync.py b/tests/utils/test_packing_host_sync.py
index 4e4adb1fa15..c4bea33e376 100644
--- a/tests/utils/test_packing_host_sync.py
+++ b/tests/utils/test_packing_host_sync.py
@@ -298,3 +298,86 @@ def test_pinned_to_device_falls_back_to_pageable(monkeypatch):
monkeypatch.setattr(torch.Tensor, "pin_memory", refuse)
_same(packing._pinned_to_device(t, "cpu"), t)
+
+
+class _SaveForBackward(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, x, saved):
+ ctx.save_for_backward(saved)
+ return x * 2
+
+ @staticmethod
+ def backward(ctx, grad):
+ return grad * 2, None
+
+
+def _survives_backward(saved):
+ x = torch.randn(4, requires_grad = True)
+ _SaveForBackward.apply(x, saved).sum().backward()
+
+
[email protected]("dtype", [torch.int32, torch.int64])
+def test_inference_mode_metadata_is_not_served_to_a_training_step(dtype):
+ """inference_mode forward then a training forward on the same batch: no inference tensors."""
+ cpu = torch.device("cpu")
+ kwargs = {"packed_seq_lengths": torch.tensor([3, 5], dtype = dtype)}
+ with torch.inference_mode():
+ info = packing.get_packed_info_from_kwargs(kwargs, cpu)
+ padded = packing.cover_padded_cu_seqlens(info, 10)
+ mask = packing.build_sdpa_packed_attention_mask(info, dtype = torch.float32, device = cpu)
+ assert torch.is_inference(info[1])
+ train = packing.get_packed_info_from_kwargs(kwargs, cpu)
+ assert train is not info and not torch.is_inference(train[1])
+ assert train[1].tolist() == [0, 3, 8] and train[2] == 5
+ _survives_backward(train[1])
+ train_padded = packing.cover_padded_cu_seqlens(train, 10)
+ assert not torch.is_inference(train_padded[0]) and train_padded[0].tolist() == [0, 3, 8, 10]
+ _survives_backward(train_padded[0])
+ train_mask = packing.build_sdpa_packed_attention_mask(train, dtype = torch.float32, device = cpu)
+ assert not torch.is_inference(train_mask)
+ torch.testing.assert_close(train_mask, mask.clone())
+ _survives_backward(train_mask)
+ assert packing.get_packed_info_from_kwargs(kwargs, cpu) is train
+ assert packing.cover_padded_cu_seqlens(train, 10) is train_padded
+ assert (
+ packing.build_sdpa_packed_attention_mask(train, dtype = torch.float32, device = cpu)
+ is train_mask
+ )
+
+
+def test_xformers_mask_lru_keeps_inference_masks_out_of_training(monkeypatch):
+ class _Mask:
+ @classmethod
+ def from_seqlens(cls, lengths):
+ return cls()
+
+ monkeypatch.setattr(packing, "_XFormersBlockMask", _Mask)
+ monkeypatch.setattr(packing, "move_xformers_attention_bias", lambda mask, device: mask)
+ packing._XFORMERS_MASK_CACHE.clear()
+ cpu = torch.device("cpu")
+ with torch.inference_mode():
+ evaluated = packing._get_cached_block_mask((3, 5), None, cpu)
+ assert packing._get_cached_block_mask((3, 5), None, cpu) is evaluated
+ trained = packing._get_cached_block_mask((3, 5), None, cpu)
+ assert trained is not evaluated
+ assert packing._get_cached_block_mask((3, 5), None, cpu) is trained
+ # A training-built mask serves inference_mode too: no second entry per lengths tuple.
+ packing._XFORMERS_MASK_CACHE.clear()
+ trained = packing._get_cached_block_mask((2, 2), None, cpu)
+ with torch.inference_mode():
+ assert packing._get_cached_block_mask((2, 2), None, cpu) is trained
+ assert len(packing._XFORMERS_MASK_CACHE) == 1
+ packing._XFORMERS_MASK_CACHE.clear()
+
+
+def test_training_metadata_is_reused_under_inference_mode():
+ cpu = torch.device("cpu")
+ kwargs = {"packed_seq_lengths": torch.tensor([3, 5], dtype = torch.int32)}
+ train = packing.get_packed_info_from_kwargs(kwargs, cpu)
+ with torch.inference_mode():
+ assert packing.get_packed_info_from_kwargs(kwargs, cpu) is train
+ other = {"packed_seq_lengths": torch.tensor([2, 2], dtype = torch.int32)}
+ info = packing.get_packed_info_from_kwargs(other, cpu)
+ assert packing.get_packed_info_from_kwargs(other, cpu) is info
+ assert packing.get_packed_info_from_kwargs(kwargs, cpu) is not train
+ _survives_backward(packing.get_packed_info_from_kwargs(kwargs, cpu)[1])
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 963bcb7f724..e77d471e950 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -29,6 +29,7 @@ from ..models._utils import *
from ..context_parallel import get_cp_manager
from .ulysses import ulysses_attention
from ..utils.packing import (
+ _inference_mode,
build_sdpa_packed_attention_mask,
build_xformers_block_causal_mask,
cover_padded_cu_seqlens,
@@ -1222,7 +1223,12 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -
"""Causal band mask of shape (1, 1, q_len, k_len). Read-only: callers must not mutate it."""
params = (q_len, k_len, sliding_window)
entry = _WINDOW_MASK_CACHE.get(device)
- if entry is not None and entry["params"] == params:
+ if (
+ entry is not None
+ and entry["params"] == params
+ # A mask built under inference_mode cannot be saved for a training step's backward.
+ and (not entry["inference"] or _inference_mode())
+ ):
return entry["mask"]
# Drop the outgoing mask first so a shape change does not peak a whole mask higher.
_WINDOW_MASK_CACHE.pop(device, None)
@@ -1233,7 +1239,7 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -
(k_pos[None, :] <= q_pos[:, None])
& (k_pos[None, :] >= (q_pos[:, None] - (sliding_window - 1)))
)[None, None, :, :]
- _WINDOW_MASK_CACHE[device] = {"params": params, "mask": mask}
+ _WINDOW_MASK_CACHE[device] = {"params": params, "mask": mask, "inference": _inference_mode()}
return mask
diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.py
index 5ee47ce6ad4..7fb9b8aae6c 100644
--- a/unsloth/utils/hf_packed_attention.py
+++ b/unsloth/utils/hf_packed_attention.py
@@ -734,6 +734,8 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):
and entry[0]() is seq_lengths
and entry[1]() is mask
and entry[2] == (hidden_states.device, seq_lengths._version, mask._version)
+ # cu_seqlens built under inference_mode cannot be saved for a training step's backward.
+ and (not entry[4] or torch.is_inference_mode_enabled())
):
if entry[3] is not None:
HF_PACKED_ATTENTION_STATS["compiled"] += 1
@@ -752,6 +754,7 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):
weakref.ref(mask),
(hidden_states.device, seq_lengths._version, mask._version),
result,
+ torch.is_inference_mode_enabled(),
)
return result
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index e63c75d5f7b..ae83c2dd991 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -46,7 +46,7 @@ except Exception:
_XFormersBidirectionalMask = None
_XFORMERS_MASK_CACHE_MAXSIZE = 32
-_XFORMERS_MASK_CACHE: OrderedDict[Tuple[torch.device, Tuple[int, ...], int, bool], Any] = (
+_XFORMERS_MASK_CACHE: OrderedDict[Tuple[torch.device, Tuple[int, ...], int, bool, bool], Any] = (
OrderedDict()
)
@@ -131,11 +131,15 @@ def _get_cached_block_mask(
device = torch.device(device)
window_key = _window_cache_key(sliding_window)
- cache_key = (device, lengths, window_key, is_causal)
- cached = _XFORMERS_MASK_CACHE.get(cache_key)
- if cached is not None:
- _XFORMERS_MASK_CACHE.move_to_end(cache_key)
- return cached
+ # Outlives the batch: inference_mode-built masks must never reach a training step.
+ inference = _inference_mode()
+ for cache_key in ((device, lengths, window_key, is_causal, False),) + (
+ ((device, lengths, window_key, is_causal, True),) if inference else ()
+ ):
+ cached = _XFORMERS_MASK_CACHE.get(cache_key)
+ if cached is not None:
+ _XFORMERS_MASK_CACHE.move_to_end(cache_key)
+ return cached
mask = mask_class.from_seqlens(list(lengths))
if window_key and mask is not None and hasattr(mask, "make_local_attention"):
@@ -1697,6 +1701,16 @@ def _is_compiling() -> bool:
return False
+def _inference_mode() -> bool:
+ """Traced builds count as normal: torch.is_inference_mode_enabled graph-breaks dynamo."""
+ return not _is_compiling() and torch.is_inference_mode_enabled()
+
+
+def _cache_hit(entry) -> bool:
+ # Inference tensors may serve inference mode only; normal tensors serve both.
+ return not entry["inference"] or _inference_mode()
+
+
def _pinned_to_device(tensor: torch.Tensor, device) -> torch.Tensor:
"""non_blocking copy from pinned memory; a plain (blocking) copy when pinning fails."""
try:
@@ -1726,7 +1740,7 @@ def get_packed_info_from_kwargs(
seq_lengths = cu_seqlens[1:] - cu_seqlens[:-1]
entry = _PACKED_INFO_CACHE.get(device)
- if entry is not None and entry["seq_lengths"] is seq_lengths:
+ if entry is not None and entry["seq_lengths"] is seq_lengths and _cache_hit(entry):
return entry["result"]
if _is_compiling():
@@ -1737,7 +1751,11 @@ def get_packed_info_from_kwargs(
torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])
max_seqlen = int(lengths.max().item())
result = (lengths, cu_seqlens, max_seqlen)
- _PACKED_INFO_CACHE[device] = {"seq_lengths": seq_lengths, "result": result}
+ _PACKED_INFO_CACHE[device] = {
+ "seq_lengths": seq_lengths,
+ "result": result,
+ "inference": False,
+ }
return result
if (
@@ -1759,7 +1777,11 @@ def get_packed_info_from_kwargs(
max_seqlen = max(host)
result = (lengths, cu_seqlens, max_seqlen)
- _PACKED_INFO_CACHE[device] = {"seq_lengths": seq_lengths, "result": result}
+ _PACKED_INFO_CACHE[device] = {
+ "seq_lengths": seq_lengths,
+ "result": result,
+ "inference": _inference_mode(),
+ }
_PACKED_HOST_LENGTHS_CACHE[lengths.device] = (lengths, cu_seqlens, host)
return result
@@ -1799,7 +1821,12 @@ def cover_padded_cu_seqlens(
_, cu_seqlens, max_seqlen = seq_info
device = cu_seqlens.device
entry = _PADDED_CU_SEQLENS_CACHE.get(device)
- if entry is not None and entry["cu_seqlens"] is cu_seqlens and entry["total"] == total_tokens:
+ if (
+ entry is not None
+ and entry["cu_seqlens"] is cu_seqlens
+ and entry["total"] == total_tokens
+ and _cache_hit(entry)
+ ):
return entry["result"]
host = _cached_host_lengths(seq_info[0], cu_seqlens)
@@ -1816,6 +1843,7 @@ def cover_padded_cu_seqlens(
"cu_seqlens": cu_seqlens,
"total": total_tokens,
"result": result,
+ "inference": _inference_mode(),
}
return result
@@ -1837,7 +1865,12 @@ def build_xformers_block_causal_mask(
device = seq_lengths.device
params = (sliding_window, total_tokens, is_causal)
entry = _XFORMERS_BLOCK_MASK_CACHE.get(device)
- if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:
+ if (
+ entry is not None
+ and entry["seq_lengths"] is seq_lengths
+ and entry["params"] == params
+ and _cache_hit(entry)
+ ):
return entry["mask"]
lengths = _cached_host_lengths(seq_lengths)
@@ -1853,6 +1886,7 @@ def build_xformers_block_causal_mask(
"seq_lengths": seq_lengths,
"params": params,
"mask": mask,
+ "inference": _inference_mode(),
}
else:
mask = base_mask
@@ -1913,7 +1947,12 @@ def build_sdpa_packed_attention_mask(
params = (dtype, sliding_window, total_tokens, is_causal)
entry = _SDPA_MASK_CACHE.get(device)
- if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:
+ if (
+ entry is not None
+ and entry["seq_lengths"] is seq_lengths
+ and entry["params"] == params
+ and _cache_hit(entry)
+ ):
return entry["mask"]
lengths = _with_padding_segment(_host_segment_lengths(seq_lengths), total_tokens)
@@ -1942,6 +1981,7 @@ def build_sdpa_packed_attention_mask(
"seq_lengths": seq_lengths,
"params": params,
"mask": result,
+ "inference": _inference_mode(),
}
return result