Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Packing: do not serve compiled inference_mode metadata to a training step (#16183)

Daniel Hana32528bb8a80
diff --git a/tests/utils/test_packing_host_sync.py b/tests/utils/test_packing_host_sync.py
index c4bea33e376..b6b0c789e6a 100644
--- a/tests/utils/test_packing_host_sync.py
+++ b/tests/utils/test_packing_host_sync.py
@@ -10,6 +10,7 @@ import pytest
import torch
import unsloth.utils.packing as packing
+from real_accelerator import has_real_cuda
def _old_packed_info(seq_lengths, device):
@@ -160,7 +161,7 @@ def test_fused_ce_int_divisor_is_bitwise_torch_tensor():
assert a.shape == b.shape and torch.equal(a.view(torch.int32), b.view(torch.int32))
-_needs_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")
+_needs_cuda = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
def _count_syncs(fn):
@@ -345,6 +346,252 @@ def test_inference_mode_metadata_is_not_served_to_a_training_step(dtype):
)
[email protected]("grad_mode", ["inference_mode", "no_grad"])
+def test_compiled_inference_mode_metadata_is_not_served_to_a_training_step(grad_mode):
+ from unsloth.utils import attention_dispatch
+ from unsloth.utils.attention_dispatch import _windowed_causal_mask
+
+ packing._PACKED_INFO_CACHE.clear()
+ attention_dispatch._WINDOW_MASK_CACHE.clear()
+ cpu = torch.device("cpu")
+ seq = torch.tensor([3, 5], dtype = torch.int32)
+
+ def build(seq):
+ info = packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, cpu)
+ return info, _windowed_causal_mask(8, 8, 2, cpu)
+
+ torch._dynamo.reset()
+ compiled = torch.compile(build, backend = "eager", fullgraph = True, dynamic = True)
+ context = torch.inference_mode() if grad_mode == "inference_mode" else torch.no_grad()
+ with torch._dynamo.config.patch(capture_scalar_outputs = True), context:
+ built, built_mask = compiled(seq)
+ assert torch.is_inference(built[1]) == (grad_mode == "inference_mode")
+ train, train_mask = build(seq)
+ assert not torch.is_inference(train[1]) and not torch.is_inference(train_mask)
+ assert train[1].tolist() == [0, 3, 8] and train[2] == 5
+ _survives_backward(train[1])
+ _survives_backward(train_mask.float())
+ assert (train is built) == (grad_mode == "no_grad")
+ assert (train_mask is built_mask) == (grad_mode == "no_grad")
+
+
[email protected]("backend", ["eager", "aot_eager", "inductor"])
[email protected]("grad_mode", ["inference_mode", "no_grad"])
[email protected]("kind", ["metadata", "window_mask"])
+def test_compiled_training_rejects_inference_mode_cache(backend, grad_mode, kind):
+ from unsloth.utils import attention_dispatch
+
+ cpu = torch.device("cpu")
+ seq = torch.tensor([3, 5], dtype = torch.int32)
+ attention_dispatch._WINDOW_MASK_CACHE.clear()
+
+ def build(seq):
+ if kind == "metadata":
+ return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, cpu)[1]
+ return attention_dispatch._windowed_causal_mask(8, 8, 2, cpu)
+
+ def train(x, seq):
+ saved = build(seq)
+ # Multiply by the original int/bool tensor: casting would hide the inference-tensor bug.
+ return (x * saved).sum(), saved
+
+ torch._dynamo.reset()
+ try:
+ compiled_build = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)
+ compiled_train = torch.compile(train, backend = backend, fullgraph = True, dynamic = True)
+ with torch._dynamo.config.patch(capture_scalar_outputs = True):
+ with getattr(torch, grad_mode)():
+ built = compiled_build(seq)
+ assert torch.is_inference(built) == (grad_mode == "inference_mode")
+ expected = built.clone()
+ if kind == "metadata":
+ assert expected.tolist() == [0, 3, 8]
+ for _ in range(3):
+ x = torch.ones(built.shape, requires_grad = True)
+ loss, saved = compiled_train(x, seq)
+ loss.backward()
+ assert not torch.is_inference(saved)
+ torch.testing.assert_close(saved, expected)
+ torch.testing.assert_close(x.grad, expected.float())
+ assert build(seq) is saved
+ finally:
+ attention_dispatch._WINDOW_MASK_CACHE.clear()
+ torch._dynamo.reset()
+
+
[email protected]("backend", ["eager", "aot_eager", "inductor"])
[email protected]("mode", [torch.no_grad, torch.inference_mode])
[email protected]("warm_inside", [False, True])
[email protected]("total", [8, 10])
+def test_compiled_mode_transition_preserves_padded_metadata(backend, mode, warm_inside, total):
+ packing.clear_packed_caches()
+ torch._dynamo.reset()
+ device = torch.device("cpu")
+ seq = torch.tensor([3, 5], dtype = torch.int32)
+
+ def build(seq):
+ return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)
+
+ def train(x, seq):
+ padded, _ = packing.cover_padded_cu_seqlens(build(seq), total)
+ return (x * padded).sum()
+
+ try:
+ with torch._dynamo.config.patch(capture_scalar_outputs = True):
+ with mode():
+ info = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)(seq)
+ if warm_inside:
+ padded, _ = packing.cover_padded_cu_seqlens(info, total)
+ if not warm_inside:
+ padded, _ = packing.cover_padded_cu_seqlens(info, total)
+ expected = torch.tensor([0.0, 3.0, 8.0] + ([10.0] if total == 10 else []))
+ x = torch.ones(expected.shape, requires_grad = True)
+ torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(x, seq).backward()
+ torch.testing.assert_close(x.grad, expected)
+ finally:
+ packing.clear_packed_caches()
+ torch._dynamo.reset()
+
+
[email protected]("backend", ["eager", "aot_eager", "inductor"])
+def test_compiled_cache_checks_replaced_tensor_provenance(backend):
+ device = torch.device("cpu")
+ seq = torch.tensor([3, 5], dtype = torch.int32)
+
+ def build(seq):
+ return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)[1]
+
+ def train(x, seq):
+ return (x * build(seq)).sum()
+
+ torch._dynamo.reset()
+ try:
+ producer = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)
+ consumer = torch.compile(train, backend = backend, fullgraph = True, dynamic = True)
+ with torch._dynamo.config.patch(capture_scalar_outputs = True):
+ for mode in (torch.no_grad, torch.inference_mode, torch.no_grad, torch.inference_mode):
+ packing.clear_packed_caches()
+ with mode():
+ producer(seq)
+ x = torch.ones(3, requires_grad = True)
+ consumer(x, seq).backward()
+ torch.testing.assert_close(x.grad, torch.tensor([0.0, 3.0, 8.0]))
+ finally:
+ torch._dynamo.reset()
+
+
[email protected]("backend", ["eager", "aot_eager", "inductor"])
[email protected]("mode", [torch.no_grad, torch.inference_mode])
+def test_compiled_cache_created_and_consumed_in_one_graph(backend, mode):
+ device = torch.device("cpu")
+
+ def train(x, seq):
+ with mode():
+ packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)
+ saved = packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)[1]
+ return (x * saved).sum()
+
+ torch._dynamo.reset()
+ try:
+ x = torch.ones(3, requires_grad = True)
+ seq = torch.tensor([3, 5], dtype = torch.int32)
+ with torch._dynamo.config.patch(capture_scalar_outputs = True):
+ torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(x, seq).backward()
+ torch.testing.assert_close(x.grad, torch.tensor([0.0, 3.0, 8.0]))
+ finally:
+ torch._dynamo.reset()
+
+
[email protected]("backend", ["eager", "aot_eager", "inductor"])
[email protected]("dtype", [torch.int32, torch.int64])
[email protected]("mode", [torch.no_grad, torch.inference_mode])
[email protected]("warm_inside", [False, True])
[email protected]("kind", ["mask", "segments"])
+def test_compiled_mode_transition_preserves_derived_cache(backend, dtype, mode, warm_inside, kind):
+ device = torch.device("cpu")
+ seq = torch.tensor([3, 5], dtype = dtype)
+
+ def build(seq):
+ return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)
+
+ def consume(info):
+ if kind == "mask":
+ mask = packing.build_sdpa_packed_attention_mask(
+ info, dtype = torch.float32, device = device, total_tokens = 10
+ )
+ return mask.isfinite().float(), mask
+ coeff = torch.tensor(packing.packed_segment_lengths(info, 10), dtype = torch.float32)
+ return coeff, coeff
+
+ def train(x, seq):
+ coeff, saved = consume(build(seq))
+ return (x * coeff).sum(), saved
+
+ torch._dynamo.reset()
+ try:
+ with torch._dynamo.config.patch(capture_scalar_outputs = True):
+ with mode():
+ info = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)(seq)
+ if warm_inside:
+ expected = consume(info)[0]
+ if not warm_inside:
+ expected = consume(info)[0]
+ expected = expected.clone()
+ x = torch.ones(expected.shape, requires_grad = True)
+ loss, saved = torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(
+ x, seq
+ )
+ loss.backward()
+ assert not torch.is_inference(saved)
+ torch.testing.assert_close(x.grad, expected)
+ finally:
+ torch._dynamo.reset()
+
+
+@_needs_cuda
[email protected]("backend", ["eager", "aot_eager"])
+def test_compiled_inference_metadata_preserves_cuda_device_alias(backend):
+ device = torch.device("cuda")
+ seq = torch.tensor([3, 5], dtype = torch.int32, device = "cpu")
+
+ def build(seq):
+ return packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, device)
+
+ def train(x, seq):
+ padded, _ = packing.cover_padded_cu_seqlens(build(seq), 10)
+ return (x * padded).sum()
+
+ torch._dynamo.reset()
+ try:
+ with torch._dynamo.config.patch(capture_scalar_outputs = True):
+ with torch.inference_mode():
+ info = torch.compile(build, backend = backend, fullgraph = True, dynamic = True)(seq)
+ packing.cover_padded_cu_seqlens(info, 10)
+ x = torch.ones(4, device = info[1].device, requires_grad = True)
+ torch.compile(train, backend = backend, fullgraph = True, dynamic = True)(x, seq).backward()
+ torch.testing.assert_close(x.grad, x.new_tensor([0.0, 3.0, 8.0, 10.0]))
+ finally:
+ torch._dynamo.reset()
+
+
+def test_new_lengths_do_not_inherit_derived_caches():
+ device = torch.device("cpu")
+ with torch.inference_mode():
+ old = packing.get_packed_info_from_kwargs(
+ {"packed_seq_lengths": torch.tensor([3, 5], dtype = torch.int64)}, device
+ )
+ packing.cover_padded_cu_seqlens(old, 10)
+ packing.build_sdpa_packed_attention_mask(old, dtype = torch.float32, device = device)
+ packing.packed_segment_lengths(old, 10)
+ new = packing.get_packed_info_from_kwargs(
+ {"packed_seq_lengths": torch.tensor([2, 6], dtype = torch.int64)}, device
+ )
+ assert packing.cover_padded_cu_seqlens(new, 10)[0].tolist() == [0, 2, 8, 10]
+ assert packing.packed_segment_lengths(new, 10) == (2, 6, 2)
+ mask = packing.build_sdpa_packed_attention_mask(new, dtype = torch.float32, device = device)
+ assert mask.isfinite().sum().item() == 24
+
+
def test_xformers_mask_lru_keeps_inference_masks_out_of_training(monkeypatch):
class _Mask:
@classmethod
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index a3d0b1c6ba5..11d431b03a5 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -30,7 +30,8 @@ from ..models._utils import *
from ..context_parallel import get_cp_manager
from .ulysses import ulysses_attention
from ..utils.packing import (
- _inference_mode,
+ _built_in_inference_mode,
+ _cache_hit,
build_sdpa_packed_attention_mask,
build_xformers_block_causal_mask,
cover_padded_cu_seqlens,
@@ -1312,12 +1313,7 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -
"""Causal band mask of shape (1, 1, q_len, k_len). Read-only: callers must not mutate it."""
params = (q_len, k_len, sliding_window)
entry = _WINDOW_MASK_CACHE.get(device)
- if (
- entry is not None
- and entry["params"] == params
- # A mask built under inference_mode cannot be saved for a training step's backward.
- and (not entry["inference"] or _inference_mode())
- ):
+ if entry is not None and entry["params"] == params and _cache_hit(entry):
return entry["mask"]
# Drop the outgoing mask first so a shape change does not peak a whole mask higher.
_WINDOW_MASK_CACHE.pop(device, None)
@@ -1328,7 +1324,11 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -
(k_pos[None, :] <= q_pos[:, None])
& (k_pos[None, :] >= (q_pos[:, None] - (sliding_window - 1)))
)[None, None, :, :]
- _WINDOW_MASK_CACHE[device] = {"params": params, "mask": mask, "inference": _inference_mode()}
+ _WINDOW_MASK_CACHE[device] = {
+ "params": params,
+ "mask": mask,
+ "inference": _built_in_inference_mode(),
+ }
return mask
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index 9db793ab0b7..feaf354a834 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -1760,9 +1760,49 @@ def _inference_mode() -> bool:
return not _is_compiling() and torch.is_inference_mode_enabled()
+def _built_in_inference_mode():
+ """None marks a traced grad-off build whose tensor provenance is not yet known."""
+ if _is_compiling():
+ return None if not torch.is_grad_enabled() else False
+ return torch.is_inference_mode_enabled()
+
+
[email protected]_constant_result
+def _cached_tensor_is_unsafe(tensor) -> bool:
+ # Constant evaluation loses inference context for intermediates created in this trace.
+ if torch.is_inference(tensor):
+ return True
+ for cache in (_PACKED_INFO_CACHE, _PADDED_CU_SEQLENS_CACHE, _SDPA_MASK_CACHE):
+ for entry in cache.values():
+ value = entry.get("result", entry.get("mask"))
+ values = value if isinstance(value, (tuple, list)) else (value,)
+ if any(tensor is item for item in values):
+ return False
+ return True
+
+
+def _holds_inference_tensor(value) -> bool:
+ if isinstance(value, torch.Tensor):
+ return _cached_tensor_is_unsafe(value) if _is_compiling() else torch.is_inference(value)
+ if isinstance(value, (tuple, list)):
+ return any(_holds_inference_tensor(item) for item in value)
+ return not isinstance(value, (int, float, bool, type(None)))
+
+
def _cache_hit(entry) -> bool:
- # Inference tensors may serve inference mode only; normal tensors serve both.
- return not entry["inference"] or _inference_mode()
+ inference = entry["inference"]
+ if inference is None:
+ if _is_compiling():
+ return not torch.is_grad_enabled() or not _holds_inference_tensor(
+ entry.get("result", entry.get("mask"))
+ )
+ if torch.is_inference_mode_enabled():
+ return True
+ if _holds_inference_tensor(entry.get("result", entry.get("mask"))):
+ return False
+ entry["inference"] = False
+ return True
+ return not inference or _inference_mode()
def _pinned_to_device(tensor: torch.Tensor, device) -> torch.Tensor:
@@ -1775,6 +1815,38 @@ def _pinned_to_device(tensor: torch.Tensor, device) -> torch.Tensor:
return pinned.to(device = device, non_blocking = True)
+def _refresh_derived_packed_caches(previous, seq_lengths, result, device):
+ if previous is None or previous["seq_lengths"] is not seq_lengths:
+ return
+ segments = _SEGMENT_LENGTHS_CACHE.get(result[0].device)
+ if segments is not None and segments[0] is previous["result"][0]:
+ _SEGMENT_LENGTHS_CACHE[result[0].device] = (result[0], segments[1], segments[2])
+ padded_device = result[1].device
+ padded = _PADDED_CU_SEQLENS_CACHE.get(padded_device)
+ if padded is not None and padded["cu_seqlens"] is previous["result"][1]:
+ value = padded["result"]
+ if _holds_inference_tensor(value):
+ value = (value[0].clone(), value[1])
+ _PADDED_CU_SEQLENS_CACHE[padded_device] = {
+ "cu_seqlens": result[1],
+ "total": padded["total"],
+ "result": value,
+ "inference": _built_in_inference_mode(),
+ }
+ cached = _SDPA_MASK_CACHE.get(device)
+ if cached is None or cached["seq_lengths"] is not previous["result"][0]:
+ return
+ mask = cached["mask"]
+ if _holds_inference_tensor(mask):
+ mask = mask.clone()
+ _SDPA_MASK_CACHE[device] = {
+ "seq_lengths": result[0],
+ "params": cached["params"],
+ "mask": mask,
+ "inference": _built_in_inference_mode(),
+ }
+
+
def get_packed_info_from_kwargs(
kwargs: dict,
device: torch.device,
@@ -1805,10 +1877,11 @@ def get_packed_info_from_kwargs(
torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])
max_seqlen = int(lengths.max().item())
result = (lengths, cu_seqlens, max_seqlen)
+ _refresh_derived_packed_caches(entry, seq_lengths, result, device)
_PACKED_INFO_CACHE[device] = {
"seq_lengths": seq_lengths,
"result": result,
- "inference": False,
+ "inference": _built_in_inference_mode(),
}
return result
@@ -1832,10 +1905,11 @@ def get_packed_info_from_kwargs(
max_seqlen = max(host)
result = (lengths, cu_seqlens, max_seqlen)
+ _refresh_derived_packed_caches(entry, seq_lengths, result, device)
_PACKED_INFO_CACHE[device] = {
"seq_lengths": seq_lengths,
"result": result,
- "inference": _inference_mode(),
+ "inference": _built_in_inference_mode(),
}
_PACKED_HOST_LENGTHS_CACHE[lengths.device] = (lengths, cu_seqlens, host)
return result
@@ -1908,7 +1982,7 @@ def cover_padded_cu_seqlens(
"cu_seqlens": cu_seqlens,
"total": total_tokens,
"result": result,
- "inference": _inference_mode(),
+ "inference": _built_in_inference_mode(),
}
return result
@@ -1951,7 +2025,7 @@ def build_xformers_block_causal_mask(
"seq_lengths": seq_lengths,
"params": params,
"mask": mask,
- "inference": _inference_mode(),
+ "inference": _built_in_inference_mode(),
}
else:
mask = base_mask
@@ -2046,7 +2120,7 @@ def build_sdpa_packed_attention_mask(
"seq_lengths": seq_lengths,
"params": params,
"mask": result,
- "inference": _inference_mode(),
+ "inference": _built_in_inference_mode(),
}
return result