Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Packed metadata: keep the tensor path under torch.compile, fall back to pageable copies (#15199)

Daniel Han18380d95876b
diff --git a/tests/utils/test_packing_host_sync.py b/tests/utils/test_packing_host_sync.py
index b065e2a4813..4e4adb1fa15 100644
--- a/tests/utils/test_packing_host_sync.py
+++ b/tests/utils/test_packing_host_sync.py
@@ -245,3 +245,56 @@ def test_non_blocking_batches_only_for_pinned_cuda_loaders(monkeypatch):
t, config = trainer()
assert _unsloth_non_blocking_batches(t) is False and config.non_blocking is False
assert _unsloth_non_blocking_batches(SimpleNamespace()) is False
+
+
+def _compile_over_segment_counts(dynamic):
+ import torch._dynamo as dynamo
+ from torch._dynamo.utils import counters
+
+ def step(x, lengths):
+ packing.clear_packed_caches()
+ _, cu_seqlens, max_seqlen = packing.get_packed_info_from_kwargs(
+ {"packed_seq_lengths": lengths}, x.device
+ )
+ return x * max_seqlen + cu_seqlens[-1]
+
+ dynamo.reset()
+ counters.clear()
+ compiled = torch.compile(step, backend = "eager", dynamic = dynamic)
+ for n in range(3, 15):
+ lengths = torch.full((n,), 4, dtype = torch.int32)
+ out = compiled(torch.ones(4), lengths)
+ assert torch.equal(out, torch.ones(4) * 4 + 4 * n)
+ graphs = counters["stats"]["unique_graphs"]
+ dynamo.reset()
+ return graphs
+
+
+def test_compiled_dynamic_does_not_specialize_on_segment_count():
+ # 12 segment counts: the tensor path traces a handful of graphs; a host tuple of lengths
+ # would trace one per count and hit the recompile limit.
+ assert _compile_over_segment_counts(dynamic = True) <= 4
+
+
+def test_compiling_takes_tensor_path(monkeypatch):
+ monkeypatch.setattr(packing, "_is_compiling", lambda: True)
+ seq = torch.tensor([3, 5, 2], dtype = torch.int64)
+ lengths, cu, max_seqlen = packing.get_packed_info_from_kwargs(
+ {"packed_seq_lengths": seq}, "cpu"
+ )
+ old = _old_packed_info(seq, "cpu")
+ _same(lengths, old[0])
+ _same(cu, old[1])
+ assert max_seqlen == old[2]
+ # No host tuple recorded: downstream helpers take their own (traceable) route.
+ assert packing._cached_host_lengths(lengths) is None
+
+
+def test_pinned_to_device_falls_back_to_pageable(monkeypatch):
+ t = torch.arange(5, dtype = torch.int32)
+
+ def refuse(self, *args, **kwargs):
+ raise RuntimeError("cudaHostAlloc failed: out of pinned memory")
+
+ monkeypatch.setattr(torch.Tensor, "pin_memory", refuse)
+ _same(packing._pinned_to_device(t, "cpu"), t)
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index ad24bf68c9c..6eddc45e0b6 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2257,7 +2257,11 @@ def _unsloth_average_gradients(ddp, bucket_bytes = 64 << 20):
present = torch.tensor([p.grad is not None for p in params], dtype = torch.uint8, device = "cpu")
# torch.tensor(..., device = cuda) is a pageable, stream-draining copy; pinned + non_blocking is not.
if device.type == "cuda":
- present = present.pin_memory().to(device, non_blocking = True)
+ try:
+ present = present.pin_memory().to(device, non_blocking = True)
+ except RuntimeError:
+ # Pinning can fail (pinned limit, WSL); a pageable copy carries the same mask.
+ present = present.to(device)
else:
present = present.to(device)
dist.all_reduce(present, op = dist.ReduceOp.MAX, group = group)
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index e5c09ba139d..9972e11c1eb 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -1690,6 +1690,24 @@ def _stage_packed_metadata(seq_lengths: torch.Tensor) -> Tuple[torch.Tensor, Tup
return staged, tuple(host_lengths.tolist())
+def _is_compiling() -> bool:
+ is_compiling = getattr(getattr(torch, "compiler", None), "is_compiling", None)
+ try:
+ return bool(is_compiling()) if is_compiling is not None else False
+ except Exception:
+ return False
+
+
+def _pinned_to_device(tensor: torch.Tensor, device) -> torch.Tensor:
+ """non_blocking copy from pinned memory; a plain (blocking) copy when pinning fails."""
+ try:
+ pinned = tensor.pin_memory()
+ except RuntimeError:
+ # No pinnable host memory (pinned limit, WSL, a CUDA-less build): pageable copy, same values.
+ return tensor.to(device = device)
+ return pinned.to(device = device, non_blocking = True)
+
+
def get_packed_info_from_kwargs(
kwargs: dict,
device: torch.device,
@@ -1712,6 +1730,17 @@ def get_packed_info_from_kwargs(
if entry is not None and entry["seq_lengths"] is seq_lengths:
return entry["result"]
+ if _is_compiling():
+ # Under a user torch.compile, a host tuple of lengths specializes the graph on the segment
+ # count (one recompile per count under dynamic = True) and pin_memory cannot be traced.
+ lengths = seq_lengths.to(device = device, dtype = torch.int32, non_blocking = True)
+ cu_seqlens = torch.zeros(lengths.numel() + 1, dtype = torch.int32, device = device)
+ torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])
+ max_seqlen = int(lengths.max().item())
+ result = (lengths, cu_seqlens, max_seqlen)
+ _PACKED_INFO_CACHE[device] = {"seq_lengths": seq_lengths, "result": result}
+ return result
+
if (
isinstance(seq_lengths, torch.Tensor)
and seq_lengths.device.type == "cpu"
@@ -1720,7 +1749,7 @@ def get_packed_info_from_kwargs(
# One pinned non_blocking H2D copy; pageable copies and .item() both drain the stream.
staged, host = _stage_packed_metadata(seq_lengths)
n = len(host)
- moved = staged.pin_memory().to(device = device, non_blocking = True)
+ moved = _pinned_to_device(staged, device)
lengths, cu_seqlens = moved[:n], moved[n:]
else:
lengths = seq_lengths.to(device = device, dtype = torch.int32, non_blocking = True)