Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Packed metadata: keep the tensor path under torch.compile, fall back to pageable copies (#15199)
Daniel Han18380d95876b
diff --git a/tests/utils/test_packing_host_sync.py b/tests/utils/test_packing_host_sync.pyindex b065e2a4813..4e4adb1fa15 100644--- a/tests/utils/test_packing_host_sync.py+++ b/tests/utils/test_packing_host_sync.py@@ -245,3 +245,56 @@ def test_non_blocking_batches_only_for_pinned_cuda_loaders(monkeypatch):t, config = trainer()assert _unsloth_non_blocking_batches(t) is False and config.non_blocking is Falseassert _unsloth_non_blocking_batches(SimpleNamespace()) is False+++def _compile_over_segment_counts(dynamic):+ import torch._dynamo as dynamo+ from torch._dynamo.utils import counters++ def step(x, lengths):+ packing.clear_packed_caches()+ _, cu_seqlens, max_seqlen = packing.get_packed_info_from_kwargs(+ {"packed_seq_lengths": lengths}, x.device+ )+ return x * max_seqlen + cu_seqlens[-1]++ dynamo.reset()+ counters.clear()+ compiled = torch.compile(step, backend = "eager", dynamic = dynamic)+ for n in range(3, 15):+ lengths = torch.full((n,), 4, dtype = torch.int32)+ out = compiled(torch.ones(4), lengths)+ assert torch.equal(out, torch.ones(4) * 4 + 4 * n)+ graphs = counters["stats"]["unique_graphs"]+ dynamo.reset()+ return graphs+++def test_compiled_dynamic_does_not_specialize_on_segment_count():+ # 12 segment counts: the tensor path traces a handful of graphs; a host tuple of lengths+ # would trace one per count and hit the recompile limit.+ assert _compile_over_segment_counts(dynamic = True) <= 4+++def test_compiling_takes_tensor_path(monkeypatch):+ monkeypatch.setattr(packing, "_is_compiling", lambda: True)+ seq = torch.tensor([3, 5, 2], dtype = torch.int64)+ lengths, cu, max_seqlen = packing.get_packed_info_from_kwargs(+ {"packed_seq_lengths": seq}, "cpu"+ )+ old = _old_packed_info(seq, "cpu")+ _same(lengths, old[0])+ _same(cu, old[1])+ assert max_seqlen == old[2]+ # No host tuple recorded: downstream helpers take their own (traceable) route.+ assert packing._cached_host_lengths(lengths) is None+++def test_pinned_to_device_falls_back_to_pageable(monkeypatch):+ t = torch.arange(5, dtype = torch.int32)++ def refuse(self, *args, **kwargs):+ raise RuntimeError("cudaHostAlloc failed: out of pinned memory")++ monkeypatch.setattr(torch.Tensor, "pin_memory", refuse)+ _same(packing._pinned_to_device(t, "cpu"), t)diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex ad24bf68c9c..6eddc45e0b6 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2257,7 +2257,11 @@ def _unsloth_average_gradients(ddp, bucket_bytes = 64 << 20):present = torch.tensor([p.grad is not None for p in params], dtype = torch.uint8, device = "cpu")# torch.tensor(..., device = cuda) is a pageable, stream-draining copy; pinned + non_blocking is not.if device.type == "cuda":- present = present.pin_memory().to(device, non_blocking = True)+ try:+ present = present.pin_memory().to(device, non_blocking = True)+ except RuntimeError:+ # Pinning can fail (pinned limit, WSL); a pageable copy carries the same mask.+ present = present.to(device)else:present = present.to(device)dist.all_reduce(present, op = dist.ReduceOp.MAX, group = group)diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex e5c09ba139d..9972e11c1eb 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -1690,6 +1690,24 @@ def _stage_packed_metadata(seq_lengths: torch.Tensor) -> Tuple[torch.Tensor, Tupreturn staged, tuple(host_lengths.tolist())+def _is_compiling() -> bool:+ is_compiling = getattr(getattr(torch, "compiler", None), "is_compiling", None)+ try:+ return bool(is_compiling()) if is_compiling is not None else False+ except Exception:+ return False+++def _pinned_to_device(tensor: torch.Tensor, device) -> torch.Tensor:+ """non_blocking copy from pinned memory; a plain (blocking) copy when pinning fails."""+ try:+ pinned = tensor.pin_memory()+ except RuntimeError:+ # No pinnable host memory (pinned limit, WSL, a CUDA-less build): pageable copy, same values.+ return tensor.to(device = device)+ return pinned.to(device = device, non_blocking = True)++def get_packed_info_from_kwargs(kwargs: dict,device: torch.device,@@ -1712,6 +1730,17 @@ def get_packed_info_from_kwargs(if entry is not None and entry["seq_lengths"] is seq_lengths:return entry["result"]+ if _is_compiling():+ # Under a user torch.compile, a host tuple of lengths specializes the graph on the segment+ # count (one recompile per count under dynamic = True) and pin_memory cannot be traced.+ lengths = seq_lengths.to(device = device, dtype = torch.int32, non_blocking = True)+ cu_seqlens = torch.zeros(lengths.numel() + 1, dtype = torch.int32, device = device)+ torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])+ max_seqlen = int(lengths.max().item())+ result = (lengths, cu_seqlens, max_seqlen)+ _PACKED_INFO_CACHE[device] = {"seq_lengths": seq_lengths, "result": result}+ return result+if (isinstance(seq_lengths, torch.Tensor)and seq_lengths.device.type == "cpu"@@ -1720,7 +1749,7 @@ def get_packed_info_from_kwargs(# One pinned non_blocking H2D copy; pageable copies and .item() both drain the stream.staged, host = _stage_packed_metadata(seq_lengths)n = len(host)- moved = staged.pin_memory().to(device = device, non_blocking = True)+ moved = _pinned_to_device(staged, device)lengths, cu_seqlens = moved[:n], moved[n:]else:lengths = seq_lengths.to(device = device, dtype = torch.int32, non_blocking = True)