Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Remove hidden host-device syncs from per-step training paths (#15133)

Daniel Han50bb320f7dff
diff --git a/tests/utils/test_packing_host_sync.py b/tests/utils/test_packing_host_sync.py
new file mode 100644
index 00000000000..b065e2a4813
--- /dev/null
+++ b/tests/utils/test_packing_host_sync.py
@@ -0,0 +1,247 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Host-built packed metadata must match the old device reads bit for bit, without per-step syncs."""
+
+import math
+import warnings
+
+import pytest
+import torch
+
+import unsloth.utils.packing as packing
+
+
+def _old_packed_info(seq_lengths, device):
+ lengths = seq_lengths.to(device = device, dtype = torch.int32)
+ cu_seqlens = torch.zeros(lengths.numel() + 1, dtype = torch.int32, device = device)
+ torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])
+ return lengths, cu_seqlens, int(lengths.max().item())
+
+
+def _old_cover(seq_info, total_tokens):
+ _, cu_seqlens, max_seqlen = seq_info
+ padding = total_tokens - int(cu_seqlens[-1].item())
+ if padding > 0:
+ tail = torch.tensor([total_tokens], dtype = cu_seqlens.dtype, device = cu_seqlens.device)
+ return torch.cat([cu_seqlens, tail]), max(max_seqlen, padding)
+ return cu_seqlens, max_seqlen
+
+
+def _random_batches(n = 200, seed = 0):
+ g = torch.Generator().manual_seed(seed)
+ for i in range(n):
+ k = int(torch.randint(1, 33, (1,), generator = g))
+ high = [2, 9, 300, 5000][i % 4]
+ dtype = [torch.int32, torch.int64][i % 2]
+ yield torch.randint(1, high, (k,), generator = g, dtype = dtype)
+
+
+def _same(a, b):
+ assert a.dtype == b.dtype and a.shape == b.shape and a.device == b.device
+ assert torch.equal(a, b)
+
+
[email protected](autouse = True)
+def _clear():
+ packing.clear_packed_caches()
+ yield
+ packing.clear_packed_caches()
+
+
+def test_stage_packed_metadata_matches_device_cumsum():
+ for seq in _random_batches():
+ staged, host = packing._stage_packed_metadata(seq)
+ lengths, cu, max_seqlen = _old_packed_info(seq, "cpu")
+ n = seq.numel()
+ assert staged.dtype == torch.int32 and staged.numel() == 2 * n + 1
+ _same(staged[:n], lengths)
+ _same(staged[n:], cu)
+ assert host == tuple(lengths.tolist()) and max(host) == max_seqlen
+ assert all(type(x) is int for x in host)
+
+
+def test_stage_packed_metadata_stays_on_host_under_a_default_device():
+ # torch.set_default_device("cuda") must not move the host staging buffer (it is pinned next).
+ seq = torch.tensor([3, 1, 4], dtype = torch.int32)
+ torch.set_default_device("meta")
+ try:
+ staged, host = packing._stage_packed_metadata(seq)
+ finally:
+ torch.set_default_device(None)
+ assert staged.device.type == "cpu" and host == (3, 1, 4)
+ assert staged.tolist() == [3, 1, 4, 0, 3, 4, 8]
+
+
+def test_packed_info_and_consumers_match_old_values():
+ cpu = torch.device("cpu")
+ for i, seq in enumerate(_random_batches()):
+ packing.clear_packed_caches()
+ lengths, cu, max_seqlen = packing.get_packed_info_from_kwargs(
+ {"packed_seq_lengths": seq}, cpu
+ )
+ ref = _old_packed_info(seq, cpu)
+ _same(lengths, ref[0])
+ _same(cu, ref[1])
+ assert max_seqlen == ref[2] and type(max_seqlen) is int
+ info = (lengths, cu, max_seqlen)
+ total = int(seq.sum()) + [0, 1, 7, 64][i % 4]
+ new_cu, new_max = packing.cover_padded_cu_seqlens(info, total)
+ old_cu, old_max = _old_cover(ref, total)
+ _same(new_cu, old_cu)
+ assert new_max == old_max
+ segs = packing.packed_segment_lengths(info, total)
+ assert segs == packing._with_padding_segment(tuple(int(x) for x in ref[0].tolist()), total)
+ if seq.numel() <= 8 and int(seq.sum()) <= 64:
+ mask = packing.build_sdpa_packed_attention_mask(
+ info, dtype = torch.float32, device = cpu, total_tokens = total
+ )
+ packing.clear_packed_caches()
+ # a seq_info not built by get_packed_info_from_kwargs takes the old .tolist() route
+ old_mask = packing.build_sdpa_packed_attention_mask(
+ (ref[0].clone(), ref[1].clone(), ref[2]),
+ dtype = torch.float32,
+ device = cpu,
+ total_tokens = total,
+ )
+ _same(mask, old_mask)
+
+
+def test_host_lengths_only_reused_for_the_cached_tensors():
+ cpu = torch.device("cpu")
+ seq = torch.tensor([3, 5, 2], dtype = torch.int32)
+ lengths, cu, _ = packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, cpu)
+ assert packing._cached_host_lengths(lengths) == (3, 5, 2)
+ assert packing._cached_host_lengths(lengths, cu) == (3, 5, 2)
+ assert packing._cached_host_lengths(lengths.clone()) is None
+ assert packing._cached_host_lengths(lengths, cu.clone()) is None
+ # a foreign seq_info whose cu_seqlens is not the cumsum of these lengths keeps reading cu_seqlens[-1]
+ foreign_cu = torch.tensor([0, 4, 6, 9], dtype = torch.int32)
+ new_cu, new_max = packing.cover_padded_cu_seqlens((lengths, foreign_cu, 5), 12)
+ assert new_cu.tolist() == [0, 4, 6, 9, 12] and new_max == 5
+ packing.clear_packed_caches()
+ assert packing._cached_host_lengths(lengths) is None
+
+
+def test_position_id_inferred_lengths_match():
+ cpu = torch.device("cpu")
+ pos = torch.cat([torch.arange(n) for n in (4, 1, 9, 3)])[None]
+ lengths, cu, max_seqlen = packing.get_packed_info_from_kwargs({}, cpu, pos)
+ assert lengths.tolist() == [4, 1, 9, 3] and cu.tolist() == [0, 4, 5, 14, 17] and max_seqlen == 9
+
+
+def test_fsdp2_clip_empty_shards_match_reference():
+ from unsloth.models.fsdp2_native import native_clip_grad_norm_
+ g = torch.Generator().manual_seed(0)
+ for p_norm in (2.0, 1.0, math.inf, -math.inf, 3.0):
+ for dtype in (torch.float32, torch.float16, torch.bfloat16):
+ shapes = [(3, 4), (0,), (5,), (0, 4), (7, 2), (0,)]
+ params = []
+ for s in shapes:
+ p = torch.nn.Parameter(torch.zeros(s, dtype = dtype))
+ p.grad = (torch.randn(s, generator = g) * 3).to(dtype)
+ params.append(p)
+ # the previous implementation: identity-filled vector, live norms scattered in by index
+ grads = [p.grad for p in params]
+ live = [k for k, t in enumerate(grads) if t.numel()]
+ values = torch.full((len(grads),), math.inf if p_norm < 0 else 0.0, dtype = dtype)
+ values[torch.tensor(live)] = torch.stack(
+ list(torch._foreach_norm([grads[k] for k in live], p_norm))
+ )
+ expect = torch.linalg.vector_norm(torch.cat([values]), p_norm)
+ got = native_clip_grad_norm_([p for p in params], max_norm = 1e9, norm_type = p_norm)
+ assert torch.equal(got.to(expect.dtype), expect), (p_norm, dtype, got, expect)
+
+
+def test_fused_ce_int_divisor_is_bitwise_torch_tensor():
+ for v in (0, 1, 7, 2**24 + 1, 2**31 + 12345, 2**40 + 999, 3.5, True):
+ a = torch.tensor(v, dtype = torch.float32)
+ b = torch.full((), v, dtype = torch.float32)
+ assert a.shape == b.shape and torch.equal(a.view(torch.int32), b.view(torch.int32))
+
+
+_needs_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")
+
+
+def _count_syncs(fn):
+ torch.cuda.synchronize()
+ with warnings.catch_warnings(record = True) as w:
+ warnings.simplefilter("always")
+ torch.cuda.set_sync_debug_mode("warn")
+ try:
+ out = fn()
+ finally:
+ torch.cuda.set_sync_debug_mode("default")
+ return out, sum("synchronizing CUDA operation" in str(x.message) for x in w)
+
+
+@_needs_cuda
+def test_cuda_host_lengths_zero_syncs_and_one_read_for_device_lengths():
+ dev = torch.device("cuda")
+ seq = torch.tensor([5, 1, 70, 7], dtype = torch.int32)
+
+ def per_step(s):
+ info = packing.get_packed_info_from_kwargs({"packed_seq_lengths": s}, dev)
+ packing.cover_padded_cu_seqlens(info, 96)
+ packing.packed_segment_lengths(info, 96)
+ packing.build_sdpa_packed_attention_mask(
+ info, dtype = torch.float16, device = dev, total_tokens = 96
+ )
+ for _ in range(4): # later layers hit the caches
+ packing.get_packed_info_from_kwargs({"packed_seq_lengths": s}, dev)
+ packing.cover_padded_cu_seqlens(info, 96)
+ return info
+
+ info, n = _count_syncs(lambda: per_step(seq))
+ assert n == 0, f"{n} syncs for host lengths"
+ ref = _old_packed_info(seq, dev)
+ _same(info[0], ref[0])
+ _same(info[1], ref[1])
+ assert info[2] == ref[2]
+ packing.clear_packed_caches()
+ dev_seq = seq.to(dev)
+ _, n = _count_syncs(lambda: per_step(dev_seq))
+ assert n == 1, f"{n} syncs for device lengths (expected the one lengths read)"
+
+
+@_needs_cuda
+def test_cuda_fsdp2_clip_and_ce_divisor_no_sync():
+ from unsloth.models.fsdp2_native import native_clip_grad_norm_
+
+ params = []
+ for s in [(3, 4), (0,), (5,)]:
+ p = torch.nn.Parameter(torch.zeros(s, device = "cuda"))
+ p.grad = torch.randn(s, device = "cuda")
+ params.append(p)
+ _, n = _count_syncs(lambda: native_clip_grad_norm_(params, max_norm = 1.0))
+ assert n == 0, f"{n} syncs in native_clip_grad_norm_"
+ _, n = _count_syncs(lambda: torch.full((), 1234, dtype = torch.float32, device = "cuda"))
+ assert n == 0
+
+
+def test_non_blocking_batches_only_for_pinned_cuda_loaders(monkeypatch):
+ from types import SimpleNamespace
+
+ from unsloth.models._utils import _unsloth_non_blocking_batches
+
+ def trainer(
+ pin = True,
+ device = "cuda",
+ non_blocking = False,
+ ):
+ config = SimpleNamespace(non_blocking = non_blocking)
+ return SimpleNamespace(
+ args = SimpleNamespace(dataloader_pin_memory = pin),
+ accelerator = SimpleNamespace(dataloader_config = config, device = torch.device(device)),
+ ), config
+
+ t, config = trainer()
+ assert _unsloth_non_blocking_batches(t) is True and config.non_blocking is True
+ for kwargs in ({"pin": False}, {"device": "cpu"}, {"non_blocking": True}):
+ t, config = trainer(**kwargs)
+ before = config.non_blocking
+ assert _unsloth_non_blocking_batches(t) is False and config.non_blocking == before
+ monkeypatch.setenv("UNSLOTH_NON_BLOCKING_BATCHES", "0")
+ t, config = trainer()
+ assert _unsloth_non_blocking_batches(t) is False and config.non_blocking is False
+ assert _unsloth_non_blocking_batches(SimpleNamespace()) is False
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 855dfc61688..11f2f824dd5 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -407,6 +407,29 @@ def _unsloth_dataset_column_names(dataset):
return list(row.keys()) if hasattr(row, "keys") else []
+def _unsloth_non_blocking_batches(self):
+ """Copy pinned batches with non_blocking=True; accelerate's blocking .to() syncs once per batch
+ tensor. Stream-ordered, so numerics are unchanged. UNSLOTH_NON_BLOCKING_BATCHES=0 opts out."""
+ import os
+
+ if os.environ.get("UNSLOTH_NON_BLOCKING_BATCHES", "1") == "0":
+ return False
+ args = getattr(self, "args", None)
+ accelerator = getattr(self, "accelerator", None)
+ config = getattr(accelerator, "dataloader_config", None)
+ if args is None or config is None or not hasattr(config, "non_blocking"):
+ return False
+ if not getattr(args, "dataloader_pin_memory", False) or config.non_blocking:
+ return False
+ try:
+ if accelerator.device.type != "cuda":
+ return False
+ except Exception:
+ return False
+ config.non_blocking = True
+ return True
+
+
def _unsloth_reset_stray_compile_cache(self):
# A manual forward/backward under torch.compile BEFORE trainer.train() caches a forward plus AOTAutograd backward graph in a one-off context, and reusing it poisons training with NaN/zero gradients, so drop the cache. Module-level, so the SFT auto-packing wrapper and the plain-Trainer loop can run it too.
import os
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 11266a12593..de94b144e33 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -1645,11 +1645,12 @@ def native_clip_grad_norm_(
else:
# complex grads give real norms; every rank must gather the same dtype.
norm_dtype = dtype.to_real() if dtype.is_complex else dtype
- values = torch.full(
- (len(idx),), math.inf if p < 0 else 0.0, device = device, dtype = norm_dtype
- )
- if live:
- values[torch.tensor(live, device = device)] = torch.stack(norms).to(device)
+ # Host-side interleave: torch.tensor(live, device = cuda) was a blocking H2D copy per step.
+ fill = torch.full((), math.inf if p < 0 else 0.0, device = device, dtype = norm_dtype)
+ slots = [fill] * len(idx)
+ for k, n in zip(live, norms):
+ slots[k] = n.to(device = device, dtype = norm_dtype)
+ values = torch.stack(slots)
chunks.append(values)
return chunks
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 1cc5261a7c7..4b5206dee99 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -4295,6 +4295,10 @@ class FastLlamaModel:
" try:\n"
" from unsloth.models._utils import _unsloth_reset_stray_compile_cache as _unsloth_reset_cc\n"
" _unsloth_reset_cc(self)\n"
+ " except Exception: pass\n"
+ " try:\n"
+ " from unsloth.models._utils import _unsloth_non_blocking_batches\n"
+ " _unsloth_non_blocking_batches(self)\n"
" except Exception: pass",
1,
)
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index cff4f03fc33..47820953eb4 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2239,7 +2239,12 @@ def _unsloth_average_gradients(ddp, bucket_bytes = 64 << 20):
if world_size <= 1 or len(params) == 0:
return
device = params[0].device
- present = torch.tensor([p.grad is not None for p in params], dtype = torch.uint8, device = device)
+ present = torch.tensor([p.grad is not None for p in params], dtype = torch.uint8, device = "cpu")
+ # torch.tensor(..., device = cuda) is a pageable, stream-draining copy; pinned + non_blocking is not.
+ if device.type == "cuda":
+ present = present.pin_memory().to(device, non_blocking = True)
+ else:
+ present = present.to(device)
dist.all_reduce(present, op = dist.ReduceOp.MAX, group = group)
buckets = {}
for p, has_grad in zip(params, present.tolist()):
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index 3715883e3b8..e5c09ba139d 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -53,6 +53,9 @@ _XFORMERS_MASK_CACHE: OrderedDict[Tuple[torch.device, Tuple[int, ...], int, bool
# Cache per device for get_packed_info_from_kwargs to avoid repeated D2H sync across layers
_PACKED_INFO_CACHE: dict = {}
+# Per device: (lengths, cu_seqlens, host tuple) from the one D2H read in get_packed_info_from_kwargs
+_PACKED_HOST_LENGTHS_CACHE: dict = {}
+
# Cache per device for build_sdpa_packed_attention_mask to avoid repeated D2H sync across layers
_SDPA_MASK_CACHE: dict = {}
@@ -1677,6 +1680,16 @@ def _wrap_generate_clears_varlen(model, hybrid_modules) -> None:
model._unsloth_varlen_generate_wrapped = True
+def _stage_packed_metadata(seq_lengths: torch.Tensor) -> Tuple[torch.Tensor, Tuple[int, ...]]:
+ """Host int32 [2n+1] = lengths ++ cu_seqlens (same cumsum as the device path), plus the lengths as ints."""
+ host_lengths = seq_lengths.reshape(-1).to(torch.int32)
+ n = host_lengths.numel()
+ staged = torch.zeros(2 * n + 1, dtype = torch.int32, device = "cpu")
+ staged[:n] = host_lengths
+ torch.cumsum(host_lengths, dim = 0, dtype = torch.int32, out = staged[n + 1 :])
+ return staged, tuple(host_lengths.tolist())
+
+
def get_packed_info_from_kwargs(
kwargs: dict,
device: torch.device,
@@ -1699,16 +1712,48 @@ def get_packed_info_from_kwargs(
if entry is not None and entry["seq_lengths"] is seq_lengths:
return entry["result"]
- lengths = seq_lengths.to(device = device, dtype = torch.int32, non_blocking = True)
- cu_seqlens = torch.zeros(lengths.numel() + 1, dtype = torch.int32, device = device)
- torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])
+ if (
+ isinstance(seq_lengths, torch.Tensor)
+ and seq_lengths.device.type == "cpu"
+ and torch.device(device).type == "cuda"
+ ):
+ # One pinned non_blocking H2D copy; pageable copies and .item() both drain the stream.
+ staged, host = _stage_packed_metadata(seq_lengths)
+ n = len(host)
+ moved = staged.pin_memory().to(device = device, non_blocking = True)
+ lengths, cu_seqlens = moved[:n], moved[n:]
+ else:
+ lengths = seq_lengths.to(device = device, dtype = torch.int32, non_blocking = True)
+ cu_seqlens = torch.zeros(lengths.numel() + 1, dtype = torch.int32, device = device)
+ torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])
+ # The one D2H read per batch.
+ host = tuple(lengths.tolist())
- max_seqlen = int(lengths.max().item())
+ max_seqlen = max(host)
result = (lengths, cu_seqlens, max_seqlen)
_PACKED_INFO_CACHE[device] = {"seq_lengths": seq_lengths, "result": result}
+ _PACKED_HOST_LENGTHS_CACHE[lengths.device] = (lengths, cu_seqlens, host)
return result
+def _cached_host_lengths(seq_lengths, cu_seqlens = None) -> Optional[Tuple[int, ...]]:
+ """Host copy of the lengths get_packed_info_from_kwargs built (no device read), else None.
+ With cu_seqlens, also requires that it is the cumsum built beside those lengths."""
+ entry = _PACKED_HOST_LENGTHS_CACHE.get(getattr(seq_lengths, "device", None))
+ if entry is None or entry[0] is not seq_lengths:
+ return None
+ if cu_seqlens is not None and entry[1] is not cu_seqlens:
+ return None
+ return entry[2]
+
+
+def _host_segment_lengths(seq_lengths) -> Tuple[int, ...]:
+ host = _cached_host_lengths(seq_lengths)
+ if host is not None:
+ return host
+ return tuple(int(length) for length in seq_lengths.tolist())
+
+
def _with_padding_segment(lengths: Tuple[int, ...], total_tokens: Optional[int]) -> Tuple[int, ...]:
# TRL pads the flattened padding-free row to pad_to_multiple_of after the lengths are
# taken; the tail gets its own block, since a row outside every block softmaxes to NaN.
@@ -1730,10 +1775,15 @@ def cover_padded_cu_seqlens(
if entry is not None and entry["cu_seqlens"] is cu_seqlens and entry["total"] == total_tokens:
return entry["result"]
- padding = total_tokens - int(cu_seqlens[-1].item())
+ host = _cached_host_lengths(seq_info[0], cu_seqlens)
+ if host is not None:
+ padding = total_tokens - sum(host)
+ else:
+ padding = total_tokens - int(cu_seqlens[-1].item())
result = (cu_seqlens, max_seqlen)
if padding > 0:
- tail = torch.tensor([total_tokens], dtype = cu_seqlens.dtype, device = device)
+ # new_full is a fill kernel; torch.tensor(..., device = cuda) is a blocking H2D copy.
+ tail = cu_seqlens.new_full((1,), total_tokens)
result = (torch.cat([cu_seqlens, tail]), max(max_seqlen, padding))
_PADDED_CU_SEQLENS_CACHE[device] = {
"cu_seqlens": cu_seqlens,
@@ -1763,10 +1813,12 @@ def build_xformers_block_causal_mask(
if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:
return entry["mask"]
- lengths_tensor = seq_lengths.to("cpu", torch.int32)
- if lengths_tensor.numel() == 0:
+ lengths = _cached_host_lengths(seq_lengths)
+ if lengths is None:
+ lengths_tensor = seq_lengths.to("cpu", torch.int32)
+ lengths = tuple(int(x) for x in lengths_tensor.tolist())
+ if len(lengths) == 0:
return None
- lengths = tuple(int(x) for x in lengths_tensor.tolist())
lengths = _with_padding_segment(lengths, total_tokens)
mask = _get_cached_block_mask(lengths, sliding_window, device, is_causal = is_causal)
@@ -1816,9 +1868,7 @@ def packed_segment_lengths(
entry = _SEGMENT_LENGTHS_CACHE.get(seq_lengths.device)
if entry is not None and entry[0] is seq_lengths and entry[1] == total_tokens:
return entry[2]
- lengths = _with_padding_segment(
- tuple(int(length) for length in seq_lengths.tolist()), total_tokens
- )
+ lengths = _with_padding_segment(_host_segment_lengths(seq_lengths), total_tokens)
_SEGMENT_LENGTHS_CACHE[seq_lengths.device] = (seq_lengths, total_tokens, lengths)
return lengths
@@ -1839,10 +1889,7 @@ def build_sdpa_packed_attention_mask(
if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:
return entry["mask"]
- lengths = _with_padding_segment(
- tuple(int(length) for length in seq_lengths.tolist()),
- total_tokens,
- )
+ lengths = _with_padding_segment(_host_segment_lengths(seq_lengths), total_tokens)
total_tokens = sum(lengths)
mask = torch.full(
(total_tokens, total_tokens),
@@ -1955,6 +2002,7 @@ def clear_packed_caches():
"""Release cached masks/metadata to free device memory."""
_XFORMERS_MASK_CACHE.clear()
_PACKED_INFO_CACHE.clear()
+ _PACKED_HOST_LENGTHS_CACHE.clear()
_SDPA_MASK_CACHE.clear()
_SEGMENT_LENGTHS_CACHE.clear()
_XFORMERS_BLOCK_MASK_CACHE.clear()