Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Remove hidden host-device syncs from per-step training paths (#15133)
Daniel Han50bb320f7dff
diff --git a/tests/utils/test_packing_host_sync.py b/tests/utils/test_packing_host_sync.pynew file mode 100644index 00000000000..b065e2a4813--- /dev/null+++ b/tests/utils/test_packing_host_sync.py@@ -0,0 +1,247 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Host-built packed metadata must match the old device reads bit for bit, without per-step syncs."""++import math+import warnings++import pytest+import torch++import unsloth.utils.packing as packing+++def _old_packed_info(seq_lengths, device):+ lengths = seq_lengths.to(device = device, dtype = torch.int32)+ cu_seqlens = torch.zeros(lengths.numel() + 1, dtype = torch.int32, device = device)+ torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])+ return lengths, cu_seqlens, int(lengths.max().item())+++def _old_cover(seq_info, total_tokens):+ _, cu_seqlens, max_seqlen = seq_info+ padding = total_tokens - int(cu_seqlens[-1].item())+ if padding > 0:+ tail = torch.tensor([total_tokens], dtype = cu_seqlens.dtype, device = cu_seqlens.device)+ return torch.cat([cu_seqlens, tail]), max(max_seqlen, padding)+ return cu_seqlens, max_seqlen+++def _random_batches(n = 200, seed = 0):+ g = torch.Generator().manual_seed(seed)+ for i in range(n):+ k = int(torch.randint(1, 33, (1,), generator = g))+ high = [2, 9, 300, 5000][i % 4]+ dtype = [torch.int32, torch.int64][i % 2]+ yield torch.randint(1, high, (k,), generator = g, dtype = dtype)+++def _same(a, b):+ assert a.dtype == b.dtype and a.shape == b.shape and a.device == b.device+ assert torch.equal(a, b)++[email protected](autouse = True)+def _clear():+ packing.clear_packed_caches()+ yield+ packing.clear_packed_caches()+++def test_stage_packed_metadata_matches_device_cumsum():+ for seq in _random_batches():+ staged, host = packing._stage_packed_metadata(seq)+ lengths, cu, max_seqlen = _old_packed_info(seq, "cpu")+ n = seq.numel()+ assert staged.dtype == torch.int32 and staged.numel() == 2 * n + 1+ _same(staged[:n], lengths)+ _same(staged[n:], cu)+ assert host == tuple(lengths.tolist()) and max(host) == max_seqlen+ assert all(type(x) is int for x in host)+++def test_stage_packed_metadata_stays_on_host_under_a_default_device():+ # torch.set_default_device("cuda") must not move the host staging buffer (it is pinned next).+ seq = torch.tensor([3, 1, 4], dtype = torch.int32)+ torch.set_default_device("meta")+ try:+ staged, host = packing._stage_packed_metadata(seq)+ finally:+ torch.set_default_device(None)+ assert staged.device.type == "cpu" and host == (3, 1, 4)+ assert staged.tolist() == [3, 1, 4, 0, 3, 4, 8]+++def test_packed_info_and_consumers_match_old_values():+ cpu = torch.device("cpu")+ for i, seq in enumerate(_random_batches()):+ packing.clear_packed_caches()+ lengths, cu, max_seqlen = packing.get_packed_info_from_kwargs(+ {"packed_seq_lengths": seq}, cpu+ )+ ref = _old_packed_info(seq, cpu)+ _same(lengths, ref[0])+ _same(cu, ref[1])+ assert max_seqlen == ref[2] and type(max_seqlen) is int+ info = (lengths, cu, max_seqlen)+ total = int(seq.sum()) + [0, 1, 7, 64][i % 4]+ new_cu, new_max = packing.cover_padded_cu_seqlens(info, total)+ old_cu, old_max = _old_cover(ref, total)+ _same(new_cu, old_cu)+ assert new_max == old_max+ segs = packing.packed_segment_lengths(info, total)+ assert segs == packing._with_padding_segment(tuple(int(x) for x in ref[0].tolist()), total)+ if seq.numel() <= 8 and int(seq.sum()) <= 64:+ mask = packing.build_sdpa_packed_attention_mask(+ info, dtype = torch.float32, device = cpu, total_tokens = total+ )+ packing.clear_packed_caches()+ # a seq_info not built by get_packed_info_from_kwargs takes the old .tolist() route+ old_mask = packing.build_sdpa_packed_attention_mask(+ (ref[0].clone(), ref[1].clone(), ref[2]),+ dtype = torch.float32,+ device = cpu,+ total_tokens = total,+ )+ _same(mask, old_mask)+++def test_host_lengths_only_reused_for_the_cached_tensors():+ cpu = torch.device("cpu")+ seq = torch.tensor([3, 5, 2], dtype = torch.int32)+ lengths, cu, _ = packing.get_packed_info_from_kwargs({"packed_seq_lengths": seq}, cpu)+ assert packing._cached_host_lengths(lengths) == (3, 5, 2)+ assert packing._cached_host_lengths(lengths, cu) == (3, 5, 2)+ assert packing._cached_host_lengths(lengths.clone()) is None+ assert packing._cached_host_lengths(lengths, cu.clone()) is None+ # a foreign seq_info whose cu_seqlens is not the cumsum of these lengths keeps reading cu_seqlens[-1]+ foreign_cu = torch.tensor([0, 4, 6, 9], dtype = torch.int32)+ new_cu, new_max = packing.cover_padded_cu_seqlens((lengths, foreign_cu, 5), 12)+ assert new_cu.tolist() == [0, 4, 6, 9, 12] and new_max == 5+ packing.clear_packed_caches()+ assert packing._cached_host_lengths(lengths) is None+++def test_position_id_inferred_lengths_match():+ cpu = torch.device("cpu")+ pos = torch.cat([torch.arange(n) for n in (4, 1, 9, 3)])[None]+ lengths, cu, max_seqlen = packing.get_packed_info_from_kwargs({}, cpu, pos)+ assert lengths.tolist() == [4, 1, 9, 3] and cu.tolist() == [0, 4, 5, 14, 17] and max_seqlen == 9+++def test_fsdp2_clip_empty_shards_match_reference():+ from unsloth.models.fsdp2_native import native_clip_grad_norm_+ g = torch.Generator().manual_seed(0)+ for p_norm in (2.0, 1.0, math.inf, -math.inf, 3.0):+ for dtype in (torch.float32, torch.float16, torch.bfloat16):+ shapes = [(3, 4), (0,), (5,), (0, 4), (7, 2), (0,)]+ params = []+ for s in shapes:+ p = torch.nn.Parameter(torch.zeros(s, dtype = dtype))+ p.grad = (torch.randn(s, generator = g) * 3).to(dtype)+ params.append(p)+ # the previous implementation: identity-filled vector, live norms scattered in by index+ grads = [p.grad for p in params]+ live = [k for k, t in enumerate(grads) if t.numel()]+ values = torch.full((len(grads),), math.inf if p_norm < 0 else 0.0, dtype = dtype)+ values[torch.tensor(live)] = torch.stack(+ list(torch._foreach_norm([grads[k] for k in live], p_norm))+ )+ expect = torch.linalg.vector_norm(torch.cat([values]), p_norm)+ got = native_clip_grad_norm_([p for p in params], max_norm = 1e9, norm_type = p_norm)+ assert torch.equal(got.to(expect.dtype), expect), (p_norm, dtype, got, expect)+++def test_fused_ce_int_divisor_is_bitwise_torch_tensor():+ for v in (0, 1, 7, 2**24 + 1, 2**31 + 12345, 2**40 + 999, 3.5, True):+ a = torch.tensor(v, dtype = torch.float32)+ b = torch.full((), v, dtype = torch.float32)+ assert a.shape == b.shape and torch.equal(a.view(torch.int32), b.view(torch.int32))+++_needs_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")+++def _count_syncs(fn):+ torch.cuda.synchronize()+ with warnings.catch_warnings(record = True) as w:+ warnings.simplefilter("always")+ torch.cuda.set_sync_debug_mode("warn")+ try:+ out = fn()+ finally:+ torch.cuda.set_sync_debug_mode("default")+ return out, sum("synchronizing CUDA operation" in str(x.message) for x in w)+++@_needs_cuda+def test_cuda_host_lengths_zero_syncs_and_one_read_for_device_lengths():+ dev = torch.device("cuda")+ seq = torch.tensor([5, 1, 70, 7], dtype = torch.int32)++ def per_step(s):+ info = packing.get_packed_info_from_kwargs({"packed_seq_lengths": s}, dev)+ packing.cover_padded_cu_seqlens(info, 96)+ packing.packed_segment_lengths(info, 96)+ packing.build_sdpa_packed_attention_mask(+ info, dtype = torch.float16, device = dev, total_tokens = 96+ )+ for _ in range(4): # later layers hit the caches+ packing.get_packed_info_from_kwargs({"packed_seq_lengths": s}, dev)+ packing.cover_padded_cu_seqlens(info, 96)+ return info++ info, n = _count_syncs(lambda: per_step(seq))+ assert n == 0, f"{n} syncs for host lengths"+ ref = _old_packed_info(seq, dev)+ _same(info[0], ref[0])+ _same(info[1], ref[1])+ assert info[2] == ref[2]+ packing.clear_packed_caches()+ dev_seq = seq.to(dev)+ _, n = _count_syncs(lambda: per_step(dev_seq))+ assert n == 1, f"{n} syncs for device lengths (expected the one lengths read)"+++@_needs_cuda+def test_cuda_fsdp2_clip_and_ce_divisor_no_sync():+ from unsloth.models.fsdp2_native import native_clip_grad_norm_++ params = []+ for s in [(3, 4), (0,), (5,)]:+ p = torch.nn.Parameter(torch.zeros(s, device = "cuda"))+ p.grad = torch.randn(s, device = "cuda")+ params.append(p)+ _, n = _count_syncs(lambda: native_clip_grad_norm_(params, max_norm = 1.0))+ assert n == 0, f"{n} syncs in native_clip_grad_norm_"+ _, n = _count_syncs(lambda: torch.full((), 1234, dtype = torch.float32, device = "cuda"))+ assert n == 0+++def test_non_blocking_batches_only_for_pinned_cuda_loaders(monkeypatch):+ from types import SimpleNamespace++ from unsloth.models._utils import _unsloth_non_blocking_batches++ def trainer(+ pin = True,+ device = "cuda",+ non_blocking = False,+ ):+ config = SimpleNamespace(non_blocking = non_blocking)+ return SimpleNamespace(+ args = SimpleNamespace(dataloader_pin_memory = pin),+ accelerator = SimpleNamespace(dataloader_config = config, device = torch.device(device)),+ ), config++ t, config = trainer()+ assert _unsloth_non_blocking_batches(t) is True and config.non_blocking is True+ for kwargs in ({"pin": False}, {"device": "cpu"}, {"non_blocking": True}):+ t, config = trainer(**kwargs)+ before = config.non_blocking+ assert _unsloth_non_blocking_batches(t) is False and config.non_blocking == before+ monkeypatch.setenv("UNSLOTH_NON_BLOCKING_BATCHES", "0")+ t, config = trainer()+ assert _unsloth_non_blocking_batches(t) is False and config.non_blocking is False+ assert _unsloth_non_blocking_batches(SimpleNamespace()) is Falsediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 855dfc61688..11f2f824dd5 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -407,6 +407,29 @@ def _unsloth_dataset_column_names(dataset):return list(row.keys()) if hasattr(row, "keys") else []+def _unsloth_non_blocking_batches(self):+ """Copy pinned batches with non_blocking=True; accelerate's blocking .to() syncs once per batch+ tensor. Stream-ordered, so numerics are unchanged. UNSLOTH_NON_BLOCKING_BATCHES=0 opts out."""+ import os++ if os.environ.get("UNSLOTH_NON_BLOCKING_BATCHES", "1") == "0":+ return False+ args = getattr(self, "args", None)+ accelerator = getattr(self, "accelerator", None)+ config = getattr(accelerator, "dataloader_config", None)+ if args is None or config is None or not hasattr(config, "non_blocking"):+ return False+ if not getattr(args, "dataloader_pin_memory", False) or config.non_blocking:+ return False+ try:+ if accelerator.device.type != "cuda":+ return False+ except Exception:+ return False+ config.non_blocking = True+ return True++def _unsloth_reset_stray_compile_cache(self):# A manual forward/backward under torch.compile BEFORE trainer.train() caches a forward plus AOTAutograd backward graph in a one-off context, and reusing it poisons training with NaN/zero gradients, so drop the cache. Module-level, so the SFT auto-packing wrapper and the plain-Trainer loop can run it too.import osdiff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 11266a12593..de94b144e33 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -1645,11 +1645,12 @@ def native_clip_grad_norm_(else:# complex grads give real norms; every rank must gather the same dtype.norm_dtype = dtype.to_real() if dtype.is_complex else dtype- values = torch.full(- (len(idx),), math.inf if p < 0 else 0.0, device = device, dtype = norm_dtype- )- if live:- values[torch.tensor(live, device = device)] = torch.stack(norms).to(device)+ # Host-side interleave: torch.tensor(live, device = cuda) was a blocking H2D copy per step.+ fill = torch.full((), math.inf if p < 0 else 0.0, device = device, dtype = norm_dtype)+ slots = [fill] * len(idx)+ for k, n in zip(live, norms):+ slots[k] = n.to(device = device, dtype = norm_dtype)+ values = torch.stack(slots)chunks.append(values)return chunksdiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 1cc5261a7c7..4b5206dee99 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -4295,6 +4295,10 @@ class FastLlamaModel:" try:\n"" from unsloth.models._utils import _unsloth_reset_stray_compile_cache as _unsloth_reset_cc\n"" _unsloth_reset_cc(self)\n"+ " except Exception: pass\n"+ " try:\n"+ " from unsloth.models._utils import _unsloth_non_blocking_batches\n"+ " _unsloth_non_blocking_batches(self)\n"" except Exception: pass",1,)diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex cff4f03fc33..47820953eb4 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2239,7 +2239,12 @@ def _unsloth_average_gradients(ddp, bucket_bytes = 64 << 20):if world_size <= 1 or len(params) == 0:returndevice = params[0].device- present = torch.tensor([p.grad is not None for p in params], dtype = torch.uint8, device = device)+ present = torch.tensor([p.grad is not None for p in params], dtype = torch.uint8, device = "cpu")+ # torch.tensor(..., device = cuda) is a pageable, stream-draining copy; pinned + non_blocking is not.+ if device.type == "cuda":+ present = present.pin_memory().to(device, non_blocking = True)+ else:+ present = present.to(device)dist.all_reduce(present, op = dist.ReduceOp.MAX, group = group)buckets = {}for p, has_grad in zip(params, present.tolist()):diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex 3715883e3b8..e5c09ba139d 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -53,6 +53,9 @@ _XFORMERS_MASK_CACHE: OrderedDict[Tuple[torch.device, Tuple[int, ...], int, bool# Cache per device for get_packed_info_from_kwargs to avoid repeated D2H sync across layers_PACKED_INFO_CACHE: dict = {}+# Per device: (lengths, cu_seqlens, host tuple) from the one D2H read in get_packed_info_from_kwargs+_PACKED_HOST_LENGTHS_CACHE: dict = {}+# Cache per device for build_sdpa_packed_attention_mask to avoid repeated D2H sync across layers_SDPA_MASK_CACHE: dict = {}@@ -1677,6 +1680,16 @@ def _wrap_generate_clears_varlen(model, hybrid_modules) -> None:model._unsloth_varlen_generate_wrapped = True+def _stage_packed_metadata(seq_lengths: torch.Tensor) -> Tuple[torch.Tensor, Tuple[int, ...]]:+ """Host int32 [2n+1] = lengths ++ cu_seqlens (same cumsum as the device path), plus the lengths as ints."""+ host_lengths = seq_lengths.reshape(-1).to(torch.int32)+ n = host_lengths.numel()+ staged = torch.zeros(2 * n + 1, dtype = torch.int32, device = "cpu")+ staged[:n] = host_lengths+ torch.cumsum(host_lengths, dim = 0, dtype = torch.int32, out = staged[n + 1 :])+ return staged, tuple(host_lengths.tolist())++def get_packed_info_from_kwargs(kwargs: dict,device: torch.device,@@ -1699,16 +1712,48 @@ def get_packed_info_from_kwargs(if entry is not None and entry["seq_lengths"] is seq_lengths:return entry["result"]- lengths = seq_lengths.to(device = device, dtype = torch.int32, non_blocking = True)- cu_seqlens = torch.zeros(lengths.numel() + 1, dtype = torch.int32, device = device)- torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])+ if (+ isinstance(seq_lengths, torch.Tensor)+ and seq_lengths.device.type == "cpu"+ and torch.device(device).type == "cuda"+ ):+ # One pinned non_blocking H2D copy; pageable copies and .item() both drain the stream.+ staged, host = _stage_packed_metadata(seq_lengths)+ n = len(host)+ moved = staged.pin_memory().to(device = device, non_blocking = True)+ lengths, cu_seqlens = moved[:n], moved[n:]+ else:+ lengths = seq_lengths.to(device = device, dtype = torch.int32, non_blocking = True)+ cu_seqlens = torch.zeros(lengths.numel() + 1, dtype = torch.int32, device = device)+ torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])+ # The one D2H read per batch.+ host = tuple(lengths.tolist())- max_seqlen = int(lengths.max().item())+ max_seqlen = max(host)result = (lengths, cu_seqlens, max_seqlen)_PACKED_INFO_CACHE[device] = {"seq_lengths": seq_lengths, "result": result}+ _PACKED_HOST_LENGTHS_CACHE[lengths.device] = (lengths, cu_seqlens, host)return result+def _cached_host_lengths(seq_lengths, cu_seqlens = None) -> Optional[Tuple[int, ...]]:+ """Host copy of the lengths get_packed_info_from_kwargs built (no device read), else None.+ With cu_seqlens, also requires that it is the cumsum built beside those lengths."""+ entry = _PACKED_HOST_LENGTHS_CACHE.get(getattr(seq_lengths, "device", None))+ if entry is None or entry[0] is not seq_lengths:+ return None+ if cu_seqlens is not None and entry[1] is not cu_seqlens:+ return None+ return entry[2]+++def _host_segment_lengths(seq_lengths) -> Tuple[int, ...]:+ host = _cached_host_lengths(seq_lengths)+ if host is not None:+ return host+ return tuple(int(length) for length in seq_lengths.tolist())++def _with_padding_segment(lengths: Tuple[int, ...], total_tokens: Optional[int]) -> Tuple[int, ...]:# TRL pads the flattened padding-free row to pad_to_multiple_of after the lengths are# taken; the tail gets its own block, since a row outside every block softmaxes to NaN.@@ -1730,10 +1775,15 @@ def cover_padded_cu_seqlens(if entry is not None and entry["cu_seqlens"] is cu_seqlens and entry["total"] == total_tokens:return entry["result"]- padding = total_tokens - int(cu_seqlens[-1].item())+ host = _cached_host_lengths(seq_info[0], cu_seqlens)+ if host is not None:+ padding = total_tokens - sum(host)+ else:+ padding = total_tokens - int(cu_seqlens[-1].item())result = (cu_seqlens, max_seqlen)if padding > 0:- tail = torch.tensor([total_tokens], dtype = cu_seqlens.dtype, device = device)+ # new_full is a fill kernel; torch.tensor(..., device = cuda) is a blocking H2D copy.+ tail = cu_seqlens.new_full((1,), total_tokens)result = (torch.cat([cu_seqlens, tail]), max(max_seqlen, padding))_PADDED_CU_SEQLENS_CACHE[device] = {"cu_seqlens": cu_seqlens,@@ -1763,10 +1813,12 @@ def build_xformers_block_causal_mask(if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:return entry["mask"]- lengths_tensor = seq_lengths.to("cpu", torch.int32)- if lengths_tensor.numel() == 0:+ lengths = _cached_host_lengths(seq_lengths)+ if lengths is None:+ lengths_tensor = seq_lengths.to("cpu", torch.int32)+ lengths = tuple(int(x) for x in lengths_tensor.tolist())+ if len(lengths) == 0:return None- lengths = tuple(int(x) for x in lengths_tensor.tolist())lengths = _with_padding_segment(lengths, total_tokens)mask = _get_cached_block_mask(lengths, sliding_window, device, is_causal = is_causal)@@ -1816,9 +1868,7 @@ def packed_segment_lengths(entry = _SEGMENT_LENGTHS_CACHE.get(seq_lengths.device)if entry is not None and entry[0] is seq_lengths and entry[1] == total_tokens:return entry[2]- lengths = _with_padding_segment(- tuple(int(length) for length in seq_lengths.tolist()), total_tokens- )+ lengths = _with_padding_segment(_host_segment_lengths(seq_lengths), total_tokens)_SEGMENT_LENGTHS_CACHE[seq_lengths.device] = (seq_lengths, total_tokens, lengths)return lengths@@ -1839,10 +1889,7 @@ def build_sdpa_packed_attention_mask(if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:return entry["mask"]- lengths = _with_padding_segment(- tuple(int(length) for length in seq_lengths.tolist()),- total_tokens,- )+ lengths = _with_padding_segment(_host_segment_lengths(seq_lengths), total_tokens)total_tokens = sum(lengths)mask = torch.full((total_tokens, total_tokens),@@ -1955,6 +2002,7 @@ def clear_packed_caches():"""Release cached masks/metadata to free device memory."""_XFORMERS_MASK_CACHE.clear()_PACKED_INFO_CACHE.clear()+ _PACKED_HOST_LENGTHS_CACHE.clear()_SDPA_MASK_CACHE.clear()_SEGMENT_LENGTHS_CACHE.clear()_XFORMERS_BLOCK_MASK_CACHE.clear()