Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Diffusion LM training: compiled chunked loss with 0 graph breaks, plus per-family samplers (#13341)
Daniel Han6a70df311a40
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.jsonindex 0e87065b9be..d31bf604ced 100644--- a/scripts/risky_loaders_baseline.json+++ b/scripts/risky_loaders_baseline.json@@ -590,7 +590,7 @@"file": "unsloth/models/diffusion_masked.py","rule": "dynamic-import","sink": "get_class_from_dynamic_module",- "digest": "1900c2baf63f723f",+ "digest": "9fc70fcf03d7ec35","count": 1,"reason": "Falls back to the auto class unless the caller passed trust_remote_code; revision pinned to the config commit hash."},diff --git a/tests/diffusion/test_diffusion_block.py b/tests/diffusion/test_diffusion_block.pyindex 0e0964f87c7..91c7c461e0f 100644--- a/tests/diffusion/test_diffusion_block.py+++ b/tests/diffusion/test_diffusion_block.py@@ -161,6 +161,17 @@ def test_left_padding_rejected():llada2.build_batch(_LM(), inputs, SimpleNamespace())+def test_left_padding_rejected_after_an_unpadded_first_batch():+ # An unpadded first batch proves nothing about the padding side, so the check must keep running.+ llada2, _ = _profiles()+ model = _LM()+ llada2.build_batch(model, _batch([2], [12], 12), SimpleNamespace())+ inputs = _batch([2], [10], 12)+ inputs["attention_mask"] = inputs["attention_mask"].flip(1)+ with pytest.raises(ValueError, match = "right padding"):+ llada2.build_batch(model, inputs, SimpleNamespace())++def _naive_loss(profile, model, clean, noisy, masked, maskable, p, valid, block):from unsloth.models.diffusion_block import block_diffusion_attention_mask@@ -202,7 +213,7 @@ def test_loss_matches_naive_full_forward(which):def test_noisy_block_never_sees_its_own_or_later_clean_tokens():llada2, _ = _profiles()- model, args = _LM(), SimpleNamespace(diffusion_block_size = 4)+ model = _LM()clean = torch.randint(0, 29, (1, 12))noisy = clean.clone()noisy[:, 4:] = MASK_ID@@ -265,6 +276,18 @@ def test_sdar_import_shim_is_scoped():assert "flash_attn" not in sys.modules+def test_sdar_import_shim_provides_sliding_window_cache(monkeypatch):+ import transformers.cache_utils as cache_utils+ from unsloth.models.diffusion_block import _sdar_remote_code_imports++ monkeypatch.delattr(cache_utils, "SlidingWindowCache", raising = False)+ with _sdar_remote_code_imports():+ from transformers.cache_utils import Cache, DynamicCache, SlidingWindowCache+ assert issubclass(SlidingWindowCache, Cache)+ assert not isinstance(DynamicCache(), SlidingWindowCache)+ assert not hasattr(cache_utils, "SlidingWindowCache")++def test_restore_rotary_buffers_refills_zeroed_inv_freq():from unsloth.models.diffusion_profiles import restore_rotary_buffers@@ -300,3 +323,27 @@ def test_unknown_time_weighting_is_rejected():inputs = _batch([3], [20], 20)with pytest.raises(ValueError, match = "diffusion_time_weighting"):llada2.compute_loss(_LM(), inputs, SimpleNamespace(diffusion_time_weighting = "inverse-t"))+++def test_inverse_t_with_zero_noise_floor_stays_finite():+ # diffusion_eps = 0 can draw p = 0 for a row with nothing masked: no 0 / 0 in its weights.+ llada2, sdar = _profiles()+ for profile in (llada2, sdar):+ model = _LM()+ inputs = _batch([2, 3], [12, 10], 12)+ clean, maskable, valid = profile.build_batch(model, inputs, SimpleNamespace())+ p = torch.tensor([0.0, 0.5])+ masked = torch.zeros_like(maskable)+ masked[1, 4] = True+ noisy = torch.where(masked, 0, clean)+ loss = profile.loss_from_noise(+ model,+ clean,+ noisy,+ masked,+ maskable,+ p,+ valid,+ SimpleNamespace(diffusion_time_weighting = "inverse_t"),+ )+ assert torch.isfinite(loss), profile.namediff --git a/tests/diffusion/test_diffusion_compile.py b/tests/diffusion/test_diffusion_compile.pynew file mode 100644index 00000000000..88070459514--- /dev/null+++ b/tests/diffusion/test_diffusion_compile.py@@ -0,0 +1,264 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Chunked diffusion CE: parity with dense cross entropy, full-graph capture, no recompiles, sync-free objectives."""++from types import SimpleNamespace++import pytest+import torch+import torch.nn.functional as F++from unsloth.models import diffusion_loss+from unsloth.models.diffusion_loss import (+ chunked_cross_entropy,+ compile_backbone_layers,+ diffusion_compile_mode,+)+++def _dense(+ hidden,+ targets,+ weights,+ weight = None,+ bias = None,+ softcap = None,+):+ z = hidden if weight is None else F.linear(hidden, weight, bias)+ z = z.float()+ if softcap is not None:+ z = torch.tanh(z / softcap) * softcap+ ce = F.cross_entropy(z.flatten(0, -2), targets.flatten(), reduction = "none", ignore_index = -100)+ return (ce * weights.flatten()).sum(), ce.sum()+++def small_chunks(monkeypatch):+ monkeypatch.setattr(diffusion_loss, "_CHUNK_BYTES", 4 * 11 * 8)+ monkeypatch.setattr(diffusion_loss, "_MIN_ROWS", 4)++[email protected]("project", [True, False])[email protected]("softcap", [None, 3.0])+def test_chunked_ce_matches_dense_values_and_grads(small_chunks, project, softcap):+ torch.manual_seed(0)+ batch, length, width, vocab = 3, 13, 8, 11+ targets = torch.randint(0, vocab, (batch, length))+ targets[0, :4] = -100+ targets[2] = -100 # a fully ignored row+ weights = torch.rand(batch, length) * (targets != -100)+ if project:+ hidden = torch.randn(batch, length, width, requires_grad = True)+ weight = torch.randn(vocab, width, requires_grad = True)+ bias = torch.randn(vocab, requires_grad = True)+ params = [hidden, weight, bias]+ else:+ hidden = torch.randn(batch, length, vocab, requires_grad = True)+ weight = bias = None+ params = [hidden]+ ours, ours_ce, count = chunked_cross_entropy(hidden, targets, weights, weight, bias, softcap)+ ref, ref_ce = _dense(hidden, targets, weights, weight, bias, softcap)+ torch.testing.assert_close(ours, ref, rtol = 1e-5, atol = 1e-6)+ torch.testing.assert_close(ours_ce, ref_ce, rtol = 1e-5, atol = 1e-6)+ assert int(count) == int((targets != -100).sum())+ for a, b in zip(torch.autograd.grad(ours, params), torch.autograd.grad(ref, params)):+ torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)+++def test_chunked_ce_all_ignored_is_zero_with_grad():+ hidden = torch.randn(2, 5, 7, requires_grad = True)+ weighted, total, count = chunked_cross_entropy(hidden, torch.full((2, 5), -100))+ assert float(weighted) == 0.0 and float(total) == 0.0 and int(count) == 0+ (grad,) = torch.autograd.grad(weighted, hidden)+ assert torch.count_nonzero(grad) == 0+++def test_chunked_ce_frozen_head_gets_no_grad():+ hidden = torch.randn(2, 5, 4, requires_grad = True)+ head = torch.nn.Linear(4, 9, bias = False).requires_grad_(False)+ weighted, _, _ = chunked_cross_entropy(hidden, torch.randint(0, 9, (2, 5)), weight = head.weight)+ weighted.backward()+ assert hidden.grad is not None and head.weight.grad is None+++def _kernel_args(+ rows = 16,+ width = 8,+ vocab = 11,+):+ return (+ torch.randn(rows, width),+ torch.randn(vocab, width),+ None,+ torch.randint(0, vocab, (rows,)),+ torch.rand(rows),+ )+++def test_chunk_kernels_capture_without_graph_breaks():+ h, w, b, t, tw = _kernel_args()+ forward = torch._dynamo.explain(diffusion_loss._chunk_forward)(h, w, b, t, tw, 3.0)+ assert forward.graph_break_count == 0 and forward.graph_count == 1+ lse = torch.logsumexp(F.linear(h, w).float(), -1)+ backward = torch._dynamo.explain(diffusion_loss._chunk_backward)(+ h, w, b, t, tw, lse, torch.tensor(1.0), 3.0+ )+ assert backward.graph_break_count == 0 and backward.graph_count == 1+++def _compiled_kernels_or_skip():+ diffusion_loss._COMPILED.clear()+ try:+ forward, _ = diffusion_loss._kernels(True)+ with torch.no_grad(): # as inside the autograd.Function, so the probe shares its guards+ forward(*_kernel_args(), None)+ except Exception as error: # no C++ toolchain for inductor on this host+ pytest.skip(f"torch.compile unavailable here: {type(error).__name__}")+ diffusion_loss._COMPILED.clear()+ torch._dynamo.reset()+++def test_compiled_loss_matches_eager_and_never_recompiles_on_new_masks():+ _compiled_kernels_or_skip()+ torch.manual_seed(0)+ hidden = torch.randn(4, 9, 8, requires_grad = True)+ weight = torch.randn(11, 8, requires_grad = True)+ with torch._dynamo.config.patch(error_on_recompile = True):+ for fraction in (0.1, 0.5, 0.9, 0.0):+ targets = torch.randint(0, 11, (4, 9)).masked_fill(torch.rand(4, 9) > fraction, -100)+ weights = torch.rand(4, 9) * (targets != -100)+ eager, _, _ = chunked_cross_entropy(hidden, targets, weights, weight)+ compiled, _, _ = chunked_cross_entropy(hidden, targets, weights, weight, compiled = True)+ torch.testing.assert_close(compiled, eager, rtol = 1e-5, atol = 1e-6)+ for a, b in zip(+ torch.autograd.grad(compiled, [hidden, weight]),+ torch.autograd.grad(eager, [hidden, weight]),+ ):+ torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)+++def test_compile_mode_parsing(monkeypatch):+ monkeypatch.delenv("UNSLOTH_COMPILE_DISABLE", raising = False)+ assert diffusion_compile_mode(None) == "auto"+ assert diffusion_compile_mode(SimpleNamespace(diffusion_compile = False)) is None+ assert diffusion_compile_mode(SimpleNamespace(diffusion_compile = "eager")) is None+ assert diffusion_compile_mode(SimpleNamespace(diffusion_compile = True)) == "full"+ assert diffusion_compile_mode(SimpleNamespace(diffusion_compile = "loss")) == "loss"+ with pytest.raises(ValueError, match = "diffusion_compile"):+ diffusion_compile_mode(SimpleNamespace(diffusion_compile = "fast"))+ from unsloth.models.diffusion_loss import compile_loss++ auto, loss = (+ SimpleNamespace(diffusion_compile = "auto"),+ SimpleNamespace(diffusion_compile = "loss"),+ )+ assert compile_loss(auto, "cuda") and not compile_loss(auto, "cpu")+ assert not compile_loss(auto, "cuda", auto = False) and compile_loss(loss, "cuda", auto = False)+ monkeypatch.setenv("UNSLOTH_COMPILE_DISABLE", "1")+ assert diffusion_compile_mode(SimpleNamespace(diffusion_compile = True)) is None+++def test_compile_backbone_layers_is_idempotent():+ class _Net(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = torch.nn.ModuleList(torch.nn.Linear(4, 4) for _ in range(3))+ self.head = torch.nn.Linear(4, 4)++ net = _Net()+ assert compile_backbone_layers(net) == 3+ assert compile_backbone_layers(net) == 0+ assert all(getattr(layer, "_unsloth_diffusion_compiled", False) for layer in net.layers)+++def test_chunked_ce_backward_follows_forward_autocast():+ # fp32 weights under bf16 autocast: backward must rebuild the bf16 logits the forward normalised.+ torch.manual_seed(0)+ hidden = torch.randn(2, 6, 8, requires_grad = True)+ weight = torch.randn(13, 8, requires_grad = True)+ targets = torch.randint(0, 13, (2, 6))+ with torch.autocast("cpu", dtype = torch.bfloat16):+ ours, _, _ = chunked_cross_entropy(hidden, targets, weight = weight)+ ref = F.cross_entropy(+ F.linear(hidden, weight).float().flatten(0, 1), targets.flatten(), reduction = "sum"+ )+ torch.testing.assert_close(ours, ref, rtol = 1e-5, atol = 1e-5)+ for a, b in zip(+ torch.autograd.grad(ours, [hidden, weight]), torch.autograd.grad(ref, [hidden, weight])+ ):+ torch.testing.assert_close(a, b, rtol = 1e-3, atol = 1e-3)+++def test_chunked_ce_empty_input_is_zero_with_grad():+ # DiffusionGemma's encoder AR slice of a one-token batch has no rows at all.+ hidden = torch.randn(0, 8, requires_grad = True)+ weight = torch.randn(11, 8, requires_grad = True)+ weighted, total, count = chunked_cross_entropy(+ hidden, torch.empty(0, dtype = torch.long), weight = weight+ )+ weighted.backward()+ assert float(weighted) == 0.0 and int(count) == 0+ assert weight.grad is not None and float(weight.grad.abs().sum()) == 0.0+++def test_chunked_ce_leaves_the_autocast_cache_alone():+ # A later projection of the same weight in the same autocast region must still reach the weight (older+ # torch reused the gradient-less cast made inside the custom forward).+ device = "cuda" if torch.cuda.is_available() else "cpu"+ torch.manual_seed(0)+ weight = torch.randn(11, 8, device = device, requires_grad = True)+ hidden = torch.randn(5, 8, device = device)+ targets = torch.randint(0, 11, (5,), device = device)++ def grads(with_ce):+ weight.grad = None+ with torch.autocast(device, dtype = torch.bfloat16):+ total = F.linear(hidden, weight).float().sum() * 0+ if with_ce:+ total = total + chunked_cross_entropy(hidden, targets, weight = weight)[0]+ total = total + F.linear(hidden, weight).float().sum()+ total.backward()+ return weight.grad.clone()++ ce_only = torch.func.grad(+ lambda w: _dense(hidden, targets, (targets >= 0).float(), weight = w)[0]+ )(weight.detach())+ torch.testing.assert_close(grads(True) - grads(False), ce_only, rtol = 5e-2, atol = 5e-2)+++def test_chunked_ce_ignored_overflow_rows_do_not_poison_grads():+ hidden = torch.tensor([[1.0, 0.0], [float("inf"), 0.0]], requires_grad = True)+ targets = torch.tensor([0, -100])+ weighted, _, _ = chunked_cross_entropy(hidden, targets)+ weighted.backward()+ assert torch.isfinite(hidden.grad).all()+++def test_embedding_learning_rate_gets_its_own_optimizer_group(tmp_path):+ import torch.nn as nn+ from unsloth.models.diffusion_trainer import DiffusionConfig, DiffusionTrainer++ class _Model(nn.Module):+ def __init__(self):+ super().__init__()+ self.embed = nn.Module()+ self.embed.modules_to_save = nn.ModuleDict({"default": nn.Embedding(8, 4)})+ self.lora_A = nn.Linear(4, 4, bias = False)++ trainer = DiffusionTrainer.__new__(DiffusionTrainer)+ trainer.model, trainer.optimizer = _Model(), None+ trainer.args = DiffusionConfig(+ output_dir = str(tmp_path),+ learning_rate = 1e-3,+ embedding_learning_rate = 1e-5,+ report_to = [],+ use_cpu = True,+ bf16 = False,+ fp16 = False,+ )+ groups = trainer.create_optimizer().param_groups+ embedding = trainer.model.embed.modules_to_save["default"].weight+ lrs = {group["lr"] for group in groups if any(p is embedding for p in group["params"])}+ assert lrs == {1e-5}+ assert any(group["lr"] == 1e-3 for group in groups)diff --git a/tests/diffusion/test_diffusion_gemma.py b/tests/diffusion/test_diffusion_gemma.pyindex 85a28393f4e..590360cacd9 100644--- a/tests/diffusion/test_diffusion_gemma.py+++ b/tests/diffusion/test_diffusion_gemma.py@@ -92,6 +92,18 @@ def test_parity_with_trl_reference(model, prediction_type):"from trl.trainer.utils import maybe_gather_lm_head_ctx","import contextlib\nmaybe_gather_lm_head_ctx = lambda *a: contextlib.nullcontext()",)+ if prediction_type == "mean_loo":+ # mean_loo lifts one logit by ~log(K / (1 - alpha)); fp32 F.cross_entropy over the 262k vocab is then+ # ~6e-7 off the fp64 logit gradient (our chunked CE ~1e-8). Score the reference in fp64 instead.+ source = source.replace(+ "import torch.nn.functional as F",+ "import torch.nn.functional as _F\n"+ "class F:\n"+ " def __getattr__(self, name): return getattr(_F, name)\n"+ " @staticmethod\n"+ " def cross_entropy(x, *a, **k): return _F.cross_entropy(x.double(), *a, **k).float()\n"+ "F = F()",+ )namespace = {"__name__": "trl_example"}exec(compile(source, TRL_EXAMPLE, "exec"), namespace)reference = namespace["DiffusionGemmaSFTTrainer"]@@ -123,12 +135,14 @@ def test_parity_with_trl_reference(model, prediction_type):model, batch, _args(diffusion_prediction_type = prediction_type))[0])- torch.testing.assert_close(our_loss, ref_loss, rtol = 0, atol = 1e-6)+ torch.testing.assert_close(our_loss, ref_loss, rtol = 1e-6, atol = 1e-6)for a, b in zip(our_grads, ref_grads):if a is None or b is None:assert a is None and b is None- else:+ elif prediction_type == "mean":torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-7)+ else:+ torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)def test_tied_encoder_decoder_lora_merge_is_refused():@@ -190,6 +204,16 @@ def test_row_without_supervised_tokens_adds_no_canvas_loss(model):assert metrics["diffusion_loss"] == 0.0[email protected]("rows", [[(1, 0)], [(1, 0), (0, 0)]])+def test_batch_without_encoder_ar_targets_is_finite(model, rows):+ # A one-token or all-padding batch has no next-token pair: the AR co-loss is 0, not a NaN mean.+ batch = _batch(model.config.text_config.vocab_size, model.config.canvas_length, rows = rows)+ torch.manual_seed(0)+ loss, _, metrics = DIFFUSION_GEMMA_PROFILE.compute_loss(model, batch, _args())+ assert torch.isfinite(loss) and loss.requires_grad+ assert float(metrics["encoder_ar_loss"]) == 0.0 and float(metrics["diffusion_loss"]) == 0.0++def test_reloaded_adapter_still_refuses_merge(tmp_path):from peft import LoraConfig, get_peft_modeldiff --git a/tests/diffusion/test_diffusion_masked.py b/tests/diffusion/test_diffusion_masked.pyindex 06b8befbdc0..b23b20941db 100644--- a/tests/diffusion/test_diffusion_masked.py+++ b/tests/diffusion/test_diffusion_masked.py@@ -394,6 +394,68 @@ def test_mask_builder_signature_exposed_from_wrapper_closure():assert list(inspect.signature(wrapper).parameters)[:2] == ["config", "inputs_embeds"]+_REMOTE_CONFIG = """+from transformers import PretrainedConfig+++class TinyRemoteConfig(PretrainedConfig):+ model_type = "tiny_remote_diffusion"+"""++_REMOTE_MODELING = """+import torch+from transformers import PreTrainedModel+from .configuration_tiny_remote import TinyRemoteConfig+++class TinyRemoteModel(PreTrainedModel):+ config_class = TinyRemoteConfig++ def __init__(self, config):+ super().__init__(config)+ self.proj = torch.nn.Linear(4, 4, bias = False)+ self.post_init()++ def _init_weights(self, module):+ # 4.x-era hub init writes in place, blind to which weights the checkpoint already filled.+ if isinstance(module, torch.nn.Linear):+ module.weight.data.normal_(0.0, 1.0)++ def forward(self, x):+ return self.proj(x)+"""+++def test_remote_class_keeps_checkpoint_weights(tmp_path):+ import json++ from safetensors.torch import save_file+ from transformers import AutoConfig++ from unsloth.models.diffusion_masked import MaskedDiffusionProfile++ (tmp_path / "configuration_tiny_remote.py").write_text(_REMOTE_CONFIG)+ (tmp_path / "modeling_tiny_remote.py").write_text(_REMOTE_MODELING)+ auto_map = {+ "AutoConfig": "configuration_tiny_remote.TinyRemoteConfig",+ "AutoModelForCausalLM": "modeling_tiny_remote.TinyRemoteModel",+ }+ (tmp_path / "config.json").write_text(+ json.dumps({"model_type": "tiny_remote_diffusion", "auto_map": auto_map})+ )+ weight = torch.arange(16, dtype = torch.float32).reshape(4, 4)+ save_file(+ {"proj.weight": weight}, str(tmp_path / "model.safetensors"), metadata = {"format": "pt"}+ )++ config = AutoConfig.from_pretrained(str(tmp_path), trust_remote_code = True)+ profile = MaskedDiffusionProfile(name = "tiny_remote_test", requires_remote_code = True)+ cls = profile.model_class(config, True)+ model = cls.from_pretrained(str(tmp_path), config = config, trust_remote_code = True)+ assert torch.equal(model.proj.weight.detach(), weight)+ assert cls.is_remote_code()++def test_converted_config_keeps_token_fields(monkeypatch, tmp_path):from transformers import TrainingArgumentsfrom trl import SFTConfig@@ -431,6 +493,18 @@ path, out = sys.argv[1], sys.argv[2]device = os.environ.get("TINY_DEVICE", "cpu")model, processor = FastModel.from_pretrained(path, trust_remote_code = True, dtype = torch.float32, device_map = {"": device})assert getattr(model, "_unsloth_slow_diffusion", False), type(model)+import glob+from safetensors import safe_open+state, matched = model.state_dict(), 0+for shard in glob.glob(os.path.join(path, "*.safetensors")):+ with safe_open(shard, "pt") as f:+ for key in f.keys():+ # A GPU load defaults to 4-bit: packed linear weights cannot equal the checkpoint, the rest must.+ if key not in state or tuple(state[key].shape) != tuple(f.get_slice(key).get_shape()):+ continue+ assert torch.equal(f.get_tensor(key).to(state[key].dtype), state[key].detach().cpu()), key+ matched += 1+assert matched, "no checkpoint tensor matched the model's names"tokenizer = getattr(processor, "tokenizer", processor)for module in model.modules():if hasattr(module, "rope_init_fn") and isinstance(getattr(module, "inv_freq", None), torch.Tensor):@@ -451,7 +525,7 @@ def fixed_noise_ce():# Per-step losses are too noisy over 20 steps (1 / t weight, fresh noise); score one batch under one draw.torch.manual_seed(0)with torch.no_grad():- return trainer.diffusion_profile.compute_loss(model, fixed, args)[2]["masked_ce"]+ return float(trainer.diffusion_profile.compute_loss(model, fixed, args)[2]["masked_ce"])first = fixed_noise_ce()@@ -500,3 +574,48 @@ def test_tiny_remote_checkpoint_trains_and_reloads(name, tmp_path):)assert out.returncode == 0, out.stdout[-3000:] + out.stderr[-3000:]assert "TINY_OK" in out.stdout+++class _WithHead(_Bidirectional):+ def get_output_embeddings(self):+ return self.head++[email protected]("name", ["llada", "dream", "nemotron_labs_diffusion"])+def test_hidden_state_loss_matches_dense_logits(name):+ # The fused path swaps the head for its input and projects per chunk; it must equal the logits path.+ inputs, _ = _batch()+ fused, dense = _WithHead(), _Bidirectional()+ dense.load_state_dict(fused.state_dict())+ fused_loss, _ = _ours(name, fused, inputs, 3)+ dense_loss, _ = _ours(name, dense, inputs, 3)+ assert fused.__dict__["_unsloth_diffusion_fused_head"] == (fused.head, fused.head)+ assert "forward" not in vars(fused.head) # the swap is undone+ torch.testing.assert_close(fused_loss, dense_loss, rtol = 1e-6, atol = 1e-6)+ fused_loss.backward()+ dense_loss.backward()+ for (n, a), b in zip(fused.named_parameters(), dense.parameters()):+ torch.testing.assert_close(a.grad, b.grad, rtol = 1e-5, atol = 1e-6, msg = n)+++def test_dream_zero_noise_level_is_finite(monkeypatch):+ # Dream draws t ~ U(0, 1) with no floor and weights by 1 / t: t = 0 masks nothing and must not give inf * 0.+ inputs, _ = _batch()+ monkeypatch.setattr(+ torch,+ "rand",+ lambda *a, **k: torch.zeros(*a, **{x: y for x, y in k.items() if x != "generator"}),+ )+ loss, metrics = _ours("dream", _Bidirectional(), inputs, 0)+ assert torch.isfinite(loss) and float(loss) == 0.0 and loss.requires_grad+ assert float(metrics["masked_fraction"]) == 0.0+++def test_fused_head_rechecked_after_head_is_replaced():+ # A head wrapped after the first loss call (e.g. PEFT modules_to_save) must not reuse the cached decision.+ from unsloth.models.diffusion_loss import fusable_head++ model = _WithHead()+ assert fusable_head(model, model) is model.head+ model.head = torch.nn.Sequential(model.head)+ assert fusable_head(model, model) is Nonediff --git a/tests/diffusion/test_diffusion_samplers.py b/tests/diffusion/test_diffusion_samplers.pynew file mode 100644index 00000000000..8eef60a0a9d--- /dev/null+++ b/tests/diffusion/test_diffusion_samplers.py@@ -0,0 +1,538 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FastLanguageModel routes text-diffusion models to the diffusion PEFT / mode helpers, and generate_text decodes+with each family's sampler."""++import importlib.util+import os+import subprocess+import sys+from types import SimpleNamespace++import pytest+import torch++# The LLaDA sampler is re-implemented; check it against the official one when its generate.py is available.+_LLADA_GENERATE = os.environ.get("UNSLOTH_LLADA_GENERATE_PY")+_TINY = os.environ.get("UNSLOTH_DIFFUSION_TINY_DIR")++VOCAB, MASK_ID, EOS_ID = 61, 60, 2+++class _TinyMaskPredictor(torch.nn.Module):+ """Bidirectional toy predictor: every logit depends on the whole row."""++ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ self.embed = torch.nn.Embedding(VOCAB, 16)+ self.mix = torch.nn.Linear(16, 16)+ self.head = torch.nn.Linear(16, VOCAB)+ self.config = SimpleNamespace(mask_token_id = MASK_ID, eos_token_id = EOS_ID)+ self.calls = 0++ @property+ def device(self):+ return self.head.weight.device++ def forward(+ self,+ input_ids,+ attention_mask = None,+ ):+ self.calls += 1+ h = self.embed(input_ids)+ h = h + self.mix(h.mean(dim = 1, keepdim = True))+ logits = self.head(torch.tanh(h))+ logits[..., MASK_ID] = -1e4+ return SimpleNamespace(logits = logits)+++def _load_official_generate():+ spec = importlib.util.spec_from_file_location("llada_generate", _LLADA_GENERATE)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module.generate+++ not _LLADA_GENERATE, reason = "set UNSLOTH_LLADA_GENERATE_PY to LLaDA's generate.py"+)[email protected]("temperature", [0.0, 0.7])[email protected]("remasking", ["low_confidence", "random"])+def test_llada_sampler_matches_official(temperature, remasking):+ from unsloth.models.diffusion_samplers import _llada_sample++ official = _load_official_generate()+ model = _TinyMaskPredictor()+ prompt = torch.tensor([[1, 7, 9, 11, 13]])+ torch.manual_seed(1)+ expected = official(+ model,+ prompt,+ steps = 8,+ gen_length = 16,+ block_length = 8,+ temperature = temperature,+ remasking = remasking,+ mask_id = MASK_ID,+ )[:, prompt.shape[1] :]+ torch.manual_seed(1)+ got = _llada_sample(+ model,+ None,+ prompt,+ None,+ max_new_tokens = 16,+ steps = 8,+ block_length = 8,+ temperature = temperature,+ mask_id = MASK_ID,+ eos_ids = [],+ remasking = remasking,+ )+ assert torch.equal(got, expected), (got, expected)+++def test_llada_sampler_stops_after_eos_block():+ from unsloth.models.diffusion_samplers import _llada_sample++ model = _TinyMaskPredictor()+ prompt = torch.tensor([[1, 7, 9]])+ full = _llada_sample(+ model,+ None,+ prompt,+ None,+ max_new_tokens = 32,+ steps = 32,+ block_length = 8,+ temperature = 0.0,+ mask_id = MASK_ID,+ eos_ids = [],+ eos_early_stop = False,+ )+ tokens = full[0].tolist()+ eos = next(t for t in tokens[:8] if t != MASK_ID)+ calls = model.calls+ early = _llada_sample(+ model,+ None,+ prompt,+ None,+ max_new_tokens = 32,+ steps = 32,+ block_length = 8,+ temperature = 0.0,+ mask_id = MASK_ID,+ eos_ids = [eos],+ )+ # The first block already holds the "EOS", so only its 8 steps ran and its tokens are unchanged.+ assert model.calls - calls == 8+ assert torch.equal(early, full[:, :8])+++def test_fast_language_model_routes_diffusion_helpers(monkeypatch):+ from unsloth import FastLanguageModel+ from unsloth.models import diffusion as diffusion_module+ from unsloth.models.llama import FastLlamaModel++ seen = {}+ monkeypatch.setattr(+ diffusion_module.FastDiffusionModel,+ "get_peft_model",+ staticmethod(lambda model, **kwargs: seen.setdefault("peft", kwargs) and model),+ )+ monkeypatch.setattr(+ diffusion_module.FastDiffusionModel,+ "for_training",+ staticmethod(+ lambda model, use_gradient_checkpointing = True: seen.setdefault(+ "train", use_gradient_checkpointing+ )+ or model+ ),+ )+ monkeypatch.setattr(+ diffusion_module.FastDiffusionModel,+ "for_inference",+ staticmethod(lambda model: seen.setdefault("infer", True) and model),+ )+ model = torch.nn.Linear(2, 2)+ model._unsloth_slow_diffusion = True+ model._unsloth_gradient_checkpointing = False+ # Positional r and target_modules keep FastLanguageModel's meaning; omitted ones stay omitted.+ FastLanguageModel.get_peft_model(+ model, 8, ["q_proj"], lora_alpha = 16, use_rslora = True, offload_layers = None+ )+ assert seen["peft"] == {+ "r": 8,+ "target_modules": ["q_proj"],+ "lora_alpha": 16,+ "use_rslora": True,+ "offload_layers": None,+ }+ FastLanguageModel.for_training(model)+ assert seen["train"] is False+ FastLanguageModel.for_inference(model)+ assert seen["infer"] is True+ import inspect++ assert inspect.signature(FastLanguageModel.get_peft_model) == inspect.signature(+ FastLlamaModel.get_peft_model+ )+++def test_fast_language_model_autoregressive_path_unchanged(monkeypatch):+ from unsloth import FastLanguageModel+ from unsloth.models.llama import FastLlamaModel++ calls = []+ monkeypatch.setattr(+ FastLlamaModel,+ "get_peft_model",+ staticmethod(lambda *a, **k: calls.append(("peft", a, k)) or "peft"),+ )+ monkeypatch.setattr(+ FastLlamaModel,+ "for_training",+ staticmethod(lambda *a, **k: calls.append(("train", a, k)) or "train"),+ )+ monkeypatch.setattr(+ FastLlamaModel,+ "for_inference",+ staticmethod(lambda *a, **k: calls.append(("infer", a, k)) or "infer"),+ )+ model = torch.nn.Linear(2, 2)+ assert FastLanguageModel.get_peft_model(model, 4, target_modules = ["x"]) == "peft"+ assert FastLanguageModel.for_training(model, True) == "train"+ assert FastLanguageModel.for_inference(model) == "infer"+ assert calls == [+ ("peft", (model, 4), {"target_modules": ["x"]}),+ ("train", (model, True), {}),+ ("infer", (model,), {}),+ ]++[email protected]("bad", [{"use_dora": True}, {"loftq_config": {"loftq_bits": 4}}])+def test_unsupported_lora_options_refused(bad):+ from unsloth.models.diffusion import FastDiffusionModel+ model = torch.nn.Linear(2, 2)+ with pytest.raises(ValueError, match = "not supported for text-diffusion"):+ FastDiffusionModel.get_peft_model(model, r = 4, **bad)+++def test_diffusion_profile_name_and_flag():+ from unsloth.models.diffusion import is_diffusion_model+ from unsloth.models.diffusion_samplers import diffusion_profile_name++ model = torch.nn.Linear(2, 2)+ assert not is_diffusion_model(model) and not is_diffusion_model(None)+ # Autoregressive Nemotron shares no profile; Nemotron-Labs-Diffusion matches by architecture.+ model.config = SimpleNamespace(model_type = "nemotron", architectures = ["NemotronForCausalLM"])+ assert not is_diffusion_model(model)+ model.config = SimpleNamespace(+ model_type = "nemotron", architectures = ["NemotronLabsDiffusionModel"]+ )+ assert is_diffusion_model(model)+ assert diffusion_profile_name(model) == "nemotron_labs_diffusion"+ del model.config+ model._unsloth_slow_diffusion = True+ model._unsloth_diffusion_profile = "llada"+ assert is_diffusion_model(model)+ assert diffusion_profile_name(model) == "llada"+++def test_generate_text_cuts_at_eos_and_budget(monkeypatch):+ from unsloth.models import diffusion_samplers++ class _Tok:+ eos_token_id = EOS_ID+ mask_token_id = None++ def __call__(+ self,+ text,+ return_tensors = None,+ add_special_tokens = True,+ ):+ return {+ "input_ids": torch.tensor([[1, 7]]),+ "attention_mask": torch.ones(1, 2, dtype = torch.long),+ }++ def decode(+ self,+ tokens,+ skip_special_tokens = True,+ ):+ return " ".join(map(str, tokens))++ model = _TinyMaskPredictor()+ model._unsloth_diffusion_profile = "fake"+ produced = torch.tensor([[11, MASK_ID, 12, 13, EOS_ID, 14, 15]])+ seen = {}++ def fake(base, tokenizer, input_ids, attention_mask, **kwargs):+ seen.update(kwargs)+ return produced++ monkeypatch.setitem(diffusion_samplers._SAMPLERS, "fake", fake)+ text = diffusion_samplers.generate_text(+ model, _Tok(), prompt = "hi", max_new_tokens = 32, steps = 4, alg = "x"+ )+ assert text == "11 12 13"+ assert (+ seen["mask_id"] == MASK_ID+ and seen["eos_ids"] == [EOS_ID]+ and seen["alg"] == "x"+ and seen["steps"] == 4+ )+ # max_new_tokens caps what is decoded even when a sampler returns more.+ assert diffusion_samplers.generate_text(model, _Tok(), prompt = "hi", max_new_tokens = 2) == "11"++ # Dream: eos_token_id is <|endoftext|> only, while its chat template closes a turn with <|im_end|>.+ turn_id = 99++ class _ChatTok(_Tok):+ all_special_ids = [EOS_ID, turn_id]++ def apply_chat_template(+ self,+ messages,+ add_generation_prompt = False,+ tokenize = False,+ ):+ turns = "".join(f"<{m['role']}>{m['content']}<turn>" for m in messages)+ return turns + ("<assistant>" if add_generation_prompt else "")++ def __call__(+ self,+ text,+ return_tensors = None,+ add_special_tokens = True,+ ):+ if return_tensors is None:+ return {"input_ids": [turn_id, 5] if text.startswith("<turn>") else [5]}+ return super().__call__(text, return_tensors, add_special_tokens)++ produced = torch.tensor([[11, 12, turn_id, 14, 15]])+ chat = [{"role": "user", "content": "hi"}]+ assert diffusion_samplers.generate_text(model, _ChatTok(), messages = chat) == "11 12"+ assert seen["eos_ids"] == [EOS_ID, turn_id]+ assert diffusion_samplers.generate_text(model, _ChatTok(), prompt = "hi") == "11 12 99 14 15"+ assert seen["eos_ids"] == [EOS_ID]+ with pytest.raises(ValueError, match = "no diffusion sampler"):+ model._unsloth_diffusion_profile = "missing"+ diffusion_samplers.generate_text(model, _Tok(), prompt = "hi")+++_PAD_SCRIPT = r"""+import os, sys+import torch+from unsloth import FastModel++path = sys.argv[1]+model, processor = FastModel.from_pretrained(path, trust_remote_code = True, dtype = torch.float32, device_map = {"": os.environ.get("TINY_DEVICE", "cpu")}, load_in_4bit = False)+tok = getattr(processor, "tokenizer", processor)+assert tok.pad_token is not None and tok.pad_token == tok.eos_token, (tok.pad_token, tok.eos_token)+assert tok.pad_token_id != getattr(model, "_unsloth_mask_token_id", None)+assert model.config.pad_token_id == tok.pad_token_id+assert getattr(model.generation_config, "pad_token_id", tok.pad_token_id) == tok.pad_token_id+batch = tok.pad([{"input_ids": [1, 2, 3]}, {"input_ids": [4]}], return_tensors = "pt")+assert batch["input_ids"].shape == (2, 3)+print("PAD_OK", tok.pad_token)+"""+++def test_tiny_nemotron_gets_eos_pad_token():+ path = os.path.join(_TINY or "", "Nemotron-Labs-Diffusion-3B")+ if not _TINY or not os.path.isdir(path):+ pytest.skip("set UNSLOTH_DIFFUSION_TINY_DIR to tiny remote-code copies")+ out = subprocess.run(+ [sys.executable, "-c", _PAD_SCRIPT, path], capture_output = True, text = True, timeout = 900+ )+ assert out.returncode == 0, out.stdout[-3000:] + out.stderr[-4000:]+ assert "PAD_OK" in out.stdout+++_TINY_SCRIPT = r"""+import os, sys+import torch+from unsloth import FastLanguageModel+from unsloth.models.diffusion import FastDiffusionModel, is_diffusion_model+from unsloth.models.diffusion_samplers import DiffusionGenerationCancelled++path = sys.argv[1]+device = os.environ.get("TINY_DEVICE", "cpu")+model, processor = FastLanguageModel.from_pretrained(path, trust_remote_code = True, dtype = torch.float32, device_map = {"": device}, load_in_4bit = False)+assert is_diffusion_model(model), type(model)+# Studio's text path: FastLanguageModel.get_peft_model with its usual knobs.+model = FastLanguageModel.get_peft_model(model, r = 8, lora_alpha = 16, lora_dropout = 0.0, bias = "none", use_gradient_checkpointing = True, random_state = 3407, use_rslora = True, loftq_config = None, use_dora = False)+config = model.peft_config["default"]+assert config.use_rslora is True+assert sum(p.numel() for n, p in model.named_parameters() if p.requires_grad and "lora_" in n) > 0+++def checkpointing_on(m):+ return any(getattr(x, "gradient_checkpointing", False) is True or getattr(x, "activation_checkpointing_strategy", None) is not None for x in m.modules())+++assert checkpointing_on(model), "get_peft_model(use_gradient_checkpointing = True) did not enable it"+FastLanguageModel.for_training(model, use_gradient_checkpointing = False)+assert model.training and not checkpointing_on(model)+FastLanguageModel.for_training(model, use_gradient_checkpointing = True)+assert checkpointing_on(model)+FastLanguageModel.for_inference(model)+assert not model.training and not checkpointing_on(model)++messages = [{"role": "user", "content": "What is 2 + 2?"}]+first = FastDiffusionModel.generate_text(model, processor, messages = messages, max_new_tokens = 16, seed = 0)+second = FastDiffusionModel.generate_text(model, processor, messages = messages, max_new_tokens = 16, seed = 0)+assert isinstance(first, str) and first == second, (first, second)+plain = FastDiffusionModel.generate_text(model, processor, prompt = "Hello", max_new_tokens = 8, seed = 0)+assert isinstance(plain, str)++calls = {"n": 0}+++def stop():+ calls["n"] += 1+ return calls["n"] > 1+++try:+ FastDiffusionModel.generate_text(model, processor, messages = messages, max_new_tokens = 64, stop_callback = stop, seed = 0)+ raise AssertionError("stop_callback did not cancel")+except DiffusionGenerationCancelled:+ pass+assert calls["n"] == 2, calls+assert not model.training+print("SAMPLER_OK", repr(first[:40]))+"""++_FAMILIES = [+ "LLaDA-8B-Instruct",+ "LLaDA-MoE-7B-A1B-Instruct",+ "Dream-v0-Instruct-7B",+ "Nemotron-Labs-Diffusion-3B",+ "LLaDA2-mini",+ "SDAR-1.7B-Chat",+ "trl-internal-testing/tiny-DiffusionGemmaForBlockDiffusion",+]++[email protected]("name", _FAMILIES)+def test_tiny_family_routes_and_generates(name):+ if "/" in name:+ path = name+ elif _TINY and os.path.isdir(os.path.join(_TINY, name)):+ path = os.path.join(_TINY, name)+ else:+ pytest.skip("set UNSLOTH_DIFFUSION_TINY_DIR to tiny remote-code copies")+ # One process per family: LLaDA's remote config registers "llada" globally, shadowing LLaDA-MoE's.+ from real_accelerator import has_real_accelerator++ # The parent's CUDA spoof does not reach the child; the sampler only needs inference on CPU.+ env = os.environ if has_real_accelerator() else {**os.environ, "UNSLOTH_ALLOW_CPU": "1"}+ out = subprocess.run(+ [sys.executable, "-c", _TINY_SCRIPT, path],+ capture_output = True,+ text = True,+ timeout = 900,+ env = env,+ )+ assert out.returncode == 0, out.stdout[-3000:] + out.stderr[-4000:]+ assert "SAMPLER_OK" in out.stdout+++def test_diffusion_lora_init_follows_random_state():+ import torch.nn as nn++ from unsloth.models.diffusion import FastDiffusionModel++ class Toy(nn.Module):+ def __init__(self):+ super().__init__()+ self.config = SimpleNamespace(model_type = "toy", to_dict = lambda: {})+ self.q_proj = nn.Linear(8, 8)++ def forward(self, x):+ return self.q_proj(x)++ def lora_a(seed):+ torch.manual_seed(seed + 1) # perturb the global RNG: only random_state may decide the init+ model = FastDiffusionModel.get_peft_model(+ Toy(),+ r = 4,+ target_modules = ["q_proj"],+ use_gradient_checkpointing = False,+ random_state = 3407,+ )+ return next(p for n, p in model.named_parameters() if "lora_A" in n).detach().clone()++ assert torch.equal(lora_a(0), lora_a(1))+++def test_get_peft_model_keeps_an_existing_adapter():+ # Continued training from a saved adapter must not re-inject a fresh one over the learned weights.+ import torch.nn as nn++ from unsloth.models.diffusion import FastDiffusionModel++ class Toy(nn.Module):+ def __init__(self):+ super().__init__()+ self.config = SimpleNamespace(model_type = "toy", to_dict = lambda: {})+ self.q_proj = nn.Linear(8, 8)++ model = FastDiffusionModel.get_peft_model(+ Toy(), r = 4, target_modules = ["q_proj"], use_gradient_checkpointing = False+ )+ lora_b = next(p for n, p in model.named_parameters() if "lora_B" in n)+ with torch.no_grad():+ lora_b.fill_(0.75)+ again = FastDiffusionModel.get_peft_model(+ model, r = 4, target_modules = ["q_proj"], use_gradient_checkpointing = False+ )+ assert again is model+ kept = next(p for n, p in again.named_parameters() if "lora_B" in n)+ assert torch.all(kept == 0.75) and kept.requires_grad+ layer = next(m for m in model.modules() if hasattr(m, "lora_A"))+ layer.gradient_checkpointing = True+ FastDiffusionModel.get_peft_model(model, r = 4, use_gradient_checkpointing = False)+ assert layer.gradient_checkpointing is False+ assert model._unsloth_gradient_checkpointing is False+++def test_cancellation_hooks_every_embedding_and_restores_attention_modes():+ import torch.nn as nn++ from unsloth.models import diffusion_samplers++ class Attn(nn.Module):+ def __init__(self):+ super().__init__()+ self.diffusion_lm = True++ class EncDec(nn.Module):+ def __init__(self):+ super().__init__()+ self.encoder_embed = nn.Embedding(4, 2)+ self.decoder_embed = nn.Embedding(4, 2)+ self.attn = Attn()++ base = EncDec()+ calls = []+ with pytest.raises(diffusion_samplers.DiffusionGenerationCancelled):+ with (+ diffusion_samplers._restore_attention_modes(base),+ diffusion_samplers._cancellation(base, lambda: calls.append(1) or len(calls) > 1),+ ):+ base.encoder_embed(torch.tensor([0]))+ base.attn.diffusion_lm = False # generate's causal prefill+ base.decoder_embed(torch.tensor([0])) # the decoder's own table also checks+ assert len(calls) == 2+ assert base.attn.diffusion_lm is True+ assert not base.encoder_embed._forward_pre_hooks and not base.decoder_embed._forward_pre_hooksdiff --git a/unsloth/models/diffusion.py b/unsloth/models/diffusion.pyindex e7e51ccd817..29e0fefc58d 100644--- a/unsloth/models/diffusion.py+++ b/unsloth/models/diffusion.py@@ -20,7 +20,7 @@ bit-identical to transformers), keeping only the safe conveniences: 4bit/8bit loimport osimport reimport torch-from transformers import AutoConfig, AutoProcessor, AutoTokenizer+from transformers import AutoConfig, AutoProcessor, AutoTokenizer, set_seedfrom unsloth_zoo.hf_utils import add_dtype_kwargsfrom ._utils import is_bfloat16_supported, maybe_prefetch_hf_snapshot@@ -40,7 +40,12 @@ from .loader_utils import (resolve_unsloth_device_map,)-__all__ = ["FastDiffusionModel", "DIFFUSION_MODEL_TYPES", "is_diffusion_model_type"]+__all__ = [+ "FastDiffusionModel",+ "DIFFUSION_MODEL_TYPES",+ "is_diffusion_model",+ "is_diffusion_model_type",+]# transformers model_type strings routed to this slow path.DIFFUSION_MODEL_TYPES = ("diffusion_gemma", "diffusion_gemma4")@@ -69,6 +74,41 @@ def is_diffusion_model_type(model_types):return any(mt in known for mt in model_types)+def is_diffusion_model(model):+ """True for a text-diffusion model: loaded through FastDiffusionModel, or with a config a DiffusionProfile+ matches. PEFT / DDP wrapped or not."""+ if model is None:+ return False+ base = unwrap_diffusion_model(model)+ if any(+ getattr(candidate, "_unsloth_slow_diffusion", False)+ or getattr(candidate, "_unsloth_diffusion_profile", None) is not None+ for candidate in (model, base)+ ):+ return True+ return resolve_diffusion_profile(getattr(base, "config", None)) is not None+++def _ensure_pad_token(model, tokenizer):+ """Collators need a pad token (Nemotron-Labs-Diffusion ships none). Pad with EOS, as the objectives treat+ padding as EOS-filled; never the [MASK] token. Only fills what is unset, so repeat calls change nothing."""+ tok = getattr(tokenizer, "tokenizer", tokenizer)+ if tok is None or getattr(tok, "pad_token", None) is not None:+ return+ if getattr(tok, "eos_token", None) is None:+ return+ tok.pad_token = tok.eos_token+ if tok is not tokenizer and getattr(tokenizer, "pad_token", None) is None:+ try:+ tokenizer.pad_token = tok.eos_token+ except AttributeError:+ pass+ base = unwrap_diffusion_model(model)+ for holder in (getattr(base, "config", None), getattr(base, "generation_config", None)):+ if holder is not None and getattr(holder, "pad_token_id", None) is None:+ holder.pad_token_id = tok.pad_token_id++def _refuse_tied_lora_merge(model):"""DiffusionGemma's encoder and decoder layers share one base tensor but get separate adapters, so a mergeadds both deltas to both. Refuse it instead of writing a silently wrong checkpoint."""@@ -454,6 +494,7 @@ class FastDiffusionModel:cache_dir = cache_dir,)+ _ensure_pad_token(model, tokenizer)if profile is not None:# On the base module: the objectives read these attributes after unwrapping PEFT / DDP.profile.prepare_model(unwrap_diffusion_model(model), tokenizer)@@ -476,7 +517,31 @@ class FastDiffusionModel:if getattr(model, "_unsloth_full_finetuning", False):print("Unsloth: Full finetuning is enabled, so .get_peft_model has no effect")return model- from peft import LoraConfig, get_peft_model as peft_get_peft_model+ # Refuse what this transformers-only path cannot honour instead of silently training plain LoRA.+ if kwargs.get("use_dora"):+ raise ValueError(+ "Unsloth: DoRA is not supported for text-diffusion models yet. Use use_dora = False."+ )+ if kwargs.get("loftq_config"):+ raise ValueError(+ "Unsloth: LoftQ is not supported for text-diffusion models yet. Use loftq_config = None."+ )+ from peft import LoraConfig, PeftModel, get_peft_model as peft_get_peft_model++ if isinstance(model, PeftModel):+ # A loaded adapter (continued training, resume) keeps its weights: re-injecting would start over.+ for name, param in model.named_parameters():+ if "lora_" in name:+ param.requires_grad_(True)+ if use_gradient_checkpointing:+ model.gradient_checkpointing_enable()+ if hasattr(model, "enable_input_require_grads"):+ model.enable_input_require_grads()+ else:+ _disable_gradient_checkpointing(model)+ model._unsloth_slow_diffusion = True+ model._unsloth_gradient_checkpointing = use_gradient_checkpointing+ return modelprofile = resolve_diffusion_profile(getattr(model, "config", None))if target_modules is None:@@ -484,8 +549,25 @@ class FastDiffusionModel:profile.lora_target_modules if profile is not None else DIFFUSION_LORA_TARGETS)- # use_dora, and any other LoraConfig kwarg outside this allowlist, is silently dropped: Unsloth does- # not reach this path today, so it is untested on diffusion models.+ # Unsloth-only layer / offload options do not apply to this path: say so rather than drop them silently.+ ignored = sorted(+ k+ for k in (+ "layers_to_transform",+ "layers_pattern",+ "finetune_last_n_layers",+ "target_parameters",+ "qat_scheme",+ "offload_layers",+ "checkpoint_skip_layers",+ )+ if kwargs.get(k) not in (None, 0, False, [], ())+ )+ if ignored:+ logger.warning(+ f"Unsloth: {', '.join(ignored)} {'is' if len(ignored) == 1 else 'are'} not supported for "+ "text-diffusion models and will be ignored."+ )lora_kwargs = dict(r = r,lora_alpha = lora_alpha,@@ -493,6 +575,7 @@ class FastDiffusionModel:bias = bias,target_modules = target_modules,task_type = task_type, # None: diffusion has no standard CAUSAL_LM head+ use_rslora = bool(kwargs.get("use_rslora", False)),**{k: v for k, v in kwargs.items() if k in ("modules_to_save", "init_lora_weights")},)# Exclude the vision tower's custom (non-Linear) modules that share suffix names.@@ -519,6 +602,8 @@ class FastDiffusionModel:if hasattr(model, "enable_input_require_grads"):model.enable_input_require_grads()+ if random_state is not None:+ set_seed(random_state)model = peft_get_peft_model(model, lora_config)model = _refuse_tied_lora_merge(model)model._unsloth_slow_diffusion = True@@ -532,9 +617,7 @@ class FastDiffusionModel:@staticmethoddef for_inference(model):model.eval()- for _, m in model.named_modules():- if hasattr(m, "gradient_checkpointing"):- m.gradient_checkpointing = False+ _disable_gradient_checkpointing(model)return model@staticmethod@@ -542,4 +625,48 @@ class FastDiffusionModel:model.train()if use_gradient_checkpointing and hasattr(model, "gradient_checkpointing_enable"):model.gradient_checkpointing_enable()+ elif not use_gradient_checkpointing:+ _disable_gradient_checkpointing(model)+ model._unsloth_gradient_checkpointing = use_gradient_checkpointingreturn model++ @staticmethod+ def generate_text(+ model,+ tokenizer,+ prompt = None,+ messages = None,+ max_new_tokens = 256,+ steps = None,+ block_length = None,+ temperature = 0.0,+ stop_callback = None,+ **kwargs,+ ):+ """One completion as text via the family's diffusion sampler (see diffusion_samplers.generate_text)."""+ from .diffusion_samplers import generate_text+ return generate_text(+ model,+ tokenizer,+ prompt = prompt,+ messages = messages,+ max_new_tokens = max_new_tokens,+ steps = steps,+ block_length = block_length,+ temperature = temperature,+ stop_callback = stop_callback,+ **kwargs,+ )+++def _disable_gradient_checkpointing(model):+ # gradient_checkpointing_disable reaches LLaDA's own set_activation_checkpointing through the profile bridge.+ disable = getattr(model, "gradient_checkpointing_disable", None)+ if callable(disable):+ try:+ disable()+ except (AttributeError, ValueError):+ pass+ for _, m in model.named_modules():+ if hasattr(m, "gradient_checkpointing"):+ m.gradient_checkpointing = Falsediff --git a/unsloth/models/diffusion_block.py b/unsloth/models/diffusion_block.pyindex 29b5df60360..b7759c0246f 100644--- a/unsloth/models/diffusion_block.py+++ b/unsloth/models/diffusion_block.py@@ -18,6 +18,12 @@ import typesimport torchimport torch.nn.functional as F+from .diffusion_loss import (+ chunked_cross_entropy,+ compile_loss,+ is_distributed_wrapper,+ is_plain_linear,+)from .diffusion_profiles import (DiffusionProfile,register_diffusion_profile,@@ -53,12 +59,7 @@ def block_diffusion_attention_mask(_unwrap = unwrap_diffusion_model-def _is_distributed_wrapper(model):- return type(model).__name__ in (- "DistributedDataParallel",- "FullyShardedDataParallel",- "DeepSpeedEngine",- )+_is_distributed_wrapper = is_distributed_wrapperclass BlockDiffusionProfile(DiffusionProfile):@@ -139,8 +140,16 @@ class BlockDiffusionProfile(DiffusionProfile):maskable = response_mask(inputs)if not self.defaults.get("eos_fill", False):return input_ids, maskable, valid- if (valid[:, 1:] & ~valid[:, :-1]).any():- raise ValueError("Unsloth: block-diffusion training expects right padding.")+ base = _unwrap(model)+ if not base.__dict__.get("_unsloth_right_padding_checked", False):+ # Padding side is fixed per run: check until a padded batch is seen, then stop.+ left_padded, padded = torch.stack(+ [(valid[:, 1:] & ~valid[:, :-1]).any(), (~valid).any()]+ ).tolist()+ if left_padded:+ raise ValueError("Unsloth: block-diffusion training expects right padding.")+ if padded:+ base.__dict__["_unsloth_right_padding_checked"] = True# Turns end with a terminator, not EOS, so stopping is learnt only from a supervised EOS tail (as the# reference's max-length EOS padding); pad to the next block past the longest row.block = self.block_size(model, args)@@ -177,17 +186,19 @@ class BlockDiffusionProfile(DiffusionProfile):p = low + (high - low) * torch.rand(batch, device = clean.device)masked = (torch.rand(batch, length, device = clean.device) < p[:, None]) & maskableif self.defaults.get("ensure_masked", False):+ # Rescue draw taken every step (fixed shape, no host sync); it only changes rows left empty.empty = maskable.any(dim = 1) & ~masked.any(dim = 1)- if empty.any():- scores = torch.rand(batch, length, device = clean.device).masked_fill(~maskable, -1.0)- pick = F.one_hot(scores.argmax(dim = 1), length).bool()- masked = masked | (pick & empty[:, None])+ scores = torch.rand(batch, length, device = clean.device).masked_fill(~maskable, -1.0)+ pick = F.one_hot(scores.argmax(dim = 1), length).bool()+ masked = masked | (pick & empty[:, None])return torch.where(masked, mask_token_id, clean), masked, pdef attention_mask(self, length, block_size, valid, dtype, device):mask = block_diffusion_attention_mask(length, block_size, device)[None, None]batch = valid.shape[0]- if self.defaults.get("key_padding", False) and not bool(valid.all()):+ if self.defaults.get("key_padding", False):+ # Unconditional: with no padding `keys` is all True and the diagonal is already allowed, so a+ # padding-free batch gets the same mask without a host sync.keys = torch.cat([valid, valid], dim = 1)[:, None, None, :]# Padding is its own segment in the reference; a padding query keeps its diagonal so no row is empty.mask = (mask & keys) | torch.eye(2 * length, dtype = torch.bool, device = device)[@@ -224,29 +235,49 @@ class BlockDiffusionProfile(DiffusionProfile):)return out[0][:, :length], head- def loss_from_noise(self, model, clean, noisy, masked, maskable, p, valid, args):+ def loss_from_noise(+ self,+ model,+ clean,+ noisy,+ masked,+ maskable,+ p,+ valid,+ args,+ with_metrics = False,+ ):block_size = self.block_size(model, args)- hidden, head = self.noisy_hidden_and_head(model, noisy, clean, valid, block_size)- if head is None:- logits = hidden[masked].float()- else:- # Project only the masked positions: the vocab-sized logits are the memory peak otherwise.- logits = head(hidden[masked]).float()- nll = F.cross_entropy(logits, clean[masked], reduction = "none")weighting = self.option(args, "diffusion_time_weighting", "none")if weighting not in ("inverse_t", "none"):raise ValueError(f"Unsloth: {self.name} does not support diffusion_time_weighting={weighting!r}.")+ hidden, head = self.noisy_hidden_and_head(model, noisy, clean, valid, block_size)+ if head is not None and not is_plain_linear(head):+ # A quantized / LoRA / offloaded / hooked head only runs through its own forward.+ hidden, head = head(hidden), None+ targets = torch.where(masked, clean, -100)+ weights = masked.float()if weighting == "inverse_t":- nll = nll / p[:, None].expand_as(masked)[masked].float()+ # Divide only where masked: an unmasked row with p = 0 (diffusion_eps = 0) would give 0 / 0.+ weights = weights / torch.where(masked, p[:, None].float(), 1.0)+ # Unmasked rows are ignored targets, so the shape is fixed whatever was masked.+ weighted, ce_sum, count = chunked_cross_entropy(+ hidden,+ targets,+ weights,+ weight = None if head is None else head.weight,+ bias = None if head is None else head.bias,+ compiled = compile_loss(args, clean.device),+ )if self.defaults.get("normalize", "masked") == "supervised":denominator = maskable.sum()else:- denominator = masked.sum()- loss = nll.sum() / denominator.clamp_min(1)- if not masked.any():- loss = loss + hidden.sum() * 0.0+ denominator = count+ loss = weighted / denominator.clamp_min(1)+ if with_metrics:+ return loss, ce_sum, countreturn lossdef compute_loss(@@ -259,10 +290,13 @@ class BlockDiffusionProfile(DiffusionProfile):clean, maskable, valid = self.build_batch(model, inputs, args)mask_token_id = self.mask_token_id(model, args)noisy, masked, p = self.sample_noise(clean, maskable, mask_token_id, args)- loss = self.loss_from_noise(model, clean, noisy, masked, maskable, p, valid, args)+ loss, ce_sum, count = self.loss_from_noise(+ model, clean, noisy, masked, maskable, p, valid, args, with_metrics = True+ )metrics = {- "mask_ratio": (masked.sum() / maskable.sum().clamp_min(1)).item(),- "masked_tokens": masked.sum().item(),+ "mask_ratio": (count / maskable.sum().clamp_min(1)).detach(),+ "masked_tokens": count.detach(),+ "masked_ce": (ce_sum / count.clamp_min(1)).detach(),}return loss, None, metrics@@ -347,9 +381,9 @@ def _module(name, **attributes):@contextlib.contextmanagerdef _sdar_remote_code_imports():- """SDAR's modeling file imports flash-attn's RMSNorm and 4.x `LossKwargs` at module level. Provide- torch stand-ins only while it imports when flash-attn is absent; the module keeps its references."""- import importlib.util+ """SDAR's modeling file imports flash-attn's RMSNorm, 4.x `LossKwargs` and `SlidingWindowCache` at+ module level. Provide stand-ins only while it imports; the module keeps its references."""+ import transformers.cache_utils as cache_utilsimport transformers.utils as transformers_utilsif not hasattr(transformers_utils, "LossKwargs"):@@ -358,6 +392,22 @@ def _sdar_remote_code_imports():num_items_in_batch: Optional[torch.Tensor]transformers_utils.LossKwargs = LossKwargs+ # Missing from some transformers 5.x releases (5.5 has none); SDAR only isinstance-checks it, and nothing builds one.+ added_sliding_cache = not hasattr(cache_utils, "SlidingWindowCache")+ if added_sliding_cache:+ cache_utils.SlidingWindowCache = type("SlidingWindowCache", (cache_utils.Cache,), {})+ try:+ with _sdar_flash_attn_stubs():+ yield+ finally:+ if added_sliding_cache:+ del cache_utils.SlidingWindowCache+++def _sdar_flash_attn_stubs():+ import importlib.util+try:has_flash = importlib.util.find_spec("flash_attn") is not Noneexcept (ImportError, ValueError):diff --git a/unsloth/models/diffusion_gemma_objective.py b/unsloth/models/diffusion_gemma_objective.pyindex bff1c8fb1a5..3eee02e1700 100644--- a/unsloth/models/diffusion_gemma_objective.py+++ b/unsloth/models/diffusion_gemma_objective.py@@ -9,6 +9,7 @@ import contextlibimport torchimport torch.nn.functional as F+from .diffusion_loss import chunked_cross_entropy, compile_lossfrom .diffusion_profiles import DiffusionProfile, register_diffusion_profile, unwrap_diffusion_model__all__ = ["DiffusionGemmaProfile"]@@ -157,32 +158,43 @@ class DiffusionGemmaProfile(DiffusionProfile):diffusion_logits = diffusion_logits.float().scatter_add(2, canvas_ids.unsqueeze(-1), bump[:, :, None].expand(-1, block_size, 1))- if (diffusion_target != -100).any():- diffusion_loss = F.cross_entropy(- diffusion_logits.flatten(0, 1), diffusion_target.flatten(), ignore_index = -100- )- else:- diffusion_loss = outputs.logits.sum() * 0.0+ # Eager under "auto": compiled chunks were 5-10% slower per step on DiffusionGemma-26B-A4B (B200) at equal memory.+ compiled = compile_loss(args, device, auto = False)+ # Sum / count instead of a mean: an all-ignored batch gives 0 (with grad), not NaN, and needs no branch.+ diffusion_sum, _, diffusion_count = chunked_cross_entropy(+ diffusion_logits, diffusion_target, compiled = compiled+ )+ diffusion_loss = diffusion_sum / diffusion_count.clamp_min(1)+ # Encoder AR co-loss straight from the hidden states: the [B, L, V] encoder logits never exist.lm_head = base.get_output_embeddings()hidden_states = outputs.encoder_last_hidden_state.to(lm_head.weight.dtype)- with _gather_lm_head(lm_head.weight, lm_head.bias):- encoder_logits = hidden_states @ lm_head.weight.t()- if lm_head.bias is not None:- encoder_logits = encoder_logits + lm_head.bias- encoder_logits = encoder_logits.float()- encoder_logits = torch.tanh(encoder_logits / softcap) * softcapar_mask = attention_mask[:, :-1].bool() & attention_mask[:, 1:].bool()ar_targets = torch.where(ar_mask, input_ids[:, 1:], -100)- ar_loss = F.cross_entropy(- encoder_logits[:, :-1].flatten(0, 1), ar_targets.flatten(), ignore_index = -100- )+ if hasattr(lm_head.weight, "ds_id"):+ # ZeRO-3 re-partitions the head when the gather exits, so the projection must happen inside it+ # (autograd then keeps the gathered tensor); the fused path would re-read it in backward.+ with _gather_lm_head(lm_head.weight, lm_head.bias):+ encoder_logits = F.linear(hidden_states[:, :-1], lm_head.weight, lm_head.bias)+ ar_sum, _, ar_count = chunked_cross_entropy(+ encoder_logits, ar_targets, softcap = softcap, compiled = compiled+ )+ else:+ ar_sum, _, ar_count = chunked_cross_entropy(+ hidden_states[:, :-1],+ ar_targets,+ weight = lm_head.weight,+ bias = lm_head.bias,+ softcap = softcap,+ compiled = compiled,+ )+ ar_loss = ar_sum / ar_count.clamp_min(1)loss = diffusion_loss + ar_weight * ar_lossmetrics = {- "diffusion_loss": diffusion_loss.detach().item(),- "encoder_ar_loss": ar_loss.detach().item(),- "mean_t": t.mean().item(),+ "diffusion_loss": diffusion_loss.detach(),+ "encoder_ar_loss": ar_loss.detach(),+ "mean_t": t.mean().detach(),}return loss, outputs, metricsdiff --git a/unsloth/models/diffusion_loss.py b/unsloth/models/diffusion_loss.pynew file mode 100644index 00000000000..9431e0ce021--- /dev/null+++ b/unsloth/models/diffusion_loss.py@@ -0,0 +1,369 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Weighted cross entropy for the diffusion objectives without [N, V] fp32 logits.++Rows are processed in fixed-size chunks (the last one padded with ignored rows), so the compiled chunk kernels+see one shape per vocabulary / chunk size and never recompile as the masked fraction or sequence length moves.+Backward recomputes each chunk's logits instead of keeping them.+"""++import os++import torch+import torch.nn.functional as F++__all__ = [+ "chunked_cross_entropy",+ "diffusion_compile_mode",+ "compile_loss",+ "compile_backbone_layers",+ "fusable_head",+ "head_returns_hidden",+ "is_distributed_wrapper",+ "is_plain_linear",+]++_CHUNK_BYTES = int(os.environ.get("UNSLOTH_DIFFUSION_CE_CHUNK_BYTES", 256 * 1024 * 1024))+_MIN_ROWS = 16+++def diffusion_compile_mode(args):+ """None (eager), "auto" (compiled CE chunks on CUDA), "loss" (compiled CE chunks anywhere) or "full"+ (compiled CE chunks + per-layer backbone compile)."""+ if os.environ.get("UNSLOTH_COMPILE_DISABLE", "0").lower() in ("1", "true", "yes"):+ return None+ value = getattr(args, "diffusion_compile", "auto") if args is not None else "auto"+ if isinstance(value, str):+ value = value.strip().lower()+ if value in (False, None, "false", "0", "no", "off", "none", "eager"):+ return None+ if value in (True, "true", "1", "yes", "on", "full"):+ return "full"+ if value in ("auto", "loss"):+ return value+ raise ValueError(+ f"Unsloth: diffusion_compile must be 'auto', 'loss', 'full', True or False, got {value!r}."+ )+++def compile_loss(+ args,+ device,+ auto = True,+):+ """`auto` = False keeps "auto" eager for objectives where compiled chunks measured no faster."""+ mode = diffusion_compile_mode(args)+ return mode in ("loss", "full") or (+ auto and mode == "auto" and torch.device(device).type == "cuda"+ )+++def _rows_per_chunk(n, vocab):+ rows = max(_MIN_ROWS, _CHUNK_BYTES // (4 * max(vocab, 1)))+ rows = 1 << (rows.bit_length() - 1)+ # Small inputs use the next power of two, so short batches neither pad to a huge chunk nor recompile per length.+ return min(rows, max(_MIN_ROWS, 1 << max(n - 1, 0).bit_length()))+++def _logits(h, weight, bias, softcap):+ z = h if weight is None else F.linear(h, weight, bias)+ z = z.float()+ if softcap is not None:+ z = torch.tanh(z / softcap) * softcap+ return z+++def _chunk_forward(h, weight, bias, targets, token_weights, softcap):+ z = _logits(h, weight, bias, softcap)+ lse = torch.logsumexp(z, dim = -1)+ valid = targets != -100+ picked = z.gather(1, targets.clamp_min(0)[:, None]).squeeze(1)+ ce = torch.where(valid, lse - picked, 0.0)+ return (ce * token_weights).sum(), ce.sum(), lse+++def _chunk_backward(h, weight, bias, targets, token_weights, lse, grad, softcap):+ z = h if weight is None else F.linear(h, weight, bias)+ z = z.float()+ if softcap is not None:+ tanh = torch.tanh(z / softcap)+ z = tanh * softcap+ valid = targets != -100+ scale = torch.where(valid, token_weights * grad, 0.0)+ dz = torch.exp(z - lse[:, None])+ dz = dz.scatter_add(1, targets.clamp_min(0)[:, None], -valid.to(dz.dtype)[:, None])+ dz = dz * scale[:, None]+ if softcap is not None:+ dz = dz * (1 - tanh * tanh)+ # Select, not multiply: an ignored row can overflow (fp16 heads), and 0 * inf is NaN.+ dz = torch.where(valid[:, None], dz, 0.0)+ return dz.to(h.dtype)+++_COMPILED = {}+++def _kernels(compiled):+ if not compiled:+ return _chunk_forward, _chunk_backward+ if not _COMPILED:+ _COMPILED["forward"] = torch.compile(_chunk_forward, fullgraph = True, dynamic = False)+ _COMPILED["backward"] = torch.compile(_chunk_backward, fullgraph = True, dynamic = False)+ return _COMPILED["forward"], _COMPILED["backward"]+++def _chunk(tensor, start, rows, fill):+ piece = tensor[start : start + rows]+ short = rows - piece.shape[0]+ if short:+ pad = [0, 0] * (piece.dim() - 1) + [0, short]+ piece = F.pad(piece, pad, value = fill)+ return piece+++class _ChunkedCrossEntropy(torch.autograd.Function):+ @staticmethod+ def forward(ctx, hidden, weight, bias, targets, token_weights, softcap, compiled):+ device_type = hidden.device.type+ ctx.autocast = (+ device_type,+ torch.is_autocast_enabled(device_type),+ torch.get_autocast_dtype(device_type),+ )+ # Casts made here carry no autograd history, so they must not land in the enclosing autocast cache,+ # where a later projection of the same weight would reuse them and lose its weight gradient.+ with torch.autocast(+ device_type = device_type,+ dtype = ctx.autocast[2],+ enabled = ctx.autocast[1],+ cache_enabled = False,+ ):+ return _ChunkedCrossEntropy._forward(+ ctx, hidden, weight, bias, targets, token_weights, softcap, compiled+ )++ @staticmethod+ def _forward(ctx, hidden, weight, bias, targets, token_weights, softcap, compiled):+ forward_kernel, _ = _kernels(compiled)+ n = hidden.shape[0]+ vocab = hidden.shape[1] if weight is None else weight.shape[0]+ rows = _rows_per_chunk(n, vocab)+ token_weights = token_weights.float()+ weighted = hidden.new_zeros((), dtype = torch.float32)+ total = hidden.new_zeros((), dtype = torch.float32)+ lses = []+ for start in range(0, n, rows):+ w_sum, ce_sum, lse = forward_kernel(+ _chunk(hidden, start, rows, 0),+ weight,+ bias,+ _chunk(targets, start, rows, -100),+ _chunk(token_weights, start, rows, 0),+ softcap,+ )+ weighted = weighted + w_sum+ total = total + ce_sum+ lses.append(lse)+ lse = torch.cat(lses) if lses else hidden.new_zeros((0,), dtype = torch.float32)+ ctx.save_for_backward(hidden, weight, bias, targets, token_weights, lse)+ ctx.rows, ctx.softcap, ctx.compiled = rows, softcap, compiled+ ctx.mark_non_differentiable(total)+ return weighted, total++ @staticmethod+ def backward(ctx, grad_weighted, grad_total):+ # Recompute the logits in the forward's precision, or softmax and logsumexp disagree.+ device_type, enabled, dtype = ctx.autocast+ with torch.autocast(+ device_type = device_type, dtype = dtype, enabled = enabled, cache_enabled = False+ ):+ return _ChunkedCrossEntropy._backward(ctx, grad_weighted)++ @staticmethod+ def _backward(ctx, grad_weighted):+ hidden, weight, bias, targets, token_weights, lse = ctx.saved_tensors+ _, backward_kernel = _kernels(ctx.compiled)+ rows, n = ctx.rows, hidden.shape[0]+ need_h, need_w, need_b = ctx.needs_input_grad[:3]+ grad_hidden = torch.empty_like(hidden) if need_h else None+ grad_w = torch.zeros_like(weight, dtype = torch.float32) if need_w else None+ grad_b = (+ torch.zeros_like(bias, dtype = torch.float32) if need_b and bias is not None else None+ )+ grad = grad_weighted.float()+ for start in range(0, n, rows):+ h = _chunk(hidden, start, rows, 0)+ dz = backward_kernel(+ h,+ weight,+ bias,+ _chunk(targets, start, rows, -100),+ _chunk(token_weights, start, rows, 0),+ lse[start : start + rows], # one padded entry per row, so every slice is full+ grad,+ ctx.softcap,+ )+ take = min(rows, n - start)+ if weight is None:+ if need_h:+ grad_hidden[start : start + take] = dz[:take]+ continue+ if need_h:+ grad_hidden[start : start + take] = (dz @ weight)[:take]+ if need_w:+ grad_w += (dz.t() @ h).float()+ if grad_b is not None:+ grad_b += dz.sum(0).float()+ if grad_w is not None:+ grad_w = grad_w.to(weight.dtype)+ if grad_b is not None:+ grad_b = grad_b.to(bias.dtype)+ return grad_hidden, grad_w, grad_b, None, None, None, None+++def chunked_cross_entropy(+ hidden,+ targets,+ token_weights = None,+ weight = None,+ bias = None,+ softcap = None,+ compiled = False,+):+ """sum(w * CE) and the unweighted CE sum over positions with target != -100.++ ``hidden`` is [..., H] projected by ``weight`` [V, H] (+ ``bias``), or already logits [..., V] when+ ``weight`` is None. Logits are computed in the input dtype and upcast like ``head(h).float()``. The second+ output carries no gradient (metrics). Returns (weighted_sum, ce_sum, count) as 0-dim tensors.+ """+ width = hidden.shape[-1]+ hidden = hidden.reshape(-1, width)+ targets = targets.reshape(-1)+ if token_weights is None:+ token_weights = (targets != -100).float()+ token_weights = token_weights.reshape(-1)+ if weight is not None and hidden.dtype != weight.dtype:+ hidden = hidden.to(weight.dtype)+ weighted, total = _ChunkedCrossEntropy.apply(+ hidden,+ weight,+ bias,+ targets,+ token_weights,+ None if softcap is None else float(softcap),+ bool(compiled),+ )+ return weighted, total, (targets != -100).sum()+++def is_distributed_wrapper(model):+ return type(model).__name__ in (+ "DistributedDataParallel",+ "FullyShardedDataParallel",+ "DeepSpeedEngine",+ )+++def is_plain_linear(module):+ """An nn.Linear whose output is exactly F.linear(x, weight, bias): no subclass (LoRA / quantized), no+ accelerate offload hook, no instance-level forward and no forward hooks."""+ return (+ type(module) is torch.nn.Linear+ and not hasattr(module, "_hf_hook")+ and "forward" not in vars(module)+ and not module._forward_hooks+ and not module._forward_pre_hooks+ )+++def fusable_head(model, base):+ """The plain nn.Linear output head whose input can be taken instead of logits, or None (dense logits).++ Decided once per head module (a head wrapped later, e.g. by PEFT, is re-checked): a LoRA / quantized /+ offloaded / tied or post-scaled head, or a distributed wrapper (FSDP shards the weight outside its+ forward), keeps the dense path."""+ if is_distributed_wrapper(model):+ return None+ head = base.get_output_embeddings() if hasattr(base, "get_output_embeddings") else None+ cached = base.__dict__.get("_unsloth_diffusion_fused_head")+ if cached is not None and cached[0] is head:+ return cached[1]+ config = getattr(base, "config", None)+ ok = (+ is_plain_linear(head)+ and not getattr(+ config, "weight_tying", False+ ) # LLaDA: tied head is F.linear on wte, not this module+ and not getattr(config, "scale_logits", False)+ and head.weight.device.type != "meta"+ )+ base.__dict__["_unsloth_diffusion_fused_head"] = (head, head if ok else None)+ return head if ok else None+++class head_returns_hidden:+ """While active the output head returns its input, so ``model(...).logits`` are the final hidden states."""++ def __init__(self, head):+ self.head = head++ # Eager even under an outer torch.compile: the swap is plain module plumbing, not math.+ @torch.compiler.disable+ def __enter__(self):+ if self.head is not None:+ self.head.forward = _identity+ return self.head++ @torch.compiler.disable+ def __exit__(self, *exc):+ if self.head is not None:+ self.head.__dict__.pop("forward", None)+ return False+++def _identity(x):+ return x+++def _decoder_layer_lists(model):+ seen = set()+ for name, module in model.named_modules():+ if not isinstance(module, torch.nn.ModuleList) or len(module) == 0:+ continue+ if name.rsplit(".", 1)[-1] not in ("layers", "blocks", "h"):+ continue+ if any(name.startswith(prefix + ".") for prefix in seen):+ continue+ seen.add(name)+ yield name, module+++def _warm_rotary_caches(model):+ """LLaDA fills its RoPE cache lazily and moves it to the parameters' device on first use. Inside a compiled,+ checkpointed layer that move happens in the forward but not in the recompute, so the two save different+ tensors (CheckpointError). Fill it on the right device before compiling."""+ try:+ device = next(model.parameters()).device+ except StopIteration:+ return+ for module in model.modules():+ rotary = getattr(module, "get_rotary_embedding", None)+ config = getattr(module, "config", None)+ length = getattr(config, "max_sequence_length", None)+ if callable(rotary) and isinstance(length, int) and length > 0:+ rotary(length, device)+++def compile_backbone_layers(model, dynamic = None):+ """Compile every decoder layer in place (regional compile); idempotent. Returns the layer count."""+ _warm_rotary_caches(model)+ compiled = 0+ for _, layers in _decoder_layer_lists(model):+ for layer in layers:+ if getattr(layer, "_unsloth_diffusion_compiled", False):+ continue+ layer.compile(dynamic = dynamic)+ layer._unsloth_diffusion_compiled = True+ compiled += 1+ return compileddiff --git a/unsloth/models/diffusion_masked.py b/unsloth/models/diffusion_masked.pyindex f1928b6a2f7..947dabc99f0 100644--- a/unsloth/models/diffusion_masked.py+++ b/unsloth/models/diffusion_masked.py@@ -12,8 +12,13 @@ import functoolsfrom dataclasses import dataclassimport torch-import torch.nn.functional as F+from .diffusion_loss import (+ chunked_cross_entropy,+ compile_loss,+ fusable_head,+ head_returns_hidden,+)from .diffusion_profiles import (DiffusionProfile,register_diffusion_profile,@@ -170,19 +175,25 @@ class MaskedDiffusionProfile(DiffusionProfile):from transformers.dynamic_module_utils import get_class_from_dynamic_moduleauto_map = getattr(config, "auto_map", None) or {}- class_ref = (- auto_map.get(self.auto_class)- or auto_map.get("AutoModelForCausalLM")- or auto_map.get("AutoModel")+ auto_class = next(+ (+ name+ for name in (self.auto_class, "AutoModelForCausalLM", "AutoModel")+ if auto_map.get(name)+ ),+ None,)- if class_ref is None or not trust_remote_code:+ if auto_class is None or not trust_remote_code:return super().model_class(config, trust_remote_code, **hub_kwargs)cls = get_class_from_dynamic_module(- class_ref,+ auto_map[auto_class],config._name_or_path,revision = getattr(config, "_commit_hash", None),**hub_kwargs,)+ # transformers < 5.17 reruns a non-Auto hub class's _init_weights over the loaded checkpoint (LLaDA-MoE, Dream).+ if getattr(cls, "_auto_class", None) is None:+ cls.register_for_auto_class(auto_class)_accept_new_validate_kwargs(cls)return _ensure_post_init(cls)@@ -240,6 +251,10 @@ class MaskedDiffusionProfile(DiffusionProfile):mask_id, eos_id = self._token_ids(model, args)eps = float(self.option(args, "diffusion_eps"))weighting = self.option(args, "diffusion_time_weighting")+ if weighting not in ("inverse_t", "none", None):+ raise ValueError(+ f"Unsloth: {self.name} does not support diffusion_time_weighting={weighting!r}."+ )maskable = response_mask(inputs)forward_mask = attention_mask@@ -258,33 +273,36 @@ class MaskedDiffusionProfile(DiffusionProfile):noisy = torch.where(masked, mask_id, input_ids)model_inputs = {"input_ids": noisy}- if forward_mask is not None and not bool(forward_mask.all()):+ if forward_mask is not None:model_inputs["attention_mask"] = forward_mask- logits = model(**model_inputs).logits+ head = fusable_head(model, unwrap_diffusion_model(model))+ with head_returns_hidden(head):+ hidden = model(**model_inputs).logitsif self.shift_logits:- logits = torch.cat([logits[:, :1], logits[:, :-1]], dim = 1)-- if not bool(masked.any()):- zero = logits.sum() * 0.0- return zero, None, {"masked_fraction": 0.0}+ hidden = torch.cat([hidden[:, :1], hidden[:, :-1]], dim = 1)- token_loss = F.cross_entropy(logits[masked].float(), input_ids[masked], reduction = "none")+ targets = torch.where(masked, input_ids, -100)+ weights = masked.float()if weighting == "inverse_t":- token_loss = token_loss / p_mask[masked]- elif weighting not in ("none", None):- raise ValueError(- f"Unsloth: {self.name} does not support diffusion_time_weighting={weighting!r}."- )-+ weights = weights / torch.where(masked, p_mask, 1.0)+ if self.loss_normalization == "answer_length":+ weights = weights / (maskable.sum(dim = 1, keepdim = True).clamp_min(1) * batch)+ weighted, ce_sum, count = chunked_cross_entropy(+ hidden,+ targets,+ weights,+ weight = None if head is None else head.weight,+ bias = None if head is None else head.bias,+ compiled = compile_loss(args, device),+ )if self.loss_normalization == "answer_length":- answer_length = maskable.sum(dim = 1, keepdim = True).clamp_min(1).expand(batch, length)- loss = (token_loss / answer_length[masked]).sum() / batch+ loss = weightedelse:- loss = token_loss.sum() / masked.sum()+ loss = weighted / count.clamp_min(1)metrics = {- "masked_fraction": (masked.sum() / maskable.sum().clamp_min(1)).item(),- "masked_ce": F.cross_entropy(logits[masked].float(), input_ids[masked]).item(),+ "masked_fraction": (count / maskable.sum().clamp_min(1)).detach(),+ "masked_ce": (ce_sum / count.clamp_min(1)).detach(),}return loss, None, metrics@@ -312,6 +330,7 @@ register_diffusion_profile(lora_target_modules = r".*\.blocks\.\d+\.(q_proj|k_proj|v_proj|attn_out|ff_proj|up_proj|ff_out)",quant_skip_modules = ["transformer.ff_out", "transformer.wte", "lm_head"],default_mask_token_id = 126336,+ compile_backbone = True,))register_diffusion_profile(@@ -324,6 +343,7 @@ register_diffusion_profile(shift_logits = True,loss_normalization = "masked_tokens",auto_class = "AutoModel",+ compile_backbone = True,# Dream q_sample draws t ~ U(0, 1) with no floor and weights by 1 / t (time_reweighting "original").defaults = {"diffusion_eps": 0.0},)@@ -335,7 +355,10 @@ register_diffusion_profile(architectures = ("NemotronLabsDiffusionModel",),requires_remote_code = True,lora_target_modules = list(_ATTENTION_AND_MLP),+ # Keep `diffusion_head` 16-bit like every other head, so the loss can take hidden states.+ quant_skip_modules = ["lm_head", "embed_tokens", "diffusion_head"],loss_normalization = "masked_tokens",auto_class = "AutoModel",+ compile_backbone = True,))diff --git a/unsloth/models/diffusion_profiles.py b/unsloth/models/diffusion_profiles.pyindex 0667da7b513..43c9bbeb448 100644--- a/unsloth/models/diffusion_profiles.py+++ b/unsloth/models/diffusion_profiles.py@@ -50,6 +50,8 @@ class DiffusionProfile:quant_skip_modules: Optional[list] = None# Profile-specific defaults a DiffusionConfig field left at None falls back to.defaults: dict = field(default_factory = dict)+ # diffusion_compile = "full" allowed; off for data-dependent MoE dispatch (per-expert token gathers).+ compile_backbone: bool = Falsedef matches(self, config) -> bool:model_type = getattr(config, "model_type", None)@@ -89,7 +91,7 @@ class DiffusionProfile:args,num_items_in_batch = None,):- """Returns (loss, outputs_or_None, metrics: dict[str, float])."""+ """Returns (loss, outputs_or_None, metrics: dict[str, float | 0-dim tensor])."""raise NotImplementedError(f"Unsloth: diffusion profile {self.name} has no training objective.")diff --git a/unsloth/models/diffusion_samplers.py b/unsloth/models/diffusion_samplers.pynew file mode 100644index 00000000000..72e3f5bf86c--- /dev/null+++ b/unsloth/models/diffusion_samplers.py@@ -0,0 +1,561 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Text generation for text-diffusion LMs: one sampler per DiffusionProfile name, each the family's published+decoding recipe. Families shipping a sampler in their modeling code (Dream, LLaDA2, Nemotron-Labs-Diffusion,+DiffusionGemma) are called as is; LLaDA / LLaDA-MoE and SDAR samplers live in separate scripts upstream and are+re-implemented here."""++import contextlib+import math++import torch+import torch.nn.functional as F++from .diffusion_profiles import resolve_diffusion_profile, unwrap_diffusion_model++__all__ = ["DiffusionGenerationCancelled", "generate_text", "diffusion_profile_name"]+++class DiffusionGenerationCancelled(RuntimeError):+ """Raised out of generate_text when stop_callback() returned True mid-generation."""+++_SAMPLERS = {}+++def _register(*names):+ def wrap(function):+ for name in names:+ _SAMPLERS[name] = function+ return function++ return wrap+++def diffusion_profile_name(model):+ """The DiffusionProfile name a loaded (possibly PEFT / DDP wrapped) model was routed with, or None."""+ base = unwrap_diffusion_model(model)+ for candidate in (model, base):+ name = getattr(candidate, "_unsloth_diffusion_profile", None)+ if name is not None:+ return getattr(name, "name", name)+ profile = resolve_diffusion_profile(getattr(base, "config", None))+ return profile.name if profile is not None else None+++def _text_tokenizer(tokenizer):+ return getattr(tokenizer, "tokenizer", tokenizer)+++_UNIFORM_NOISE = frozenset({"diffusion_gemma"})+++def _profile_mask_id(base, tokenizer, name):+ from .diffusion_profiles import diffusion_profiles++ profile = next((p for p in diffusion_profiles() if p.name == name), None)+ candidates = (+ getattr(base, "_unsloth_mask_token_id", None),+ getattr(base, "_unsloth_diffusion_mask_token_id", None),+ getattr(getattr(base, "config", None), "mask_token_id", None),+ getattr(profile, "default_mask_token_id", None),+ (getattr(profile, "defaults", None) or {}).get("mask_token_id"),+ getattr(_text_tokenizer(tokenizer), "mask_token_id", None),+ )+ for value in candidates:+ if value is not None:+ return int(value)+ if name in _UNIFORM_NOISE:+ return None # uniform-noise families (DiffusionGemma) have no [MASK] token to strip+ raise ValueError(f"Unsloth: no [MASK] token id for the {name} sampler.")+++def _eos_ids(base, tokenizer):+ ids = []+ for source in (+ getattr(base, "generation_config", None),+ getattr(base, "config", None),+ _text_tokenizer(tokenizer),+ ):+ value = getattr(source, "eos_token_id", None)+ for item in value if isinstance(value, (list, tuple)) else (value,):+ if item is not None and int(item) not in ids:+ ids.append(int(item))+ return ids+++_TURN_PROBE = "Unsloth turn probe"+++def _turn_end_ids(tokenizer):+ """The special token closing an assistant turn (Dream: <|im_end|>, while its eos_token_id is <|endoftext|>)."""+ text_tokenizer = _text_tokenizer(tokenizer)+ try:+ rendered = text_tokenizer.apply_chat_template(+ [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": _TURN_PROBE}],+ tokenize = False,+ )+ ids = text_tokenizer(rendered.split(_TURN_PROBE, 1)[1], add_special_tokens = False)[+ "input_ids"+ ]+ except Exception:+ return []+ special = set(getattr(text_tokenizer, "all_special_ids", None) or ())+ decoder = getattr(text_tokenizer, "added_tokens_decoder", None) or {}+ special.update(i for i, token in decoder.items() if getattr(token, "special", False))+ return [int(ids[0])] if ids and ids[0] in special else []+++def _device(base):+ try:+ return next(base.parameters()).device+ except StopIteration:+ return torch.device("cpu")+++def _encode(tokenizer, prompt, messages, device):+ if messages is not None:+ text = None+ for source in (tokenizer, _text_tokenizer(tokenizer)):+ try:+ text = source.apply_chat_template(+ messages, add_generation_prompt = True, tokenize = False+ )+ break+ except Exception:+ continue+ if text is None:+ raise ValueError(+ "Unsloth: this tokenizer has no usable chat template; pass prompt= instead."+ )+ add_special_tokens = False+ elif prompt is not None:+ text, add_special_tokens = prompt, True+ else:+ raise ValueError("Unsloth: generate_text needs prompt= or messages=.")+ encoded = _text_tokenizer(tokenizer)(+ text, return_tensors = "pt", add_special_tokens = add_special_tokens+ )+ return encoded["input_ids"].to(device), encoded.get("attention_mask")+++def _round_up(value, multiple):+ return int(math.ceil(value / multiple) * multiple)+++def _contains_any(tokens, ids):+ return bool(ids) and bool(torch.isin(tokens, torch.tensor(ids, device = tokens.device)).any())+++@_register("llada", "llada_moe")[email protected]_grad()+def _llada_sample(+ base,+ tokenizer,+ input_ids,+ attention_mask,+ *,+ max_new_tokens,+ steps,+ block_length,+ temperature,+ mask_id,+ eos_ids,+ remasking = "low_confidence",+ eos_early_stop = True,+ **unused,+):+ """Semi-autoregressive low-confidence remasking (LLaDA generate.py, also LLaDA-MoE's model card):+ blocks of `block_length` masks, a linear schedule unmasking the most confident predictions each step."""+ block_length = int(block_length or 32)+ gen_length = _round_up(max_new_tokens, block_length)+ num_blocks = gen_length // block_length+ steps = _round_up(int(steps or gen_length), num_blocks)+ steps_per_block = steps // num_blocks+ batch, prompt_length = input_ids.shape+ x = torch.full(+ (batch, prompt_length + gen_length), mask_id, dtype = torch.long, device = input_ids.device+ )+ x[:, :prompt_length] = input_ids+ for block in range(num_blocks):+ start = prompt_length + block * block_length+ end = start + block_length+ remaining = (x[:, start:end] == mask_id).sum(dim = 1, keepdim = True)+ schedule = remaining // steps_per_block + (+ torch.arange(steps_per_block, device = x.device)[None, :] < remaining % steps_per_block+ )+ for step in range(steps_per_block):+ mask_index = x == mask_id+ logits = base(x).logits+ if temperature > 0:+ # Gumbel-max in float64: low precision noise hurts MDM generation quality (arXiv:2409.02908).+ noisy = logits.to(torch.float64)+ noisy = noisy.exp() / (-torch.log(torch.rand_like(noisy))) ** temperature+ else:+ noisy = logits+ x0 = noisy.argmax(dim = -1)+ if remasking == "low_confidence":+ probs = F.softmax(logits, dim = -1)+ x0_p = probs.gather(-1, x0.unsqueeze(-1)).squeeze(-1)+ elif remasking == "random":+ x0_p = torch.rand(x0.shape, device = x0.device)+ else:+ raise ValueError(f"Unsloth: unknown LLaDA remasking {remasking!r}.")+ x0_p[:, end:] = -math.inf+ x0 = torch.where(mask_index, x0, x)+ confidence = torch.where(mask_index, x0_p, -math.inf)+ transfer = torch.zeros_like(x0, dtype = torch.bool)+ for row in range(batch):+ _, index = torch.topk(confidence[row], k = int(schedule[row, step]))+ transfer[row, index] = True+ x[transfer] = x0[transfer]+ if eos_early_stop and _contains_any(x[:, prompt_length:end], eos_ids):+ # Later blocks never change earlier ones, so stopping here only drops text after the EOS.+ return x[:, prompt_length:end]+ return x[:, prompt_length:]+++def _block_sample(+ base,+ tokenizer,+ input_ids,+ attention_mask,+ *,+ max_new_tokens,+ steps,+ block_length,+ temperature,+ mask_id,+ eos_ids,+ mask_format,+ confidence_threshold,+ top_k = 0,+ top_p = 1.0,+ **unused,+):+ """Block diffusion decoding shared by SDAR (JetAstra/SDAR generate.py, MIT, low_confidence_dynamic) and+ LLaDA2 (its modeling generate, Apache-2.0): blocks aligned to absolute positions, block-causal attention,+ every token above `confidence_threshold` accepted at once, else the scheduled most confident ones. The+ visible prefix is recomputed each step (as LLaDA2 does) instead of SDAR's custom KV cache. temperature 0+ is greedy."""+ prompt_length = input_ids.shape[1]+ num_blocks = (prompt_length + max_new_tokens + block_length - 1) // block_length+ total_length = num_blocks * block_length+ device = input_ids.device+ block_causal = torch.tril(torch.ones(num_blocks, num_blocks, device = device, dtype = torch.bool))+ allowed = block_causal.repeat_interleave(block_length, 0).repeat_interleave(block_length, 1)[+ None+ ]+ if mask_format == "additive":+ dtype = next(base.parameters()).dtype+ attention = torch.zeros(allowed.shape, dtype = dtype, device = device)+ attention = attention.masked_fill(~allowed, torch.finfo(dtype).min)[:, None]+ else:+ attention = allowed+ position_ids = torch.arange(total_length, device = device)[None]+ x = torch.full((input_ids.shape[0], total_length), mask_id, dtype = torch.long, device = device)+ x[:, :prompt_length] = input_ids+ schedule = torch.full((steps,), block_length // steps, dtype = torch.long)+ schedule[: block_length % steps] += 1+ for block in range(prompt_length // block_length, num_blocks):+ start, end = block * block_length, (block + 1) * block_length+ for step in range(steps + 1):+ mask_index = x[:, start:end] == mask_id+ if not bool(mask_index.any()):+ break+ logits = (+ base(+ x[:, :end],+ attention_mask = attention[..., :end, :end],+ position_ids = position_ids[:, :end],+ )+ .logits[:, start:end]+ .float()+ )+ if temperature > 0:+ logits = logits / temperature+ if top_k and top_k > 0:+ kth = torch.topk(logits, top_k).values[..., -1, None]+ logits = logits.masked_fill(logits < kth, -math.inf)+ if top_p is not None and top_p < 1.0:+ sorted_logits, order = torch.sort(logits, descending = True)+ drop = F.softmax(sorted_logits, dim = -1).cumsum(-1) > top_p+ drop[..., 1:] = drop[..., :-1].clone()+ drop[..., 0] = False+ logits = logits.masked_fill(drop.scatter(-1, order, drop), -math.inf)+ probs = F.softmax(logits, dim = -1)+ x0 = torch.multinomial(probs.flatten(0, 1), 1).view(probs.shape[:-1])+ else:+ probs = F.softmax(logits, dim = -1)+ x0 = probs.argmax(dim = -1)+ x0_p = probs.gather(-1, x0.unsqueeze(-1)).squeeze(-1)+ confidence = torch.where(mask_index, x0_p, -math.inf)+ count = int(schedule[min(step, steps - 1)])+ transfer = torch.zeros_like(mask_index)+ for row in range(x.shape[0]):+ high = confidence[row] > confidence_threshold+ if int(high.sum()) >= count:+ transfer[row] = high+ else:+ # Never more than the masks left: the official top-k could overwrite prompt tokens sharing+ # the first decoded block.+ k = min(count, int(mask_index[row].sum()))+ transfer[row, torch.topk(confidence[row], k).indices] = True+ window = x[:, start:end]+ window[transfer] = x0[transfer]+ if _contains_any(x[:, prompt_length:end], eos_ids):+ return x[:, prompt_length:end]+ return x[:, prompt_length:]+++@_register("sdar")[email protected]_grad()+def _sdar_sample(base, tokenizer, input_ids, attention_mask, *, steps, block_length, **kwargs):+ """SDAR model card / generate.py defaults: block 4, steps = block length, threshold 0.85."""+ block_length = int(block_length or getattr(base.config, "block_size", None) or 4)+ kwargs.setdefault("confidence_threshold", 0.85)+ return _block_sample(+ base,+ tokenizer,+ input_ids,+ attention_mask,+ steps = int(steps or block_length),+ block_length = block_length,+ mask_format = "bool",+ **kwargs,+ )+++@_register("llada2")[email protected]_grad()+def _llada2_sample(+ base,+ tokenizer,+ input_ids,+ attention_mask,+ *,+ steps,+ block_length,+ threshold = 0.95,+ **kwargs,+):+ """LLaDA2 model card defaults: block 32, steps 32, threshold 0.95. Its own generate builds a bfloat16 mask+ (fails on fp16 / fp32 weights) and samples even at temperature 0, so the shared block sampler runs instead."""+ kwargs.setdefault("confidence_threshold", threshold)+ return _block_sample(+ base,+ tokenizer,+ input_ids,+ attention_mask,+ steps = int(steps or 32),+ block_length = int(block_length or 32),+ mask_format = "additive",+ **kwargs,+ )+++@_register("dream")[email protected]_grad()+def _dream_sample(+ base,+ tokenizer,+ input_ids,+ attention_mask,+ *,+ max_new_tokens,+ steps,+ block_length,+ temperature,+ mask_id,+ eos_ids,+ top_p = None,+ alg = "entropy",+ alg_temp = 0.0,+ **unused,+):+ """Dream's own diffusion_generate (model card: steps = max_new_tokens, alg "entropy", alg_temp 0)."""+ if top_p is None and temperature > 0:+ top_p = 0.95+ out = base.diffusion_generate(+ input_ids,+ attention_mask = None if attention_mask is None else attention_mask.to(input_ids.device),+ max_new_tokens = max_new_tokens,+ steps = int(steps or max_new_tokens),+ temperature = temperature,+ top_p = top_p,+ alg = alg,+ alg_temp = alg_temp,+ output_history = False,+ return_dict_in_generate = True,+ )+ return out.sequences[:, input_ids.shape[1] :]+++@_register("nemotron_labs_diffusion")[email protected]_grad()+def _nemotron_sample(+ base,+ tokenizer,+ input_ids,+ attention_mask,+ *,+ max_new_tokens,+ steps,+ block_length,+ temperature,+ mask_id,+ eos_ids,+ threshold = None,+ **unused,+):+ """Nemotron-Labs-Diffusion's own generate: block diffusion with a causal context between blocks."""+ block_length = int(block_length or getattr(base.config, "block_size", None) or 32)+ output_ids, _nfe = base.generate(+ input_ids,+ max_new_tokens = _round_up(max_new_tokens, block_length),+ block_length = block_length,+ threshold = threshold,+ temperature = temperature,+ eos_token_id = eos_ids[0] if eos_ids else None,+ )+ return output_ids[:, input_ids.shape[1] :]+++@_register("diffusion_gemma")[email protected]_grad()+def _diffusion_gemma_sample(+ base,+ tokenizer,+ input_ids,+ attention_mask,+ *,+ max_new_tokens,+ steps,+ block_length,+ temperature,+ mask_id,+ eos_ids,+ **unused,+):+ """transformers' DiffusionGemma generate (entropy-bound sampler, its own temperature schedule)."""+ kwargs = {"max_new_tokens": max_new_tokens}+ if steps is not None:+ kwargs["max_denoising_steps"] = int(steps)+ if attention_mask is not None:+ kwargs["attention_mask"] = attention_mask.to(input_ids.device)+ out = base.generate(input_ids, **kwargs)+ return getattr(out, "sequences", out)[:, input_ids.shape[1] :]+++def _cancellation(base, stop_callback):+ if stop_callback is None:+ yield+ return++ def check(+ module,+ args,+ kwargs = None,+ ):+ if stop_callback():+ raise DiffusionGenerationCancelled("Unsloth: diffusion generation cancelled.")++ # Every embedding table: DiffusionGemma's generate calls encoder and decoder directly, bypassing forward.+ targets = [m for m in base.modules() if isinstance(m, torch.nn.Embedding)] or [base]+ handles = [m.register_forward_pre_hook(check, with_kwargs = True) for m in targets]+ try:+ yield+ finally:+ for handle in handles:+ handle.remove()+++def _restore_attention_modes(base):+ """Nemotron-Labs-Diffusion's generate flips each layer's `self_attn.diffusion_lm` for its causal prefill+ and flips it back at the end; a cancelled generate would leave the model causal. Put them back."""+ saved = [+ (module, module.diffusion_lm)+ for module in base.modules()+ if isinstance(getattr(module, "diffusion_lm", None), bool)+ ]+ try:+ yield+ finally:+ for module, value in saved:+ module.diffusion_lm = value+++def _eval_mode(module):+ was_training = module.training+ module.eval()+ try:+ yield+ finally:+ module.train(was_training)+++def generate_text(+ model,+ tokenizer,+ prompt = None,+ messages = None,+ max_new_tokens = 256,+ steps = None,+ block_length = None,+ temperature = 0.0,+ stop_callback = None,+ seed = None,+ **kwargs,+):+ """Decode one completion with the model family's diffusion sampler and return it as text (prompt+ removed, cut at the first EOS, special tokens skipped). stop_callback() is checked before every forward;+ returning True raises DiffusionGenerationCancelled. `steps` / `block_length` default to each family's+ published settings; extra kwargs go to the family sampler (e.g. remasking, threshold, top_p, alg)."""+ name = diffusion_profile_name(model)+ sampler = _SAMPLERS.get(name)+ if sampler is None:+ raise ValueError(+ f"Unsloth: no diffusion sampler for {name!r}; supported: {sorted(_SAMPLERS)}."+ )+ base = unwrap_diffusion_model(model)+ device = _device(base)+ input_ids, attention_mask = _encode(tokenizer, prompt, messages, device)+ eos_ids = _eos_ids(base, tokenizer)+ if messages is not None:+ eos_ids += [i for i in _turn_end_ids(tokenizer) if i not in eos_ids]+ mask_id = _profile_mask_id(base, tokenizer, name)+ # manual_seed reseeds every CUDA device, so snapshot every device a sharded model uses.+ cuda_devices = sorted({p.device.index for p in base.parameters() if p.device.type == "cuda"})+ rng = (+ torch.random.fork_rng(devices = cuda_devices)+ if seed is not None+ else contextlib.nullcontext()+ )+ with rng, _eval_mode(base), _restore_attention_modes(base), _cancellation(base, stop_callback):+ if seed is not None:+ torch.manual_seed(int(seed))+ generated = sampler(+ base,+ tokenizer,+ input_ids,+ attention_mask,+ max_new_tokens = int(max_new_tokens),+ steps = steps,+ block_length = block_length,+ temperature = float(temperature or 0.0),+ mask_id = mask_id,+ eos_ids = eos_ids,+ **kwargs,+ )+ tokens = generated[0][: int(max_new_tokens)].tolist()+ for index, token in enumerate(tokens):+ if token in eos_ids:+ tokens = tokens[:index]+ break+ if mask_id is not None:+ tokens = [token for token in tokens if token != mask_id]+ return _text_tokenizer(tokenizer).decode(tokens, skip_special_tokens = True).strip()diff --git a/unsloth/models/diffusion_trainer.py b/unsloth/models/diffusion_trainer.pyindex 4a56423fb87..2a89effb2a7 100644--- a/unsloth/models/diffusion_trainer.py+++ b/unsloth/models/diffusion_trainer.py@@ -8,9 +8,11 @@ from dataclasses import dataclass, field, fieldsfrom typing import Optionalimport torch-from trl import SFTConfig, SFTTrainer+from trl import SFTConfig-from .diffusion_profiles import resolve_diffusion_profile+from ..trainer import UnslothTrainer+from .diffusion_loss import compile_backbone_layers, diffusion_compile_mode+from .diffusion_profiles import resolve_diffusion_profile, unwrap_diffusion_model__all__ = ["DiffusionConfig", "DiffusionTrainer"]@@ -47,6 +49,19 @@ class DiffusionConfig(SFTConfig):default = None,metadata = {"help": "Family-specific prediction parameterisation (e.g. 'mean', 'mean_loo')."},)+ embedding_learning_rate: Optional[float] = field(+ default = None,+ metadata = {+ "help": "Learning rate for embed_tokens / lm_head in modules_to_save (continued pretraining)."+ },+ )+ diffusion_compile: Optional[str] = field(+ default = "auto",+ metadata = {+ "help": "'auto' compiles the chunked loss on CUDA; 'loss' always; 'full' (or True) also compiles "+ "each backbone layer; False / 'eager' disables. UNSLOTH_COMPILE_DISABLE=1 forces eager."+ },+ )def __post_init__(self):# Packed rows share one bidirectional attention window, so documents would see each other.@@ -66,7 +81,7 @@ def _unwrap_config(model):return None-class DiffusionTrainer(SFTTrainer):+class DiffusionTrainer(UnslothTrainer):def __init__(self,model = None,@@ -97,9 +112,12 @@ class DiffusionTrainer(SFTTrainer):)self.diffusion_profile = profileself._diffusion_metrics = defaultdict(list)+ diffusion_compile_mode(args) # reject a bad value before any worksuper().__init__(model, args, *pos, **kwargs)# The loss is ours, so the Trainer must scale it for gradient accumulation itself.self.model_accepts_loss_kwargs = False+ if diffusion_compile_mode(args) == "full" and profile.compile_backbone:+ compile_backbone_layers(unwrap_diffusion_model(self.model))def compute_loss(self,@@ -112,8 +130,11 @@ class DiffusionTrainer(SFTTrainer):model, inputs, self.args, num_items_in_batch = num_items_in_batch)mode = "train" if model.training else "eval"+ # Kept on device; one host sync per log() instead of one per metric per micro-batch.for key, value in (metrics or {}).items():- self._diffusion_metrics[f"{mode}/{key}"].append(float(value))+ if isinstance(value, torch.Tensor):+ value = value.detach().float().reshape(())+ self._diffusion_metrics[f"{mode}/{key}"].append(value)return (loss, outputs) if return_outputs else lossdef prediction_step(@@ -134,5 +155,10 @@ class DiffusionTrainer(SFTTrainer):for key in [k for k in self._diffusion_metrics if k.startswith(mode + "/")]:values = self._diffusion_metrics.pop(key)if values:- logs[prefix + key.split("/", 1)[1]] = sum(values) / len(values)+ tensors = [v for v in values if isinstance(v, torch.Tensor)]+ total = sum(float(v) for v in values if not isinstance(v, torch.Tensor))+ if tensors:+ device = tensors[0].device+ total += float(torch.stack([v.to(device) for v in tensors]).sum())+ logs[prefix + key.split("/", 1)[1]] = total / len(values)return super().log(logs, *args, **kwargs)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 1b2f350f294..995476ce1fe 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -62,7 +62,7 @@ from .loader_utils import (_is_offline_related_error,raise_if_fast_inference_under_fsdp2,)-import os, contextlib, sys+import os, contextlib, functools, systry:from huggingface_hub import get_token@@ -972,6 +972,26 @@ def _vllm_unavailable_error():)+def _diffusion_peft_kwargs(+ model,+ args,+ kwargs,+ signature_of = None,+):+ """The caller's get_peft_model arguments, bound against its own signature (FastLanguageModel's by default)+ so positional ones keep their meaning, for FastDiffusionModel.get_peft_model. Omitted ones stay omitted+ (profile defaults apply)."""+ import inspect++ bound = inspect.signature(signature_of or FastLlamaModel.get_peft_model).bind(+ model, *args, **kwargs+ )+ arguments = dict(bound.arguments)+ arguments.pop("model", None)+ arguments.update(arguments.pop("kwargs", {}))+ return arguments++# load_vllm-only kwargs; transformers' from_pretrained rejects them._VLLM_ENGINE_KWARGS = ("max_loras",@@ -995,6 +1015,32 @@ def _drop_vllm_engine_kwargs(kwargs):class FastLanguageModel(FastLlamaModel):+ # Text-diffusion models load through FastModel's slow path; these route them there for the PEFT / mode helpers.+ @staticmethod+ @functools.wraps(FastLlamaModel.get_peft_model)+ def get_peft_model(model, *args, **kwargs):+ if getattr(model, "_unsloth_slow_diffusion", False):+ return FastDiffusionModel.get_peft_model(+ model, **_diffusion_peft_kwargs(model, args, kwargs)+ )+ return FastLlamaModel.get_peft_model(model, *args, **kwargs)++ @staticmethod+ @functools.wraps(FastLlamaModel.for_inference)+ def for_inference(model):+ if getattr(model, "_unsloth_slow_diffusion", False):+ return FastDiffusionModel.for_inference(model)+ return FastLlamaModel.for_inference(model)++ @staticmethod+ @functools.wraps(FastLlamaModel.for_training)+ def for_training(model, use_gradient_checkpointing = None):+ if getattr(model, "_unsloth_slow_diffusion", False):+ if use_gradient_checkpointing is None:+ use_gradient_checkpointing = getattr(model, "_unsloth_gradient_checkpointing", True)+ return FastDiffusionModel.for_training(model, use_gradient_checkpointing)+ return FastLlamaModel.for_training(model, use_gradient_checkpointing)+@staticmethod@tokenizers_v1_on_return@_offline_aware_load@@ -1872,7 +1918,10 @@ class FastModel(FastBaseModel):def get_peft_model(model, *args, **kwargs):# Route text-diffusion models (slow path) to the transformers-only PEFT helper.if getattr(model, "_unsloth_slow_diffusion", False):- return FastDiffusionModel.get_peft_model(model, *args, **kwargs)+ return FastDiffusionModel.get_peft_model(+ model,+ **_diffusion_peft_kwargs(model, args, kwargs, FastBaseModel.get_peft_model),+ )return FastBaseModel.get_peft_model(model, *args, **kwargs)@staticmethod