Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Diffusion LM training: compiled chunked loss with 0 graph breaks, plus per-family samplers (#13341)

Daniel Han6a70df311a40
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.json
index 0e87065b9be..d31bf604ced 100644
--- a/scripts/risky_loaders_baseline.json
+++ b/scripts/risky_loaders_baseline.json
@@ -590,7 +590,7 @@
"file": "unsloth/models/diffusion_masked.py",
"rule": "dynamic-import",
"sink": "get_class_from_dynamic_module",
- "digest": "1900c2baf63f723f",
+ "digest": "9fc70fcf03d7ec35",
"count": 1,
"reason": "Falls back to the auto class unless the caller passed trust_remote_code; revision pinned to the config commit hash."
},
diff --git a/tests/diffusion/test_diffusion_block.py b/tests/diffusion/test_diffusion_block.py
index 0e0964f87c7..91c7c461e0f 100644
--- a/tests/diffusion/test_diffusion_block.py
+++ b/tests/diffusion/test_diffusion_block.py
@@ -161,6 +161,17 @@ def test_left_padding_rejected():
llada2.build_batch(_LM(), inputs, SimpleNamespace())
+def test_left_padding_rejected_after_an_unpadded_first_batch():
+ # An unpadded first batch proves nothing about the padding side, so the check must keep running.
+ llada2, _ = _profiles()
+ model = _LM()
+ llada2.build_batch(model, _batch([2], [12], 12), SimpleNamespace())
+ inputs = _batch([2], [10], 12)
+ inputs["attention_mask"] = inputs["attention_mask"].flip(1)
+ with pytest.raises(ValueError, match = "right padding"):
+ llada2.build_batch(model, inputs, SimpleNamespace())
+
+
def _naive_loss(profile, model, clean, noisy, masked, maskable, p, valid, block):
from unsloth.models.diffusion_block import block_diffusion_attention_mask
@@ -202,7 +213,7 @@ def test_loss_matches_naive_full_forward(which):
def test_noisy_block_never_sees_its_own_or_later_clean_tokens():
llada2, _ = _profiles()
- model, args = _LM(), SimpleNamespace(diffusion_block_size = 4)
+ model = _LM()
clean = torch.randint(0, 29, (1, 12))
noisy = clean.clone()
noisy[:, 4:] = MASK_ID
@@ -265,6 +276,18 @@ def test_sdar_import_shim_is_scoped():
assert "flash_attn" not in sys.modules
+def test_sdar_import_shim_provides_sliding_window_cache(monkeypatch):
+ import transformers.cache_utils as cache_utils
+ from unsloth.models.diffusion_block import _sdar_remote_code_imports
+
+ monkeypatch.delattr(cache_utils, "SlidingWindowCache", raising = False)
+ with _sdar_remote_code_imports():
+ from transformers.cache_utils import Cache, DynamicCache, SlidingWindowCache
+ assert issubclass(SlidingWindowCache, Cache)
+ assert not isinstance(DynamicCache(), SlidingWindowCache)
+ assert not hasattr(cache_utils, "SlidingWindowCache")
+
+
def test_restore_rotary_buffers_refills_zeroed_inv_freq():
from unsloth.models.diffusion_profiles import restore_rotary_buffers
@@ -300,3 +323,27 @@ def test_unknown_time_weighting_is_rejected():
inputs = _batch([3], [20], 20)
with pytest.raises(ValueError, match = "diffusion_time_weighting"):
llada2.compute_loss(_LM(), inputs, SimpleNamespace(diffusion_time_weighting = "inverse-t"))
+
+
+def test_inverse_t_with_zero_noise_floor_stays_finite():
+ # diffusion_eps = 0 can draw p = 0 for a row with nothing masked: no 0 / 0 in its weights.
+ llada2, sdar = _profiles()
+ for profile in (llada2, sdar):
+ model = _LM()
+ inputs = _batch([2, 3], [12, 10], 12)
+ clean, maskable, valid = profile.build_batch(model, inputs, SimpleNamespace())
+ p = torch.tensor([0.0, 0.5])
+ masked = torch.zeros_like(maskable)
+ masked[1, 4] = True
+ noisy = torch.where(masked, 0, clean)
+ loss = profile.loss_from_noise(
+ model,
+ clean,
+ noisy,
+ masked,
+ maskable,
+ p,
+ valid,
+ SimpleNamespace(diffusion_time_weighting = "inverse_t"),
+ )
+ assert torch.isfinite(loss), profile.name
diff --git a/tests/diffusion/test_diffusion_compile.py b/tests/diffusion/test_diffusion_compile.py
new file mode 100644
index 00000000000..88070459514
--- /dev/null
+++ b/tests/diffusion/test_diffusion_compile.py
@@ -0,0 +1,264 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Chunked diffusion CE: parity with dense cross entropy, full-graph capture, no recompiles, sync-free objectives."""
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from unsloth.models import diffusion_loss
+from unsloth.models.diffusion_loss import (
+ chunked_cross_entropy,
+ compile_backbone_layers,
+ diffusion_compile_mode,
+)
+
+
+def _dense(
+ hidden,
+ targets,
+ weights,
+ weight = None,
+ bias = None,
+ softcap = None,
+):
+ z = hidden if weight is None else F.linear(hidden, weight, bias)
+ z = z.float()
+ if softcap is not None:
+ z = torch.tanh(z / softcap) * softcap
+ ce = F.cross_entropy(z.flatten(0, -2), targets.flatten(), reduction = "none", ignore_index = -100)
+ return (ce * weights.flatten()).sum(), ce.sum()
+
+
+def small_chunks(monkeypatch):
+ monkeypatch.setattr(diffusion_loss, "_CHUNK_BYTES", 4 * 11 * 8)
+ monkeypatch.setattr(diffusion_loss, "_MIN_ROWS", 4)
+
+
[email protected]("project", [True, False])
[email protected]("softcap", [None, 3.0])
+def test_chunked_ce_matches_dense_values_and_grads(small_chunks, project, softcap):
+ torch.manual_seed(0)
+ batch, length, width, vocab = 3, 13, 8, 11
+ targets = torch.randint(0, vocab, (batch, length))
+ targets[0, :4] = -100
+ targets[2] = -100 # a fully ignored row
+ weights = torch.rand(batch, length) * (targets != -100)
+ if project:
+ hidden = torch.randn(batch, length, width, requires_grad = True)
+ weight = torch.randn(vocab, width, requires_grad = True)
+ bias = torch.randn(vocab, requires_grad = True)
+ params = [hidden, weight, bias]
+ else:
+ hidden = torch.randn(batch, length, vocab, requires_grad = True)
+ weight = bias = None
+ params = [hidden]
+ ours, ours_ce, count = chunked_cross_entropy(hidden, targets, weights, weight, bias, softcap)
+ ref, ref_ce = _dense(hidden, targets, weights, weight, bias, softcap)
+ torch.testing.assert_close(ours, ref, rtol = 1e-5, atol = 1e-6)
+ torch.testing.assert_close(ours_ce, ref_ce, rtol = 1e-5, atol = 1e-6)
+ assert int(count) == int((targets != -100).sum())
+ for a, b in zip(torch.autograd.grad(ours, params), torch.autograd.grad(ref, params)):
+ torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)
+
+
+def test_chunked_ce_all_ignored_is_zero_with_grad():
+ hidden = torch.randn(2, 5, 7, requires_grad = True)
+ weighted, total, count = chunked_cross_entropy(hidden, torch.full((2, 5), -100))
+ assert float(weighted) == 0.0 and float(total) == 0.0 and int(count) == 0
+ (grad,) = torch.autograd.grad(weighted, hidden)
+ assert torch.count_nonzero(grad) == 0
+
+
+def test_chunked_ce_frozen_head_gets_no_grad():
+ hidden = torch.randn(2, 5, 4, requires_grad = True)
+ head = torch.nn.Linear(4, 9, bias = False).requires_grad_(False)
+ weighted, _, _ = chunked_cross_entropy(hidden, torch.randint(0, 9, (2, 5)), weight = head.weight)
+ weighted.backward()
+ assert hidden.grad is not None and head.weight.grad is None
+
+
+def _kernel_args(
+ rows = 16,
+ width = 8,
+ vocab = 11,
+):
+ return (
+ torch.randn(rows, width),
+ torch.randn(vocab, width),
+ None,
+ torch.randint(0, vocab, (rows,)),
+ torch.rand(rows),
+ )
+
+
+def test_chunk_kernels_capture_without_graph_breaks():
+ h, w, b, t, tw = _kernel_args()
+ forward = torch._dynamo.explain(diffusion_loss._chunk_forward)(h, w, b, t, tw, 3.0)
+ assert forward.graph_break_count == 0 and forward.graph_count == 1
+ lse = torch.logsumexp(F.linear(h, w).float(), -1)
+ backward = torch._dynamo.explain(diffusion_loss._chunk_backward)(
+ h, w, b, t, tw, lse, torch.tensor(1.0), 3.0
+ )
+ assert backward.graph_break_count == 0 and backward.graph_count == 1
+
+
+def _compiled_kernels_or_skip():
+ diffusion_loss._COMPILED.clear()
+ try:
+ forward, _ = diffusion_loss._kernels(True)
+ with torch.no_grad(): # as inside the autograd.Function, so the probe shares its guards
+ forward(*_kernel_args(), None)
+ except Exception as error: # no C++ toolchain for inductor on this host
+ pytest.skip(f"torch.compile unavailable here: {type(error).__name__}")
+ diffusion_loss._COMPILED.clear()
+ torch._dynamo.reset()
+
+
+def test_compiled_loss_matches_eager_and_never_recompiles_on_new_masks():
+ _compiled_kernels_or_skip()
+ torch.manual_seed(0)
+ hidden = torch.randn(4, 9, 8, requires_grad = True)
+ weight = torch.randn(11, 8, requires_grad = True)
+ with torch._dynamo.config.patch(error_on_recompile = True):
+ for fraction in (0.1, 0.5, 0.9, 0.0):
+ targets = torch.randint(0, 11, (4, 9)).masked_fill(torch.rand(4, 9) > fraction, -100)
+ weights = torch.rand(4, 9) * (targets != -100)
+ eager, _, _ = chunked_cross_entropy(hidden, targets, weights, weight)
+ compiled, _, _ = chunked_cross_entropy(hidden, targets, weights, weight, compiled = True)
+ torch.testing.assert_close(compiled, eager, rtol = 1e-5, atol = 1e-6)
+ for a, b in zip(
+ torch.autograd.grad(compiled, [hidden, weight]),
+ torch.autograd.grad(eager, [hidden, weight]),
+ ):
+ torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)
+
+
+def test_compile_mode_parsing(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_COMPILE_DISABLE", raising = False)
+ assert diffusion_compile_mode(None) == "auto"
+ assert diffusion_compile_mode(SimpleNamespace(diffusion_compile = False)) is None
+ assert diffusion_compile_mode(SimpleNamespace(diffusion_compile = "eager")) is None
+ assert diffusion_compile_mode(SimpleNamespace(diffusion_compile = True)) == "full"
+ assert diffusion_compile_mode(SimpleNamespace(diffusion_compile = "loss")) == "loss"
+ with pytest.raises(ValueError, match = "diffusion_compile"):
+ diffusion_compile_mode(SimpleNamespace(diffusion_compile = "fast"))
+ from unsloth.models.diffusion_loss import compile_loss
+
+ auto, loss = (
+ SimpleNamespace(diffusion_compile = "auto"),
+ SimpleNamespace(diffusion_compile = "loss"),
+ )
+ assert compile_loss(auto, "cuda") and not compile_loss(auto, "cpu")
+ assert not compile_loss(auto, "cuda", auto = False) and compile_loss(loss, "cuda", auto = False)
+ monkeypatch.setenv("UNSLOTH_COMPILE_DISABLE", "1")
+ assert diffusion_compile_mode(SimpleNamespace(diffusion_compile = True)) is None
+
+
+def test_compile_backbone_layers_is_idempotent():
+ class _Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = torch.nn.ModuleList(torch.nn.Linear(4, 4) for _ in range(3))
+ self.head = torch.nn.Linear(4, 4)
+
+ net = _Net()
+ assert compile_backbone_layers(net) == 3
+ assert compile_backbone_layers(net) == 0
+ assert all(getattr(layer, "_unsloth_diffusion_compiled", False) for layer in net.layers)
+
+
+def test_chunked_ce_backward_follows_forward_autocast():
+ # fp32 weights under bf16 autocast: backward must rebuild the bf16 logits the forward normalised.
+ torch.manual_seed(0)
+ hidden = torch.randn(2, 6, 8, requires_grad = True)
+ weight = torch.randn(13, 8, requires_grad = True)
+ targets = torch.randint(0, 13, (2, 6))
+ with torch.autocast("cpu", dtype = torch.bfloat16):
+ ours, _, _ = chunked_cross_entropy(hidden, targets, weight = weight)
+ ref = F.cross_entropy(
+ F.linear(hidden, weight).float().flatten(0, 1), targets.flatten(), reduction = "sum"
+ )
+ torch.testing.assert_close(ours, ref, rtol = 1e-5, atol = 1e-5)
+ for a, b in zip(
+ torch.autograd.grad(ours, [hidden, weight]), torch.autograd.grad(ref, [hidden, weight])
+ ):
+ torch.testing.assert_close(a, b, rtol = 1e-3, atol = 1e-3)
+
+
+def test_chunked_ce_empty_input_is_zero_with_grad():
+ # DiffusionGemma's encoder AR slice of a one-token batch has no rows at all.
+ hidden = torch.randn(0, 8, requires_grad = True)
+ weight = torch.randn(11, 8, requires_grad = True)
+ weighted, total, count = chunked_cross_entropy(
+ hidden, torch.empty(0, dtype = torch.long), weight = weight
+ )
+ weighted.backward()
+ assert float(weighted) == 0.0 and int(count) == 0
+ assert weight.grad is not None and float(weight.grad.abs().sum()) == 0.0
+
+
+def test_chunked_ce_leaves_the_autocast_cache_alone():
+ # A later projection of the same weight in the same autocast region must still reach the weight (older
+ # torch reused the gradient-less cast made inside the custom forward).
+ device = "cuda" if torch.cuda.is_available() else "cpu"
+ torch.manual_seed(0)
+ weight = torch.randn(11, 8, device = device, requires_grad = True)
+ hidden = torch.randn(5, 8, device = device)
+ targets = torch.randint(0, 11, (5,), device = device)
+
+ def grads(with_ce):
+ weight.grad = None
+ with torch.autocast(device, dtype = torch.bfloat16):
+ total = F.linear(hidden, weight).float().sum() * 0
+ if with_ce:
+ total = total + chunked_cross_entropy(hidden, targets, weight = weight)[0]
+ total = total + F.linear(hidden, weight).float().sum()
+ total.backward()
+ return weight.grad.clone()
+
+ ce_only = torch.func.grad(
+ lambda w: _dense(hidden, targets, (targets >= 0).float(), weight = w)[0]
+ )(weight.detach())
+ torch.testing.assert_close(grads(True) - grads(False), ce_only, rtol = 5e-2, atol = 5e-2)
+
+
+def test_chunked_ce_ignored_overflow_rows_do_not_poison_grads():
+ hidden = torch.tensor([[1.0, 0.0], [float("inf"), 0.0]], requires_grad = True)
+ targets = torch.tensor([0, -100])
+ weighted, _, _ = chunked_cross_entropy(hidden, targets)
+ weighted.backward()
+ assert torch.isfinite(hidden.grad).all()
+
+
+def test_embedding_learning_rate_gets_its_own_optimizer_group(tmp_path):
+ import torch.nn as nn
+ from unsloth.models.diffusion_trainer import DiffusionConfig, DiffusionTrainer
+
+ class _Model(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embed = nn.Module()
+ self.embed.modules_to_save = nn.ModuleDict({"default": nn.Embedding(8, 4)})
+ self.lora_A = nn.Linear(4, 4, bias = False)
+
+ trainer = DiffusionTrainer.__new__(DiffusionTrainer)
+ trainer.model, trainer.optimizer = _Model(), None
+ trainer.args = DiffusionConfig(
+ output_dir = str(tmp_path),
+ learning_rate = 1e-3,
+ embedding_learning_rate = 1e-5,
+ report_to = [],
+ use_cpu = True,
+ bf16 = False,
+ fp16 = False,
+ )
+ groups = trainer.create_optimizer().param_groups
+ embedding = trainer.model.embed.modules_to_save["default"].weight
+ lrs = {group["lr"] for group in groups if any(p is embedding for p in group["params"])}
+ assert lrs == {1e-5}
+ assert any(group["lr"] == 1e-3 for group in groups)
diff --git a/tests/diffusion/test_diffusion_gemma.py b/tests/diffusion/test_diffusion_gemma.py
index 85a28393f4e..590360cacd9 100644
--- a/tests/diffusion/test_diffusion_gemma.py
+++ b/tests/diffusion/test_diffusion_gemma.py
@@ -92,6 +92,18 @@ def test_parity_with_trl_reference(model, prediction_type):
"from trl.trainer.utils import maybe_gather_lm_head_ctx",
"import contextlib\nmaybe_gather_lm_head_ctx = lambda *a: contextlib.nullcontext()",
)
+ if prediction_type == "mean_loo":
+ # mean_loo lifts one logit by ~log(K / (1 - alpha)); fp32 F.cross_entropy over the 262k vocab is then
+ # ~6e-7 off the fp64 logit gradient (our chunked CE ~1e-8). Score the reference in fp64 instead.
+ source = source.replace(
+ "import torch.nn.functional as F",
+ "import torch.nn.functional as _F\n"
+ "class F:\n"
+ " def __getattr__(self, name): return getattr(_F, name)\n"
+ " @staticmethod\n"
+ " def cross_entropy(x, *a, **k): return _F.cross_entropy(x.double(), *a, **k).float()\n"
+ "F = F()",
+ )
namespace = {"__name__": "trl_example"}
exec(compile(source, TRL_EXAMPLE, "exec"), namespace)
reference = namespace["DiffusionGemmaSFTTrainer"]
@@ -123,12 +135,14 @@ def test_parity_with_trl_reference(model, prediction_type):
model, batch, _args(diffusion_prediction_type = prediction_type)
)[0]
)
- torch.testing.assert_close(our_loss, ref_loss, rtol = 0, atol = 1e-6)
+ torch.testing.assert_close(our_loss, ref_loss, rtol = 1e-6, atol = 1e-6)
for a, b in zip(our_grads, ref_grads):
if a is None or b is None:
assert a is None and b is None
- else:
+ elif prediction_type == "mean":
torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-7)
+ else:
+ torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)
def test_tied_encoder_decoder_lora_merge_is_refused():
@@ -190,6 +204,16 @@ def test_row_without_supervised_tokens_adds_no_canvas_loss(model):
assert metrics["diffusion_loss"] == 0.0
[email protected]("rows", [[(1, 0)], [(1, 0), (0, 0)]])
+def test_batch_without_encoder_ar_targets_is_finite(model, rows):
+ # A one-token or all-padding batch has no next-token pair: the AR co-loss is 0, not a NaN mean.
+ batch = _batch(model.config.text_config.vocab_size, model.config.canvas_length, rows = rows)
+ torch.manual_seed(0)
+ loss, _, metrics = DIFFUSION_GEMMA_PROFILE.compute_loss(model, batch, _args())
+ assert torch.isfinite(loss) and loss.requires_grad
+ assert float(metrics["encoder_ar_loss"]) == 0.0 and float(metrics["diffusion_loss"]) == 0.0
+
+
def test_reloaded_adapter_still_refuses_merge(tmp_path):
from peft import LoraConfig, get_peft_model
diff --git a/tests/diffusion/test_diffusion_masked.py b/tests/diffusion/test_diffusion_masked.py
index 06b8befbdc0..b23b20941db 100644
--- a/tests/diffusion/test_diffusion_masked.py
+++ b/tests/diffusion/test_diffusion_masked.py
@@ -394,6 +394,68 @@ def test_mask_builder_signature_exposed_from_wrapper_closure():
assert list(inspect.signature(wrapper).parameters)[:2] == ["config", "inputs_embeds"]
+_REMOTE_CONFIG = """
+from transformers import PretrainedConfig
+
+
+class TinyRemoteConfig(PretrainedConfig):
+ model_type = "tiny_remote_diffusion"
+"""
+
+_REMOTE_MODELING = """
+import torch
+from transformers import PreTrainedModel
+from .configuration_tiny_remote import TinyRemoteConfig
+
+
+class TinyRemoteModel(PreTrainedModel):
+ config_class = TinyRemoteConfig
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.proj = torch.nn.Linear(4, 4, bias = False)
+ self.post_init()
+
+ def _init_weights(self, module):
+ # 4.x-era hub init writes in place, blind to which weights the checkpoint already filled.
+ if isinstance(module, torch.nn.Linear):
+ module.weight.data.normal_(0.0, 1.0)
+
+ def forward(self, x):
+ return self.proj(x)
+"""
+
+
+def test_remote_class_keeps_checkpoint_weights(tmp_path):
+ import json
+
+ from safetensors.torch import save_file
+ from transformers import AutoConfig
+
+ from unsloth.models.diffusion_masked import MaskedDiffusionProfile
+
+ (tmp_path / "configuration_tiny_remote.py").write_text(_REMOTE_CONFIG)
+ (tmp_path / "modeling_tiny_remote.py").write_text(_REMOTE_MODELING)
+ auto_map = {
+ "AutoConfig": "configuration_tiny_remote.TinyRemoteConfig",
+ "AutoModelForCausalLM": "modeling_tiny_remote.TinyRemoteModel",
+ }
+ (tmp_path / "config.json").write_text(
+ json.dumps({"model_type": "tiny_remote_diffusion", "auto_map": auto_map})
+ )
+ weight = torch.arange(16, dtype = torch.float32).reshape(4, 4)
+ save_file(
+ {"proj.weight": weight}, str(tmp_path / "model.safetensors"), metadata = {"format": "pt"}
+ )
+
+ config = AutoConfig.from_pretrained(str(tmp_path), trust_remote_code = True)
+ profile = MaskedDiffusionProfile(name = "tiny_remote_test", requires_remote_code = True)
+ cls = profile.model_class(config, True)
+ model = cls.from_pretrained(str(tmp_path), config = config, trust_remote_code = True)
+ assert torch.equal(model.proj.weight.detach(), weight)
+ assert cls.is_remote_code()
+
+
def test_converted_config_keeps_token_fields(monkeypatch, tmp_path):
from transformers import TrainingArguments
from trl import SFTConfig
@@ -431,6 +493,18 @@ path, out = sys.argv[1], sys.argv[2]
device = os.environ.get("TINY_DEVICE", "cpu")
model, processor = FastModel.from_pretrained(path, trust_remote_code = True, dtype = torch.float32, device_map = {"": device})
assert getattr(model, "_unsloth_slow_diffusion", False), type(model)
+import glob
+from safetensors import safe_open
+state, matched = model.state_dict(), 0
+for shard in glob.glob(os.path.join(path, "*.safetensors")):
+ with safe_open(shard, "pt") as f:
+ for key in f.keys():
+ # A GPU load defaults to 4-bit: packed linear weights cannot equal the checkpoint, the rest must.
+ if key not in state or tuple(state[key].shape) != tuple(f.get_slice(key).get_shape()):
+ continue
+ assert torch.equal(f.get_tensor(key).to(state[key].dtype), state[key].detach().cpu()), key
+ matched += 1
+assert matched, "no checkpoint tensor matched the model's names"
tokenizer = getattr(processor, "tokenizer", processor)
for module in model.modules():
if hasattr(module, "rope_init_fn") and isinstance(getattr(module, "inv_freq", None), torch.Tensor):
@@ -451,7 +525,7 @@ def fixed_noise_ce():
# Per-step losses are too noisy over 20 steps (1 / t weight, fresh noise); score one batch under one draw.
torch.manual_seed(0)
with torch.no_grad():
- return trainer.diffusion_profile.compute_loss(model, fixed, args)[2]["masked_ce"]
+ return float(trainer.diffusion_profile.compute_loss(model, fixed, args)[2]["masked_ce"])
first = fixed_noise_ce()
@@ -500,3 +574,48 @@ def test_tiny_remote_checkpoint_trains_and_reloads(name, tmp_path):
)
assert out.returncode == 0, out.stdout[-3000:] + out.stderr[-3000:]
assert "TINY_OK" in out.stdout
+
+
+class _WithHead(_Bidirectional):
+ def get_output_embeddings(self):
+ return self.head
+
+
[email protected]("name", ["llada", "dream", "nemotron_labs_diffusion"])
+def test_hidden_state_loss_matches_dense_logits(name):
+ # The fused path swaps the head for its input and projects per chunk; it must equal the logits path.
+ inputs, _ = _batch()
+ fused, dense = _WithHead(), _Bidirectional()
+ dense.load_state_dict(fused.state_dict())
+ fused_loss, _ = _ours(name, fused, inputs, 3)
+ dense_loss, _ = _ours(name, dense, inputs, 3)
+ assert fused.__dict__["_unsloth_diffusion_fused_head"] == (fused.head, fused.head)
+ assert "forward" not in vars(fused.head) # the swap is undone
+ torch.testing.assert_close(fused_loss, dense_loss, rtol = 1e-6, atol = 1e-6)
+ fused_loss.backward()
+ dense_loss.backward()
+ for (n, a), b in zip(fused.named_parameters(), dense.parameters()):
+ torch.testing.assert_close(a.grad, b.grad, rtol = 1e-5, atol = 1e-6, msg = n)
+
+
+def test_dream_zero_noise_level_is_finite(monkeypatch):
+ # Dream draws t ~ U(0, 1) with no floor and weights by 1 / t: t = 0 masks nothing and must not give inf * 0.
+ inputs, _ = _batch()
+ monkeypatch.setattr(
+ torch,
+ "rand",
+ lambda *a, **k: torch.zeros(*a, **{x: y for x, y in k.items() if x != "generator"}),
+ )
+ loss, metrics = _ours("dream", _Bidirectional(), inputs, 0)
+ assert torch.isfinite(loss) and float(loss) == 0.0 and loss.requires_grad
+ assert float(metrics["masked_fraction"]) == 0.0
+
+
+def test_fused_head_rechecked_after_head_is_replaced():
+ # A head wrapped after the first loss call (e.g. PEFT modules_to_save) must not reuse the cached decision.
+ from unsloth.models.diffusion_loss import fusable_head
+
+ model = _WithHead()
+ assert fusable_head(model, model) is model.head
+ model.head = torch.nn.Sequential(model.head)
+ assert fusable_head(model, model) is None
diff --git a/tests/diffusion/test_diffusion_samplers.py b/tests/diffusion/test_diffusion_samplers.py
new file mode 100644
index 00000000000..8eef60a0a9d
--- /dev/null
+++ b/tests/diffusion/test_diffusion_samplers.py
@@ -0,0 +1,538 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FastLanguageModel routes text-diffusion models to the diffusion PEFT / mode helpers, and generate_text decodes
+with each family's sampler."""
+
+import importlib.util
+import os
+import subprocess
+import sys
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+# The LLaDA sampler is re-implemented; check it against the official one when its generate.py is available.
+_LLADA_GENERATE = os.environ.get("UNSLOTH_LLADA_GENERATE_PY")
+_TINY = os.environ.get("UNSLOTH_DIFFUSION_TINY_DIR")
+
+VOCAB, MASK_ID, EOS_ID = 61, 60, 2
+
+
+class _TinyMaskPredictor(torch.nn.Module):
+ """Bidirectional toy predictor: every logit depends on the whole row."""
+
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.embed = torch.nn.Embedding(VOCAB, 16)
+ self.mix = torch.nn.Linear(16, 16)
+ self.head = torch.nn.Linear(16, VOCAB)
+ self.config = SimpleNamespace(mask_token_id = MASK_ID, eos_token_id = EOS_ID)
+ self.calls = 0
+
+ @property
+ def device(self):
+ return self.head.weight.device
+
+ def forward(
+ self,
+ input_ids,
+ attention_mask = None,
+ ):
+ self.calls += 1
+ h = self.embed(input_ids)
+ h = h + self.mix(h.mean(dim = 1, keepdim = True))
+ logits = self.head(torch.tanh(h))
+ logits[..., MASK_ID] = -1e4
+ return SimpleNamespace(logits = logits)
+
+
+def _load_official_generate():
+ spec = importlib.util.spec_from_file_location("llada_generate", _LLADA_GENERATE)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module.generate
+
+
+ not _LLADA_GENERATE, reason = "set UNSLOTH_LLADA_GENERATE_PY to LLaDA's generate.py"
+)
[email protected]("temperature", [0.0, 0.7])
[email protected]("remasking", ["low_confidence", "random"])
+def test_llada_sampler_matches_official(temperature, remasking):
+ from unsloth.models.diffusion_samplers import _llada_sample
+
+ official = _load_official_generate()
+ model = _TinyMaskPredictor()
+ prompt = torch.tensor([[1, 7, 9, 11, 13]])
+ torch.manual_seed(1)
+ expected = official(
+ model,
+ prompt,
+ steps = 8,
+ gen_length = 16,
+ block_length = 8,
+ temperature = temperature,
+ remasking = remasking,
+ mask_id = MASK_ID,
+ )[:, prompt.shape[1] :]
+ torch.manual_seed(1)
+ got = _llada_sample(
+ model,
+ None,
+ prompt,
+ None,
+ max_new_tokens = 16,
+ steps = 8,
+ block_length = 8,
+ temperature = temperature,
+ mask_id = MASK_ID,
+ eos_ids = [],
+ remasking = remasking,
+ )
+ assert torch.equal(got, expected), (got, expected)
+
+
+def test_llada_sampler_stops_after_eos_block():
+ from unsloth.models.diffusion_samplers import _llada_sample
+
+ model = _TinyMaskPredictor()
+ prompt = torch.tensor([[1, 7, 9]])
+ full = _llada_sample(
+ model,
+ None,
+ prompt,
+ None,
+ max_new_tokens = 32,
+ steps = 32,
+ block_length = 8,
+ temperature = 0.0,
+ mask_id = MASK_ID,
+ eos_ids = [],
+ eos_early_stop = False,
+ )
+ tokens = full[0].tolist()
+ eos = next(t for t in tokens[:8] if t != MASK_ID)
+ calls = model.calls
+ early = _llada_sample(
+ model,
+ None,
+ prompt,
+ None,
+ max_new_tokens = 32,
+ steps = 32,
+ block_length = 8,
+ temperature = 0.0,
+ mask_id = MASK_ID,
+ eos_ids = [eos],
+ )
+ # The first block already holds the "EOS", so only its 8 steps ran and its tokens are unchanged.
+ assert model.calls - calls == 8
+ assert torch.equal(early, full[:, :8])
+
+
+def test_fast_language_model_routes_diffusion_helpers(monkeypatch):
+ from unsloth import FastLanguageModel
+ from unsloth.models import diffusion as diffusion_module
+ from unsloth.models.llama import FastLlamaModel
+
+ seen = {}
+ monkeypatch.setattr(
+ diffusion_module.FastDiffusionModel,
+ "get_peft_model",
+ staticmethod(lambda model, **kwargs: seen.setdefault("peft", kwargs) and model),
+ )
+ monkeypatch.setattr(
+ diffusion_module.FastDiffusionModel,
+ "for_training",
+ staticmethod(
+ lambda model, use_gradient_checkpointing = True: seen.setdefault(
+ "train", use_gradient_checkpointing
+ )
+ or model
+ ),
+ )
+ monkeypatch.setattr(
+ diffusion_module.FastDiffusionModel,
+ "for_inference",
+ staticmethod(lambda model: seen.setdefault("infer", True) and model),
+ )
+ model = torch.nn.Linear(2, 2)
+ model._unsloth_slow_diffusion = True
+ model._unsloth_gradient_checkpointing = False
+ # Positional r and target_modules keep FastLanguageModel's meaning; omitted ones stay omitted.
+ FastLanguageModel.get_peft_model(
+ model, 8, ["q_proj"], lora_alpha = 16, use_rslora = True, offload_layers = None
+ )
+ assert seen["peft"] == {
+ "r": 8,
+ "target_modules": ["q_proj"],
+ "lora_alpha": 16,
+ "use_rslora": True,
+ "offload_layers": None,
+ }
+ FastLanguageModel.for_training(model)
+ assert seen["train"] is False
+ FastLanguageModel.for_inference(model)
+ assert seen["infer"] is True
+ import inspect
+
+ assert inspect.signature(FastLanguageModel.get_peft_model) == inspect.signature(
+ FastLlamaModel.get_peft_model
+ )
+
+
+def test_fast_language_model_autoregressive_path_unchanged(monkeypatch):
+ from unsloth import FastLanguageModel
+ from unsloth.models.llama import FastLlamaModel
+
+ calls = []
+ monkeypatch.setattr(
+ FastLlamaModel,
+ "get_peft_model",
+ staticmethod(lambda *a, **k: calls.append(("peft", a, k)) or "peft"),
+ )
+ monkeypatch.setattr(
+ FastLlamaModel,
+ "for_training",
+ staticmethod(lambda *a, **k: calls.append(("train", a, k)) or "train"),
+ )
+ monkeypatch.setattr(
+ FastLlamaModel,
+ "for_inference",
+ staticmethod(lambda *a, **k: calls.append(("infer", a, k)) or "infer"),
+ )
+ model = torch.nn.Linear(2, 2)
+ assert FastLanguageModel.get_peft_model(model, 4, target_modules = ["x"]) == "peft"
+ assert FastLanguageModel.for_training(model, True) == "train"
+ assert FastLanguageModel.for_inference(model) == "infer"
+ assert calls == [
+ ("peft", (model, 4), {"target_modules": ["x"]}),
+ ("train", (model, True), {}),
+ ("infer", (model,), {}),
+ ]
+
+
[email protected]("bad", [{"use_dora": True}, {"loftq_config": {"loftq_bits": 4}}])
+def test_unsupported_lora_options_refused(bad):
+ from unsloth.models.diffusion import FastDiffusionModel
+ model = torch.nn.Linear(2, 2)
+ with pytest.raises(ValueError, match = "not supported for text-diffusion"):
+ FastDiffusionModel.get_peft_model(model, r = 4, **bad)
+
+
+def test_diffusion_profile_name_and_flag():
+ from unsloth.models.diffusion import is_diffusion_model
+ from unsloth.models.diffusion_samplers import diffusion_profile_name
+
+ model = torch.nn.Linear(2, 2)
+ assert not is_diffusion_model(model) and not is_diffusion_model(None)
+ # Autoregressive Nemotron shares no profile; Nemotron-Labs-Diffusion matches by architecture.
+ model.config = SimpleNamespace(model_type = "nemotron", architectures = ["NemotronForCausalLM"])
+ assert not is_diffusion_model(model)
+ model.config = SimpleNamespace(
+ model_type = "nemotron", architectures = ["NemotronLabsDiffusionModel"]
+ )
+ assert is_diffusion_model(model)
+ assert diffusion_profile_name(model) == "nemotron_labs_diffusion"
+ del model.config
+ model._unsloth_slow_diffusion = True
+ model._unsloth_diffusion_profile = "llada"
+ assert is_diffusion_model(model)
+ assert diffusion_profile_name(model) == "llada"
+
+
+def test_generate_text_cuts_at_eos_and_budget(monkeypatch):
+ from unsloth.models import diffusion_samplers
+
+ class _Tok:
+ eos_token_id = EOS_ID
+ mask_token_id = None
+
+ def __call__(
+ self,
+ text,
+ return_tensors = None,
+ add_special_tokens = True,
+ ):
+ return {
+ "input_ids": torch.tensor([[1, 7]]),
+ "attention_mask": torch.ones(1, 2, dtype = torch.long),
+ }
+
+ def decode(
+ self,
+ tokens,
+ skip_special_tokens = True,
+ ):
+ return " ".join(map(str, tokens))
+
+ model = _TinyMaskPredictor()
+ model._unsloth_diffusion_profile = "fake"
+ produced = torch.tensor([[11, MASK_ID, 12, 13, EOS_ID, 14, 15]])
+ seen = {}
+
+ def fake(base, tokenizer, input_ids, attention_mask, **kwargs):
+ seen.update(kwargs)
+ return produced
+
+ monkeypatch.setitem(diffusion_samplers._SAMPLERS, "fake", fake)
+ text = diffusion_samplers.generate_text(
+ model, _Tok(), prompt = "hi", max_new_tokens = 32, steps = 4, alg = "x"
+ )
+ assert text == "11 12 13"
+ assert (
+ seen["mask_id"] == MASK_ID
+ and seen["eos_ids"] == [EOS_ID]
+ and seen["alg"] == "x"
+ and seen["steps"] == 4
+ )
+ # max_new_tokens caps what is decoded even when a sampler returns more.
+ assert diffusion_samplers.generate_text(model, _Tok(), prompt = "hi", max_new_tokens = 2) == "11"
+
+ # Dream: eos_token_id is <|endoftext|> only, while its chat template closes a turn with <|im_end|>.
+ turn_id = 99
+
+ class _ChatTok(_Tok):
+ all_special_ids = [EOS_ID, turn_id]
+
+ def apply_chat_template(
+ self,
+ messages,
+ add_generation_prompt = False,
+ tokenize = False,
+ ):
+ turns = "".join(f"<{m['role']}>{m['content']}<turn>" for m in messages)
+ return turns + ("<assistant>" if add_generation_prompt else "")
+
+ def __call__(
+ self,
+ text,
+ return_tensors = None,
+ add_special_tokens = True,
+ ):
+ if return_tensors is None:
+ return {"input_ids": [turn_id, 5] if text.startswith("<turn>") else [5]}
+ return super().__call__(text, return_tensors, add_special_tokens)
+
+ produced = torch.tensor([[11, 12, turn_id, 14, 15]])
+ chat = [{"role": "user", "content": "hi"}]
+ assert diffusion_samplers.generate_text(model, _ChatTok(), messages = chat) == "11 12"
+ assert seen["eos_ids"] == [EOS_ID, turn_id]
+ assert diffusion_samplers.generate_text(model, _ChatTok(), prompt = "hi") == "11 12 99 14 15"
+ assert seen["eos_ids"] == [EOS_ID]
+ with pytest.raises(ValueError, match = "no diffusion sampler"):
+ model._unsloth_diffusion_profile = "missing"
+ diffusion_samplers.generate_text(model, _Tok(), prompt = "hi")
+
+
+_PAD_SCRIPT = r"""
+import os, sys
+import torch
+from unsloth import FastModel
+
+path = sys.argv[1]
+model, processor = FastModel.from_pretrained(path, trust_remote_code = True, dtype = torch.float32, device_map = {"": os.environ.get("TINY_DEVICE", "cpu")}, load_in_4bit = False)
+tok = getattr(processor, "tokenizer", processor)
+assert tok.pad_token is not None and tok.pad_token == tok.eos_token, (tok.pad_token, tok.eos_token)
+assert tok.pad_token_id != getattr(model, "_unsloth_mask_token_id", None)
+assert model.config.pad_token_id == tok.pad_token_id
+assert getattr(model.generation_config, "pad_token_id", tok.pad_token_id) == tok.pad_token_id
+batch = tok.pad([{"input_ids": [1, 2, 3]}, {"input_ids": [4]}], return_tensors = "pt")
+assert batch["input_ids"].shape == (2, 3)
+print("PAD_OK", tok.pad_token)
+"""
+
+
+def test_tiny_nemotron_gets_eos_pad_token():
+ path = os.path.join(_TINY or "", "Nemotron-Labs-Diffusion-3B")
+ if not _TINY or not os.path.isdir(path):
+ pytest.skip("set UNSLOTH_DIFFUSION_TINY_DIR to tiny remote-code copies")
+ out = subprocess.run(
+ [sys.executable, "-c", _PAD_SCRIPT, path], capture_output = True, text = True, timeout = 900
+ )
+ assert out.returncode == 0, out.stdout[-3000:] + out.stderr[-4000:]
+ assert "PAD_OK" in out.stdout
+
+
+_TINY_SCRIPT = r"""
+import os, sys
+import torch
+from unsloth import FastLanguageModel
+from unsloth.models.diffusion import FastDiffusionModel, is_diffusion_model
+from unsloth.models.diffusion_samplers import DiffusionGenerationCancelled
+
+path = sys.argv[1]
+device = os.environ.get("TINY_DEVICE", "cpu")
+model, processor = FastLanguageModel.from_pretrained(path, trust_remote_code = True, dtype = torch.float32, device_map = {"": device}, load_in_4bit = False)
+assert is_diffusion_model(model), type(model)
+# Studio's text path: FastLanguageModel.get_peft_model with its usual knobs.
+model = FastLanguageModel.get_peft_model(model, r = 8, lora_alpha = 16, lora_dropout = 0.0, bias = "none", use_gradient_checkpointing = True, random_state = 3407, use_rslora = True, loftq_config = None, use_dora = False)
+config = model.peft_config["default"]
+assert config.use_rslora is True
+assert sum(p.numel() for n, p in model.named_parameters() if p.requires_grad and "lora_" in n) > 0
+
+
+def checkpointing_on(m):
+ return any(getattr(x, "gradient_checkpointing", False) is True or getattr(x, "activation_checkpointing_strategy", None) is not None for x in m.modules())
+
+
+assert checkpointing_on(model), "get_peft_model(use_gradient_checkpointing = True) did not enable it"
+FastLanguageModel.for_training(model, use_gradient_checkpointing = False)
+assert model.training and not checkpointing_on(model)
+FastLanguageModel.for_training(model, use_gradient_checkpointing = True)
+assert checkpointing_on(model)
+FastLanguageModel.for_inference(model)
+assert not model.training and not checkpointing_on(model)
+
+messages = [{"role": "user", "content": "What is 2 + 2?"}]
+first = FastDiffusionModel.generate_text(model, processor, messages = messages, max_new_tokens = 16, seed = 0)
+second = FastDiffusionModel.generate_text(model, processor, messages = messages, max_new_tokens = 16, seed = 0)
+assert isinstance(first, str) and first == second, (first, second)
+plain = FastDiffusionModel.generate_text(model, processor, prompt = "Hello", max_new_tokens = 8, seed = 0)
+assert isinstance(plain, str)
+
+calls = {"n": 0}
+
+
+def stop():
+ calls["n"] += 1
+ return calls["n"] > 1
+
+
+try:
+ FastDiffusionModel.generate_text(model, processor, messages = messages, max_new_tokens = 64, stop_callback = stop, seed = 0)
+ raise AssertionError("stop_callback did not cancel")
+except DiffusionGenerationCancelled:
+ pass
+assert calls["n"] == 2, calls
+assert not model.training
+print("SAMPLER_OK", repr(first[:40]))
+"""
+
+_FAMILIES = [
+ "LLaDA-8B-Instruct",
+ "LLaDA-MoE-7B-A1B-Instruct",
+ "Dream-v0-Instruct-7B",
+ "Nemotron-Labs-Diffusion-3B",
+ "LLaDA2-mini",
+ "SDAR-1.7B-Chat",
+ "trl-internal-testing/tiny-DiffusionGemmaForBlockDiffusion",
+]
+
+
[email protected]("name", _FAMILIES)
+def test_tiny_family_routes_and_generates(name):
+ if "/" in name:
+ path = name
+ elif _TINY and os.path.isdir(os.path.join(_TINY, name)):
+ path = os.path.join(_TINY, name)
+ else:
+ pytest.skip("set UNSLOTH_DIFFUSION_TINY_DIR to tiny remote-code copies")
+ # One process per family: LLaDA's remote config registers "llada" globally, shadowing LLaDA-MoE's.
+ from real_accelerator import has_real_accelerator
+
+ # The parent's CUDA spoof does not reach the child; the sampler only needs inference on CPU.
+ env = os.environ if has_real_accelerator() else {**os.environ, "UNSLOTH_ALLOW_CPU": "1"}
+ out = subprocess.run(
+ [sys.executable, "-c", _TINY_SCRIPT, path],
+ capture_output = True,
+ text = True,
+ timeout = 900,
+ env = env,
+ )
+ assert out.returncode == 0, out.stdout[-3000:] + out.stderr[-4000:]
+ assert "SAMPLER_OK" in out.stdout
+
+
+def test_diffusion_lora_init_follows_random_state():
+ import torch.nn as nn
+
+ from unsloth.models.diffusion import FastDiffusionModel
+
+ class Toy(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(model_type = "toy", to_dict = lambda: {})
+ self.q_proj = nn.Linear(8, 8)
+
+ def forward(self, x):
+ return self.q_proj(x)
+
+ def lora_a(seed):
+ torch.manual_seed(seed + 1) # perturb the global RNG: only random_state may decide the init
+ model = FastDiffusionModel.get_peft_model(
+ Toy(),
+ r = 4,
+ target_modules = ["q_proj"],
+ use_gradient_checkpointing = False,
+ random_state = 3407,
+ )
+ return next(p for n, p in model.named_parameters() if "lora_A" in n).detach().clone()
+
+ assert torch.equal(lora_a(0), lora_a(1))
+
+
+def test_get_peft_model_keeps_an_existing_adapter():
+ # Continued training from a saved adapter must not re-inject a fresh one over the learned weights.
+ import torch.nn as nn
+
+ from unsloth.models.diffusion import FastDiffusionModel
+
+ class Toy(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(model_type = "toy", to_dict = lambda: {})
+ self.q_proj = nn.Linear(8, 8)
+
+ model = FastDiffusionModel.get_peft_model(
+ Toy(), r = 4, target_modules = ["q_proj"], use_gradient_checkpointing = False
+ )
+ lora_b = next(p for n, p in model.named_parameters() if "lora_B" in n)
+ with torch.no_grad():
+ lora_b.fill_(0.75)
+ again = FastDiffusionModel.get_peft_model(
+ model, r = 4, target_modules = ["q_proj"], use_gradient_checkpointing = False
+ )
+ assert again is model
+ kept = next(p for n, p in again.named_parameters() if "lora_B" in n)
+ assert torch.all(kept == 0.75) and kept.requires_grad
+ layer = next(m for m in model.modules() if hasattr(m, "lora_A"))
+ layer.gradient_checkpointing = True
+ FastDiffusionModel.get_peft_model(model, r = 4, use_gradient_checkpointing = False)
+ assert layer.gradient_checkpointing is False
+ assert model._unsloth_gradient_checkpointing is False
+
+
+def test_cancellation_hooks_every_embedding_and_restores_attention_modes():
+ import torch.nn as nn
+
+ from unsloth.models import diffusion_samplers
+
+ class Attn(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.diffusion_lm = True
+
+ class EncDec(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.encoder_embed = nn.Embedding(4, 2)
+ self.decoder_embed = nn.Embedding(4, 2)
+ self.attn = Attn()
+
+ base = EncDec()
+ calls = []
+ with pytest.raises(diffusion_samplers.DiffusionGenerationCancelled):
+ with (
+ diffusion_samplers._restore_attention_modes(base),
+ diffusion_samplers._cancellation(base, lambda: calls.append(1) or len(calls) > 1),
+ ):
+ base.encoder_embed(torch.tensor([0]))
+ base.attn.diffusion_lm = False # generate's causal prefill
+ base.decoder_embed(torch.tensor([0])) # the decoder's own table also checks
+ assert len(calls) == 2
+ assert base.attn.diffusion_lm is True
+ assert not base.encoder_embed._forward_pre_hooks and not base.decoder_embed._forward_pre_hooks
diff --git a/unsloth/models/diffusion.py b/unsloth/models/diffusion.py
index e7e51ccd817..29e0fefc58d 100644
--- a/unsloth/models/diffusion.py
+++ b/unsloth/models/diffusion.py
@@ -20,7 +20,7 @@ bit-identical to transformers), keeping only the safe conveniences: 4bit/8bit lo
import os
import re
import torch
-from transformers import AutoConfig, AutoProcessor, AutoTokenizer
+from transformers import AutoConfig, AutoProcessor, AutoTokenizer, set_seed
from unsloth_zoo.hf_utils import add_dtype_kwargs
from ._utils import is_bfloat16_supported, maybe_prefetch_hf_snapshot
@@ -40,7 +40,12 @@ from .loader_utils import (
resolve_unsloth_device_map,
)
-__all__ = ["FastDiffusionModel", "DIFFUSION_MODEL_TYPES", "is_diffusion_model_type"]
+__all__ = [
+ "FastDiffusionModel",
+ "DIFFUSION_MODEL_TYPES",
+ "is_diffusion_model",
+ "is_diffusion_model_type",
+]
# transformers model_type strings routed to this slow path.
DIFFUSION_MODEL_TYPES = ("diffusion_gemma", "diffusion_gemma4")
@@ -69,6 +74,41 @@ def is_diffusion_model_type(model_types):
return any(mt in known for mt in model_types)
+def is_diffusion_model(model):
+ """True for a text-diffusion model: loaded through FastDiffusionModel, or with a config a DiffusionProfile
+ matches. PEFT / DDP wrapped or not."""
+ if model is None:
+ return False
+ base = unwrap_diffusion_model(model)
+ if any(
+ getattr(candidate, "_unsloth_slow_diffusion", False)
+ or getattr(candidate, "_unsloth_diffusion_profile", None) is not None
+ for candidate in (model, base)
+ ):
+ return True
+ return resolve_diffusion_profile(getattr(base, "config", None)) is not None
+
+
+def _ensure_pad_token(model, tokenizer):
+ """Collators need a pad token (Nemotron-Labs-Diffusion ships none). Pad with EOS, as the objectives treat
+ padding as EOS-filled; never the [MASK] token. Only fills what is unset, so repeat calls change nothing."""
+ tok = getattr(tokenizer, "tokenizer", tokenizer)
+ if tok is None or getattr(tok, "pad_token", None) is not None:
+ return
+ if getattr(tok, "eos_token", None) is None:
+ return
+ tok.pad_token = tok.eos_token
+ if tok is not tokenizer and getattr(tokenizer, "pad_token", None) is None:
+ try:
+ tokenizer.pad_token = tok.eos_token
+ except AttributeError:
+ pass
+ base = unwrap_diffusion_model(model)
+ for holder in (getattr(base, "config", None), getattr(base, "generation_config", None)):
+ if holder is not None and getattr(holder, "pad_token_id", None) is None:
+ holder.pad_token_id = tok.pad_token_id
+
+
def _refuse_tied_lora_merge(model):
"""DiffusionGemma's encoder and decoder layers share one base tensor but get separate adapters, so a merge
adds both deltas to both. Refuse it instead of writing a silently wrong checkpoint."""
@@ -454,6 +494,7 @@ class FastDiffusionModel:
cache_dir = cache_dir,
)
+ _ensure_pad_token(model, tokenizer)
if profile is not None:
# On the base module: the objectives read these attributes after unwrapping PEFT / DDP.
profile.prepare_model(unwrap_diffusion_model(model), tokenizer)
@@ -476,7 +517,31 @@ class FastDiffusionModel:
if getattr(model, "_unsloth_full_finetuning", False):
print("Unsloth: Full finetuning is enabled, so .get_peft_model has no effect")
return model
- from peft import LoraConfig, get_peft_model as peft_get_peft_model
+ # Refuse what this transformers-only path cannot honour instead of silently training plain LoRA.
+ if kwargs.get("use_dora"):
+ raise ValueError(
+ "Unsloth: DoRA is not supported for text-diffusion models yet. Use use_dora = False."
+ )
+ if kwargs.get("loftq_config"):
+ raise ValueError(
+ "Unsloth: LoftQ is not supported for text-diffusion models yet. Use loftq_config = None."
+ )
+ from peft import LoraConfig, PeftModel, get_peft_model as peft_get_peft_model
+
+ if isinstance(model, PeftModel):
+ # A loaded adapter (continued training, resume) keeps its weights: re-injecting would start over.
+ for name, param in model.named_parameters():
+ if "lora_" in name:
+ param.requires_grad_(True)
+ if use_gradient_checkpointing:
+ model.gradient_checkpointing_enable()
+ if hasattr(model, "enable_input_require_grads"):
+ model.enable_input_require_grads()
+ else:
+ _disable_gradient_checkpointing(model)
+ model._unsloth_slow_diffusion = True
+ model._unsloth_gradient_checkpointing = use_gradient_checkpointing
+ return model
profile = resolve_diffusion_profile(getattr(model, "config", None))
if target_modules is None:
@@ -484,8 +549,25 @@ class FastDiffusionModel:
profile.lora_target_modules if profile is not None else DIFFUSION_LORA_TARGETS
)
- # use_dora, and any other LoraConfig kwarg outside this allowlist, is silently dropped: Unsloth does
- # not reach this path today, so it is untested on diffusion models.
+ # Unsloth-only layer / offload options do not apply to this path: say so rather than drop them silently.
+ ignored = sorted(
+ k
+ for k in (
+ "layers_to_transform",
+ "layers_pattern",
+ "finetune_last_n_layers",
+ "target_parameters",
+ "qat_scheme",
+ "offload_layers",
+ "checkpoint_skip_layers",
+ )
+ if kwargs.get(k) not in (None, 0, False, [], ())
+ )
+ if ignored:
+ logger.warning(
+ f"Unsloth: {', '.join(ignored)} {'is' if len(ignored) == 1 else 'are'} not supported for "
+ "text-diffusion models and will be ignored."
+ )
lora_kwargs = dict(
r = r,
lora_alpha = lora_alpha,
@@ -493,6 +575,7 @@ class FastDiffusionModel:
bias = bias,
target_modules = target_modules,
task_type = task_type, # None: diffusion has no standard CAUSAL_LM head
+ use_rslora = bool(kwargs.get("use_rslora", False)),
**{k: v for k, v in kwargs.items() if k in ("modules_to_save", "init_lora_weights")},
)
# Exclude the vision tower's custom (non-Linear) modules that share suffix names.
@@ -519,6 +602,8 @@ class FastDiffusionModel:
if hasattr(model, "enable_input_require_grads"):
model.enable_input_require_grads()
+ if random_state is not None:
+ set_seed(random_state)
model = peft_get_peft_model(model, lora_config)
model = _refuse_tied_lora_merge(model)
model._unsloth_slow_diffusion = True
@@ -532,9 +617,7 @@ class FastDiffusionModel:
@staticmethod
def for_inference(model):
model.eval()
- for _, m in model.named_modules():
- if hasattr(m, "gradient_checkpointing"):
- m.gradient_checkpointing = False
+ _disable_gradient_checkpointing(model)
return model
@staticmethod
@@ -542,4 +625,48 @@ class FastDiffusionModel:
model.train()
if use_gradient_checkpointing and hasattr(model, "gradient_checkpointing_enable"):
model.gradient_checkpointing_enable()
+ elif not use_gradient_checkpointing:
+ _disable_gradient_checkpointing(model)
+ model._unsloth_gradient_checkpointing = use_gradient_checkpointing
return model
+
+ @staticmethod
+ def generate_text(
+ model,
+ tokenizer,
+ prompt = None,
+ messages = None,
+ max_new_tokens = 256,
+ steps = None,
+ block_length = None,
+ temperature = 0.0,
+ stop_callback = None,
+ **kwargs,
+ ):
+ """One completion as text via the family's diffusion sampler (see diffusion_samplers.generate_text)."""
+ from .diffusion_samplers import generate_text
+ return generate_text(
+ model,
+ tokenizer,
+ prompt = prompt,
+ messages = messages,
+ max_new_tokens = max_new_tokens,
+ steps = steps,
+ block_length = block_length,
+ temperature = temperature,
+ stop_callback = stop_callback,
+ **kwargs,
+ )
+
+
+def _disable_gradient_checkpointing(model):
+ # gradient_checkpointing_disable reaches LLaDA's own set_activation_checkpointing through the profile bridge.
+ disable = getattr(model, "gradient_checkpointing_disable", None)
+ if callable(disable):
+ try:
+ disable()
+ except (AttributeError, ValueError):
+ pass
+ for _, m in model.named_modules():
+ if hasattr(m, "gradient_checkpointing"):
+ m.gradient_checkpointing = False
diff --git a/unsloth/models/diffusion_block.py b/unsloth/models/diffusion_block.py
index 29b5df60360..b7759c0246f 100644
--- a/unsloth/models/diffusion_block.py
+++ b/unsloth/models/diffusion_block.py
@@ -18,6 +18,12 @@ import types
import torch
import torch.nn.functional as F
+from .diffusion_loss import (
+ chunked_cross_entropy,
+ compile_loss,
+ is_distributed_wrapper,
+ is_plain_linear,
+)
from .diffusion_profiles import (
DiffusionProfile,
register_diffusion_profile,
@@ -53,12 +59,7 @@ def block_diffusion_attention_mask(
_unwrap = unwrap_diffusion_model
-def _is_distributed_wrapper(model):
- return type(model).__name__ in (
- "DistributedDataParallel",
- "FullyShardedDataParallel",
- "DeepSpeedEngine",
- )
+_is_distributed_wrapper = is_distributed_wrapper
class BlockDiffusionProfile(DiffusionProfile):
@@ -139,8 +140,16 @@ class BlockDiffusionProfile(DiffusionProfile):
maskable = response_mask(inputs)
if not self.defaults.get("eos_fill", False):
return input_ids, maskable, valid
- if (valid[:, 1:] & ~valid[:, :-1]).any():
- raise ValueError("Unsloth: block-diffusion training expects right padding.")
+ base = _unwrap(model)
+ if not base.__dict__.get("_unsloth_right_padding_checked", False):
+ # Padding side is fixed per run: check until a padded batch is seen, then stop.
+ left_padded, padded = torch.stack(
+ [(valid[:, 1:] & ~valid[:, :-1]).any(), (~valid).any()]
+ ).tolist()
+ if left_padded:
+ raise ValueError("Unsloth: block-diffusion training expects right padding.")
+ if padded:
+ base.__dict__["_unsloth_right_padding_checked"] = True
# Turns end with a terminator, not EOS, so stopping is learnt only from a supervised EOS tail (as the
# reference's max-length EOS padding); pad to the next block past the longest row.
block = self.block_size(model, args)
@@ -177,17 +186,19 @@ class BlockDiffusionProfile(DiffusionProfile):
p = low + (high - low) * torch.rand(batch, device = clean.device)
masked = (torch.rand(batch, length, device = clean.device) < p[:, None]) & maskable
if self.defaults.get("ensure_masked", False):
+ # Rescue draw taken every step (fixed shape, no host sync); it only changes rows left empty.
empty = maskable.any(dim = 1) & ~masked.any(dim = 1)
- if empty.any():
- scores = torch.rand(batch, length, device = clean.device).masked_fill(~maskable, -1.0)
- pick = F.one_hot(scores.argmax(dim = 1), length).bool()
- masked = masked | (pick & empty[:, None])
+ scores = torch.rand(batch, length, device = clean.device).masked_fill(~maskable, -1.0)
+ pick = F.one_hot(scores.argmax(dim = 1), length).bool()
+ masked = masked | (pick & empty[:, None])
return torch.where(masked, mask_token_id, clean), masked, p
def attention_mask(self, length, block_size, valid, dtype, device):
mask = block_diffusion_attention_mask(length, block_size, device)[None, None]
batch = valid.shape[0]
- if self.defaults.get("key_padding", False) and not bool(valid.all()):
+ if self.defaults.get("key_padding", False):
+ # Unconditional: with no padding `keys` is all True and the diagonal is already allowed, so a
+ # padding-free batch gets the same mask without a host sync.
keys = torch.cat([valid, valid], dim = 1)[:, None, None, :]
# Padding is its own segment in the reference; a padding query keeps its diagonal so no row is empty.
mask = (mask & keys) | torch.eye(2 * length, dtype = torch.bool, device = device)[
@@ -224,29 +235,49 @@ class BlockDiffusionProfile(DiffusionProfile):
)
return out[0][:, :length], head
- def loss_from_noise(self, model, clean, noisy, masked, maskable, p, valid, args):
+ def loss_from_noise(
+ self,
+ model,
+ clean,
+ noisy,
+ masked,
+ maskable,
+ p,
+ valid,
+ args,
+ with_metrics = False,
+ ):
block_size = self.block_size(model, args)
- hidden, head = self.noisy_hidden_and_head(model, noisy, clean, valid, block_size)
- if head is None:
- logits = hidden[masked].float()
- else:
- # Project only the masked positions: the vocab-sized logits are the memory peak otherwise.
- logits = head(hidden[masked]).float()
- nll = F.cross_entropy(logits, clean[masked], reduction = "none")
weighting = self.option(args, "diffusion_time_weighting", "none")
if weighting not in ("inverse_t", "none"):
raise ValueError(
f"Unsloth: {self.name} does not support diffusion_time_weighting={weighting!r}."
)
+ hidden, head = self.noisy_hidden_and_head(model, noisy, clean, valid, block_size)
+ if head is not None and not is_plain_linear(head):
+ # A quantized / LoRA / offloaded / hooked head only runs through its own forward.
+ hidden, head = head(hidden), None
+ targets = torch.where(masked, clean, -100)
+ weights = masked.float()
if weighting == "inverse_t":
- nll = nll / p[:, None].expand_as(masked)[masked].float()
+ # Divide only where masked: an unmasked row with p = 0 (diffusion_eps = 0) would give 0 / 0.
+ weights = weights / torch.where(masked, p[:, None].float(), 1.0)
+ # Unmasked rows are ignored targets, so the shape is fixed whatever was masked.
+ weighted, ce_sum, count = chunked_cross_entropy(
+ hidden,
+ targets,
+ weights,
+ weight = None if head is None else head.weight,
+ bias = None if head is None else head.bias,
+ compiled = compile_loss(args, clean.device),
+ )
if self.defaults.get("normalize", "masked") == "supervised":
denominator = maskable.sum()
else:
- denominator = masked.sum()
- loss = nll.sum() / denominator.clamp_min(1)
- if not masked.any():
- loss = loss + hidden.sum() * 0.0
+ denominator = count
+ loss = weighted / denominator.clamp_min(1)
+ if with_metrics:
+ return loss, ce_sum, count
return loss
def compute_loss(
@@ -259,10 +290,13 @@ class BlockDiffusionProfile(DiffusionProfile):
clean, maskable, valid = self.build_batch(model, inputs, args)
mask_token_id = self.mask_token_id(model, args)
noisy, masked, p = self.sample_noise(clean, maskable, mask_token_id, args)
- loss = self.loss_from_noise(model, clean, noisy, masked, maskable, p, valid, args)
+ loss, ce_sum, count = self.loss_from_noise(
+ model, clean, noisy, masked, maskable, p, valid, args, with_metrics = True
+ )
metrics = {
- "mask_ratio": (masked.sum() / maskable.sum().clamp_min(1)).item(),
- "masked_tokens": masked.sum().item(),
+ "mask_ratio": (count / maskable.sum().clamp_min(1)).detach(),
+ "masked_tokens": count.detach(),
+ "masked_ce": (ce_sum / count.clamp_min(1)).detach(),
}
return loss, None, metrics
@@ -347,9 +381,9 @@ def _module(name, **attributes):
@contextlib.contextmanager
def _sdar_remote_code_imports():
- """SDAR's modeling file imports flash-attn's RMSNorm and 4.x `LossKwargs` at module level. Provide
- torch stand-ins only while it imports when flash-attn is absent; the module keeps its references."""
- import importlib.util
+ """SDAR's modeling file imports flash-attn's RMSNorm, 4.x `LossKwargs` and `SlidingWindowCache` at
+ module level. Provide stand-ins only while it imports; the module keeps its references."""
+ import transformers.cache_utils as cache_utils
import transformers.utils as transformers_utils
if not hasattr(transformers_utils, "LossKwargs"):
@@ -358,6 +392,22 @@ def _sdar_remote_code_imports():
num_items_in_batch: Optional[torch.Tensor]
transformers_utils.LossKwargs = LossKwargs
+ # Missing from some transformers 5.x releases (5.5 has none); SDAR only isinstance-checks it, and nothing builds one.
+ added_sliding_cache = not hasattr(cache_utils, "SlidingWindowCache")
+ if added_sliding_cache:
+ cache_utils.SlidingWindowCache = type("SlidingWindowCache", (cache_utils.Cache,), {})
+ try:
+ with _sdar_flash_attn_stubs():
+ yield
+ finally:
+ if added_sliding_cache:
+ del cache_utils.SlidingWindowCache
+
+
+def _sdar_flash_attn_stubs():
+ import importlib.util
+
try:
has_flash = importlib.util.find_spec("flash_attn") is not None
except (ImportError, ValueError):
diff --git a/unsloth/models/diffusion_gemma_objective.py b/unsloth/models/diffusion_gemma_objective.py
index bff1c8fb1a5..3eee02e1700 100644
--- a/unsloth/models/diffusion_gemma_objective.py
+++ b/unsloth/models/diffusion_gemma_objective.py
@@ -9,6 +9,7 @@ import contextlib
import torch
import torch.nn.functional as F
+from .diffusion_loss import chunked_cross_entropy, compile_loss
from .diffusion_profiles import DiffusionProfile, register_diffusion_profile, unwrap_diffusion_model
__all__ = ["DiffusionGemmaProfile"]
@@ -157,32 +158,43 @@ class DiffusionGemmaProfile(DiffusionProfile):
diffusion_logits = diffusion_logits.float().scatter_add(
2, canvas_ids.unsqueeze(-1), bump[:, :, None].expand(-1, block_size, 1)
)
- if (diffusion_target != -100).any():
- diffusion_loss = F.cross_entropy(
- diffusion_logits.flatten(0, 1), diffusion_target.flatten(), ignore_index = -100
- )
- else:
- diffusion_loss = outputs.logits.sum() * 0.0
+ # Eager under "auto": compiled chunks were 5-10% slower per step on DiffusionGemma-26B-A4B (B200) at equal memory.
+ compiled = compile_loss(args, device, auto = False)
+ # Sum / count instead of a mean: an all-ignored batch gives 0 (with grad), not NaN, and needs no branch.
+ diffusion_sum, _, diffusion_count = chunked_cross_entropy(
+ diffusion_logits, diffusion_target, compiled = compiled
+ )
+ diffusion_loss = diffusion_sum / diffusion_count.clamp_min(1)
+ # Encoder AR co-loss straight from the hidden states: the [B, L, V] encoder logits never exist.
lm_head = base.get_output_embeddings()
hidden_states = outputs.encoder_last_hidden_state.to(lm_head.weight.dtype)
- with _gather_lm_head(lm_head.weight, lm_head.bias):
- encoder_logits = hidden_states @ lm_head.weight.t()
- if lm_head.bias is not None:
- encoder_logits = encoder_logits + lm_head.bias
- encoder_logits = encoder_logits.float()
- encoder_logits = torch.tanh(encoder_logits / softcap) * softcap
ar_mask = attention_mask[:, :-1].bool() & attention_mask[:, 1:].bool()
ar_targets = torch.where(ar_mask, input_ids[:, 1:], -100)
- ar_loss = F.cross_entropy(
- encoder_logits[:, :-1].flatten(0, 1), ar_targets.flatten(), ignore_index = -100
- )
+ if hasattr(lm_head.weight, "ds_id"):
+ # ZeRO-3 re-partitions the head when the gather exits, so the projection must happen inside it
+ # (autograd then keeps the gathered tensor); the fused path would re-read it in backward.
+ with _gather_lm_head(lm_head.weight, lm_head.bias):
+ encoder_logits = F.linear(hidden_states[:, :-1], lm_head.weight, lm_head.bias)
+ ar_sum, _, ar_count = chunked_cross_entropy(
+ encoder_logits, ar_targets, softcap = softcap, compiled = compiled
+ )
+ else:
+ ar_sum, _, ar_count = chunked_cross_entropy(
+ hidden_states[:, :-1],
+ ar_targets,
+ weight = lm_head.weight,
+ bias = lm_head.bias,
+ softcap = softcap,
+ compiled = compiled,
+ )
+ ar_loss = ar_sum / ar_count.clamp_min(1)
loss = diffusion_loss + ar_weight * ar_loss
metrics = {
- "diffusion_loss": diffusion_loss.detach().item(),
- "encoder_ar_loss": ar_loss.detach().item(),
- "mean_t": t.mean().item(),
+ "diffusion_loss": diffusion_loss.detach(),
+ "encoder_ar_loss": ar_loss.detach(),
+ "mean_t": t.mean().detach(),
}
return loss, outputs, metrics
diff --git a/unsloth/models/diffusion_loss.py b/unsloth/models/diffusion_loss.py
new file mode 100644
index 00000000000..9431e0ce021
--- /dev/null
+++ b/unsloth/models/diffusion_loss.py
@@ -0,0 +1,369 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Weighted cross entropy for the diffusion objectives without [N, V] fp32 logits.
+
+Rows are processed in fixed-size chunks (the last one padded with ignored rows), so the compiled chunk kernels
+see one shape per vocabulary / chunk size and never recompile as the masked fraction or sequence length moves.
+Backward recomputes each chunk's logits instead of keeping them.
+"""
+
+import os
+
+import torch
+import torch.nn.functional as F
+
+__all__ = [
+ "chunked_cross_entropy",
+ "diffusion_compile_mode",
+ "compile_loss",
+ "compile_backbone_layers",
+ "fusable_head",
+ "head_returns_hidden",
+ "is_distributed_wrapper",
+ "is_plain_linear",
+]
+
+_CHUNK_BYTES = int(os.environ.get("UNSLOTH_DIFFUSION_CE_CHUNK_BYTES", 256 * 1024 * 1024))
+_MIN_ROWS = 16
+
+
+def diffusion_compile_mode(args):
+ """None (eager), "auto" (compiled CE chunks on CUDA), "loss" (compiled CE chunks anywhere) or "full"
+ (compiled CE chunks + per-layer backbone compile)."""
+ if os.environ.get("UNSLOTH_COMPILE_DISABLE", "0").lower() in ("1", "true", "yes"):
+ return None
+ value = getattr(args, "diffusion_compile", "auto") if args is not None else "auto"
+ if isinstance(value, str):
+ value = value.strip().lower()
+ if value in (False, None, "false", "0", "no", "off", "none", "eager"):
+ return None
+ if value in (True, "true", "1", "yes", "on", "full"):
+ return "full"
+ if value in ("auto", "loss"):
+ return value
+ raise ValueError(
+ f"Unsloth: diffusion_compile must be 'auto', 'loss', 'full', True or False, got {value!r}."
+ )
+
+
+def compile_loss(
+ args,
+ device,
+ auto = True,
+):
+ """`auto` = False keeps "auto" eager for objectives where compiled chunks measured no faster."""
+ mode = diffusion_compile_mode(args)
+ return mode in ("loss", "full") or (
+ auto and mode == "auto" and torch.device(device).type == "cuda"
+ )
+
+
+def _rows_per_chunk(n, vocab):
+ rows = max(_MIN_ROWS, _CHUNK_BYTES // (4 * max(vocab, 1)))
+ rows = 1 << (rows.bit_length() - 1)
+ # Small inputs use the next power of two, so short batches neither pad to a huge chunk nor recompile per length.
+ return min(rows, max(_MIN_ROWS, 1 << max(n - 1, 0).bit_length()))
+
+
+def _logits(h, weight, bias, softcap):
+ z = h if weight is None else F.linear(h, weight, bias)
+ z = z.float()
+ if softcap is not None:
+ z = torch.tanh(z / softcap) * softcap
+ return z
+
+
+def _chunk_forward(h, weight, bias, targets, token_weights, softcap):
+ z = _logits(h, weight, bias, softcap)
+ lse = torch.logsumexp(z, dim = -1)
+ valid = targets != -100
+ picked = z.gather(1, targets.clamp_min(0)[:, None]).squeeze(1)
+ ce = torch.where(valid, lse - picked, 0.0)
+ return (ce * token_weights).sum(), ce.sum(), lse
+
+
+def _chunk_backward(h, weight, bias, targets, token_weights, lse, grad, softcap):
+ z = h if weight is None else F.linear(h, weight, bias)
+ z = z.float()
+ if softcap is not None:
+ tanh = torch.tanh(z / softcap)
+ z = tanh * softcap
+ valid = targets != -100
+ scale = torch.where(valid, token_weights * grad, 0.0)
+ dz = torch.exp(z - lse[:, None])
+ dz = dz.scatter_add(1, targets.clamp_min(0)[:, None], -valid.to(dz.dtype)[:, None])
+ dz = dz * scale[:, None]
+ if softcap is not None:
+ dz = dz * (1 - tanh * tanh)
+ # Select, not multiply: an ignored row can overflow (fp16 heads), and 0 * inf is NaN.
+ dz = torch.where(valid[:, None], dz, 0.0)
+ return dz.to(h.dtype)
+
+
+_COMPILED = {}
+
+
+def _kernels(compiled):
+ if not compiled:
+ return _chunk_forward, _chunk_backward
+ if not _COMPILED:
+ _COMPILED["forward"] = torch.compile(_chunk_forward, fullgraph = True, dynamic = False)
+ _COMPILED["backward"] = torch.compile(_chunk_backward, fullgraph = True, dynamic = False)
+ return _COMPILED["forward"], _COMPILED["backward"]
+
+
+def _chunk(tensor, start, rows, fill):
+ piece = tensor[start : start + rows]
+ short = rows - piece.shape[0]
+ if short:
+ pad = [0, 0] * (piece.dim() - 1) + [0, short]
+ piece = F.pad(piece, pad, value = fill)
+ return piece
+
+
+class _ChunkedCrossEntropy(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, hidden, weight, bias, targets, token_weights, softcap, compiled):
+ device_type = hidden.device.type
+ ctx.autocast = (
+ device_type,
+ torch.is_autocast_enabled(device_type),
+ torch.get_autocast_dtype(device_type),
+ )
+ # Casts made here carry no autograd history, so they must not land in the enclosing autocast cache,
+ # where a later projection of the same weight would reuse them and lose its weight gradient.
+ with torch.autocast(
+ device_type = device_type,
+ dtype = ctx.autocast[2],
+ enabled = ctx.autocast[1],
+ cache_enabled = False,
+ ):
+ return _ChunkedCrossEntropy._forward(
+ ctx, hidden, weight, bias, targets, token_weights, softcap, compiled
+ )
+
+ @staticmethod
+ def _forward(ctx, hidden, weight, bias, targets, token_weights, softcap, compiled):
+ forward_kernel, _ = _kernels(compiled)
+ n = hidden.shape[0]
+ vocab = hidden.shape[1] if weight is None else weight.shape[0]
+ rows = _rows_per_chunk(n, vocab)
+ token_weights = token_weights.float()
+ weighted = hidden.new_zeros((), dtype = torch.float32)
+ total = hidden.new_zeros((), dtype = torch.float32)
+ lses = []
+ for start in range(0, n, rows):
+ w_sum, ce_sum, lse = forward_kernel(
+ _chunk(hidden, start, rows, 0),
+ weight,
+ bias,
+ _chunk(targets, start, rows, -100),
+ _chunk(token_weights, start, rows, 0),
+ softcap,
+ )
+ weighted = weighted + w_sum
+ total = total + ce_sum
+ lses.append(lse)
+ lse = torch.cat(lses) if lses else hidden.new_zeros((0,), dtype = torch.float32)
+ ctx.save_for_backward(hidden, weight, bias, targets, token_weights, lse)
+ ctx.rows, ctx.softcap, ctx.compiled = rows, softcap, compiled
+ ctx.mark_non_differentiable(total)
+ return weighted, total
+
+ @staticmethod
+ def backward(ctx, grad_weighted, grad_total):
+ # Recompute the logits in the forward's precision, or softmax and logsumexp disagree.
+ device_type, enabled, dtype = ctx.autocast
+ with torch.autocast(
+ device_type = device_type, dtype = dtype, enabled = enabled, cache_enabled = False
+ ):
+ return _ChunkedCrossEntropy._backward(ctx, grad_weighted)
+
+ @staticmethod
+ def _backward(ctx, grad_weighted):
+ hidden, weight, bias, targets, token_weights, lse = ctx.saved_tensors
+ _, backward_kernel = _kernels(ctx.compiled)
+ rows, n = ctx.rows, hidden.shape[0]
+ need_h, need_w, need_b = ctx.needs_input_grad[:3]
+ grad_hidden = torch.empty_like(hidden) if need_h else None
+ grad_w = torch.zeros_like(weight, dtype = torch.float32) if need_w else None
+ grad_b = (
+ torch.zeros_like(bias, dtype = torch.float32) if need_b and bias is not None else None
+ )
+ grad = grad_weighted.float()
+ for start in range(0, n, rows):
+ h = _chunk(hidden, start, rows, 0)
+ dz = backward_kernel(
+ h,
+ weight,
+ bias,
+ _chunk(targets, start, rows, -100),
+ _chunk(token_weights, start, rows, 0),
+ lse[start : start + rows], # one padded entry per row, so every slice is full
+ grad,
+ ctx.softcap,
+ )
+ take = min(rows, n - start)
+ if weight is None:
+ if need_h:
+ grad_hidden[start : start + take] = dz[:take]
+ continue
+ if need_h:
+ grad_hidden[start : start + take] = (dz @ weight)[:take]
+ if need_w:
+ grad_w += (dz.t() @ h).float()
+ if grad_b is not None:
+ grad_b += dz.sum(0).float()
+ if grad_w is not None:
+ grad_w = grad_w.to(weight.dtype)
+ if grad_b is not None:
+ grad_b = grad_b.to(bias.dtype)
+ return grad_hidden, grad_w, grad_b, None, None, None, None
+
+
+def chunked_cross_entropy(
+ hidden,
+ targets,
+ token_weights = None,
+ weight = None,
+ bias = None,
+ softcap = None,
+ compiled = False,
+):
+ """sum(w * CE) and the unweighted CE sum over positions with target != -100.
+
+ ``hidden`` is [..., H] projected by ``weight`` [V, H] (+ ``bias``), or already logits [..., V] when
+ ``weight`` is None. Logits are computed in the input dtype and upcast like ``head(h).float()``. The second
+ output carries no gradient (metrics). Returns (weighted_sum, ce_sum, count) as 0-dim tensors.
+ """
+ width = hidden.shape[-1]
+ hidden = hidden.reshape(-1, width)
+ targets = targets.reshape(-1)
+ if token_weights is None:
+ token_weights = (targets != -100).float()
+ token_weights = token_weights.reshape(-1)
+ if weight is not None and hidden.dtype != weight.dtype:
+ hidden = hidden.to(weight.dtype)
+ weighted, total = _ChunkedCrossEntropy.apply(
+ hidden,
+ weight,
+ bias,
+ targets,
+ token_weights,
+ None if softcap is None else float(softcap),
+ bool(compiled),
+ )
+ return weighted, total, (targets != -100).sum()
+
+
+def is_distributed_wrapper(model):
+ return type(model).__name__ in (
+ "DistributedDataParallel",
+ "FullyShardedDataParallel",
+ "DeepSpeedEngine",
+ )
+
+
+def is_plain_linear(module):
+ """An nn.Linear whose output is exactly F.linear(x, weight, bias): no subclass (LoRA / quantized), no
+ accelerate offload hook, no instance-level forward and no forward hooks."""
+ return (
+ type(module) is torch.nn.Linear
+ and not hasattr(module, "_hf_hook")
+ and "forward" not in vars(module)
+ and not module._forward_hooks
+ and not module._forward_pre_hooks
+ )
+
+
+def fusable_head(model, base):
+ """The plain nn.Linear output head whose input can be taken instead of logits, or None (dense logits).
+
+ Decided once per head module (a head wrapped later, e.g. by PEFT, is re-checked): a LoRA / quantized /
+ offloaded / tied or post-scaled head, or a distributed wrapper (FSDP shards the weight outside its
+ forward), keeps the dense path."""
+ if is_distributed_wrapper(model):
+ return None
+ head = base.get_output_embeddings() if hasattr(base, "get_output_embeddings") else None
+ cached = base.__dict__.get("_unsloth_diffusion_fused_head")
+ if cached is not None and cached[0] is head:
+ return cached[1]
+ config = getattr(base, "config", None)
+ ok = (
+ is_plain_linear(head)
+ and not getattr(
+ config, "weight_tying", False
+ ) # LLaDA: tied head is F.linear on wte, not this module
+ and not getattr(config, "scale_logits", False)
+ and head.weight.device.type != "meta"
+ )
+ base.__dict__["_unsloth_diffusion_fused_head"] = (head, head if ok else None)
+ return head if ok else None
+
+
+class head_returns_hidden:
+ """While active the output head returns its input, so ``model(...).logits`` are the final hidden states."""
+
+ def __init__(self, head):
+ self.head = head
+
+ # Eager even under an outer torch.compile: the swap is plain module plumbing, not math.
+ @torch.compiler.disable
+ def __enter__(self):
+ if self.head is not None:
+ self.head.forward = _identity
+ return self.head
+
+ @torch.compiler.disable
+ def __exit__(self, *exc):
+ if self.head is not None:
+ self.head.__dict__.pop("forward", None)
+ return False
+
+
+def _identity(x):
+ return x
+
+
+def _decoder_layer_lists(model):
+ seen = set()
+ for name, module in model.named_modules():
+ if not isinstance(module, torch.nn.ModuleList) or len(module) == 0:
+ continue
+ if name.rsplit(".", 1)[-1] not in ("layers", "blocks", "h"):
+ continue
+ if any(name.startswith(prefix + ".") for prefix in seen):
+ continue
+ seen.add(name)
+ yield name, module
+
+
+def _warm_rotary_caches(model):
+ """LLaDA fills its RoPE cache lazily and moves it to the parameters' device on first use. Inside a compiled,
+ checkpointed layer that move happens in the forward but not in the recompute, so the two save different
+ tensors (CheckpointError). Fill it on the right device before compiling."""
+ try:
+ device = next(model.parameters()).device
+ except StopIteration:
+ return
+ for module in model.modules():
+ rotary = getattr(module, "get_rotary_embedding", None)
+ config = getattr(module, "config", None)
+ length = getattr(config, "max_sequence_length", None)
+ if callable(rotary) and isinstance(length, int) and length > 0:
+ rotary(length, device)
+
+
+def compile_backbone_layers(model, dynamic = None):
+ """Compile every decoder layer in place (regional compile); idempotent. Returns the layer count."""
+ _warm_rotary_caches(model)
+ compiled = 0
+ for _, layers in _decoder_layer_lists(model):
+ for layer in layers:
+ if getattr(layer, "_unsloth_diffusion_compiled", False):
+ continue
+ layer.compile(dynamic = dynamic)
+ layer._unsloth_diffusion_compiled = True
+ compiled += 1
+ return compiled
diff --git a/unsloth/models/diffusion_masked.py b/unsloth/models/diffusion_masked.py
index f1928b6a2f7..947dabc99f0 100644
--- a/unsloth/models/diffusion_masked.py
+++ b/unsloth/models/diffusion_masked.py
@@ -12,8 +12,13 @@ import functools
from dataclasses import dataclass
import torch
-import torch.nn.functional as F
+from .diffusion_loss import (
+ chunked_cross_entropy,
+ compile_loss,
+ fusable_head,
+ head_returns_hidden,
+)
from .diffusion_profiles import (
DiffusionProfile,
register_diffusion_profile,
@@ -170,19 +175,25 @@ class MaskedDiffusionProfile(DiffusionProfile):
from transformers.dynamic_module_utils import get_class_from_dynamic_module
auto_map = getattr(config, "auto_map", None) or {}
- class_ref = (
- auto_map.get(self.auto_class)
- or auto_map.get("AutoModelForCausalLM")
- or auto_map.get("AutoModel")
+ auto_class = next(
+ (
+ name
+ for name in (self.auto_class, "AutoModelForCausalLM", "AutoModel")
+ if auto_map.get(name)
+ ),
+ None,
)
- if class_ref is None or not trust_remote_code:
+ if auto_class is None or not trust_remote_code:
return super().model_class(config, trust_remote_code, **hub_kwargs)
cls = get_class_from_dynamic_module(
- class_ref,
+ auto_map[auto_class],
config._name_or_path,
revision = getattr(config, "_commit_hash", None),
**hub_kwargs,
)
+ # transformers < 5.17 reruns a non-Auto hub class's _init_weights over the loaded checkpoint (LLaDA-MoE, Dream).
+ if getattr(cls, "_auto_class", None) is None:
+ cls.register_for_auto_class(auto_class)
_accept_new_validate_kwargs(cls)
return _ensure_post_init(cls)
@@ -240,6 +251,10 @@ class MaskedDiffusionProfile(DiffusionProfile):
mask_id, eos_id = self._token_ids(model, args)
eps = float(self.option(args, "diffusion_eps"))
weighting = self.option(args, "diffusion_time_weighting")
+ if weighting not in ("inverse_t", "none", None):
+ raise ValueError(
+ f"Unsloth: {self.name} does not support diffusion_time_weighting={weighting!r}."
+ )
maskable = response_mask(inputs)
forward_mask = attention_mask
@@ -258,33 +273,36 @@ class MaskedDiffusionProfile(DiffusionProfile):
noisy = torch.where(masked, mask_id, input_ids)
model_inputs = {"input_ids": noisy}
- if forward_mask is not None and not bool(forward_mask.all()):
+ if forward_mask is not None:
model_inputs["attention_mask"] = forward_mask
- logits = model(**model_inputs).logits
+ head = fusable_head(model, unwrap_diffusion_model(model))
+ with head_returns_hidden(head):
+ hidden = model(**model_inputs).logits
if self.shift_logits:
- logits = torch.cat([logits[:, :1], logits[:, :-1]], dim = 1)
-
- if not bool(masked.any()):
- zero = logits.sum() * 0.0
- return zero, None, {"masked_fraction": 0.0}
+ hidden = torch.cat([hidden[:, :1], hidden[:, :-1]], dim = 1)
- token_loss = F.cross_entropy(logits[masked].float(), input_ids[masked], reduction = "none")
+ targets = torch.where(masked, input_ids, -100)
+ weights = masked.float()
if weighting == "inverse_t":
- token_loss = token_loss / p_mask[masked]
- elif weighting not in ("none", None):
- raise ValueError(
- f"Unsloth: {self.name} does not support diffusion_time_weighting={weighting!r}."
- )
-
+ weights = weights / torch.where(masked, p_mask, 1.0)
+ if self.loss_normalization == "answer_length":
+ weights = weights / (maskable.sum(dim = 1, keepdim = True).clamp_min(1) * batch)
+ weighted, ce_sum, count = chunked_cross_entropy(
+ hidden,
+ targets,
+ weights,
+ weight = None if head is None else head.weight,
+ bias = None if head is None else head.bias,
+ compiled = compile_loss(args, device),
+ )
if self.loss_normalization == "answer_length":
- answer_length = maskable.sum(dim = 1, keepdim = True).clamp_min(1).expand(batch, length)
- loss = (token_loss / answer_length[masked]).sum() / batch
+ loss = weighted
else:
- loss = token_loss.sum() / masked.sum()
+ loss = weighted / count.clamp_min(1)
metrics = {
- "masked_fraction": (masked.sum() / maskable.sum().clamp_min(1)).item(),
- "masked_ce": F.cross_entropy(logits[masked].float(), input_ids[masked]).item(),
+ "masked_fraction": (count / maskable.sum().clamp_min(1)).detach(),
+ "masked_ce": (ce_sum / count.clamp_min(1)).detach(),
}
return loss, None, metrics
@@ -312,6 +330,7 @@ register_diffusion_profile(
lora_target_modules = r".*\.blocks\.\d+\.(q_proj|k_proj|v_proj|attn_out|ff_proj|up_proj|ff_out)",
quant_skip_modules = ["transformer.ff_out", "transformer.wte", "lm_head"],
default_mask_token_id = 126336,
+ compile_backbone = True,
)
)
register_diffusion_profile(
@@ -324,6 +343,7 @@ register_diffusion_profile(
shift_logits = True,
loss_normalization = "masked_tokens",
auto_class = "AutoModel",
+ compile_backbone = True,
# Dream q_sample draws t ~ U(0, 1) with no floor and weights by 1 / t (time_reweighting "original").
defaults = {"diffusion_eps": 0.0},
)
@@ -335,7 +355,10 @@ register_diffusion_profile(
architectures = ("NemotronLabsDiffusionModel",),
requires_remote_code = True,
lora_target_modules = list(_ATTENTION_AND_MLP),
+ # Keep `diffusion_head` 16-bit like every other head, so the loss can take hidden states.
+ quant_skip_modules = ["lm_head", "embed_tokens", "diffusion_head"],
loss_normalization = "masked_tokens",
auto_class = "AutoModel",
+ compile_backbone = True,
)
)
diff --git a/unsloth/models/diffusion_profiles.py b/unsloth/models/diffusion_profiles.py
index 0667da7b513..43c9bbeb448 100644
--- a/unsloth/models/diffusion_profiles.py
+++ b/unsloth/models/diffusion_profiles.py
@@ -50,6 +50,8 @@ class DiffusionProfile:
quant_skip_modules: Optional[list] = None
# Profile-specific defaults a DiffusionConfig field left at None falls back to.
defaults: dict = field(default_factory = dict)
+ # diffusion_compile = "full" allowed; off for data-dependent MoE dispatch (per-expert token gathers).
+ compile_backbone: bool = False
def matches(self, config) -> bool:
model_type = getattr(config, "model_type", None)
@@ -89,7 +91,7 @@ class DiffusionProfile:
args,
num_items_in_batch = None,
):
- """Returns (loss, outputs_or_None, metrics: dict[str, float])."""
+ """Returns (loss, outputs_or_None, metrics: dict[str, float | 0-dim tensor])."""
raise NotImplementedError(
f"Unsloth: diffusion profile {self.name} has no training objective."
)
diff --git a/unsloth/models/diffusion_samplers.py b/unsloth/models/diffusion_samplers.py
new file mode 100644
index 00000000000..72e3f5bf86c
--- /dev/null
+++ b/unsloth/models/diffusion_samplers.py
@@ -0,0 +1,561 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Text generation for text-diffusion LMs: one sampler per DiffusionProfile name, each the family's published
+decoding recipe. Families shipping a sampler in their modeling code (Dream, LLaDA2, Nemotron-Labs-Diffusion,
+DiffusionGemma) are called as is; LLaDA / LLaDA-MoE and SDAR samplers live in separate scripts upstream and are
+re-implemented here."""
+
+import contextlib
+import math
+
+import torch
+import torch.nn.functional as F
+
+from .diffusion_profiles import resolve_diffusion_profile, unwrap_diffusion_model
+
+__all__ = ["DiffusionGenerationCancelled", "generate_text", "diffusion_profile_name"]
+
+
+class DiffusionGenerationCancelled(RuntimeError):
+ """Raised out of generate_text when stop_callback() returned True mid-generation."""
+
+
+_SAMPLERS = {}
+
+
+def _register(*names):
+ def wrap(function):
+ for name in names:
+ _SAMPLERS[name] = function
+ return function
+
+ return wrap
+
+
+def diffusion_profile_name(model):
+ """The DiffusionProfile name a loaded (possibly PEFT / DDP wrapped) model was routed with, or None."""
+ base = unwrap_diffusion_model(model)
+ for candidate in (model, base):
+ name = getattr(candidate, "_unsloth_diffusion_profile", None)
+ if name is not None:
+ return getattr(name, "name", name)
+ profile = resolve_diffusion_profile(getattr(base, "config", None))
+ return profile.name if profile is not None else None
+
+
+def _text_tokenizer(tokenizer):
+ return getattr(tokenizer, "tokenizer", tokenizer)
+
+
+_UNIFORM_NOISE = frozenset({"diffusion_gemma"})
+
+
+def _profile_mask_id(base, tokenizer, name):
+ from .diffusion_profiles import diffusion_profiles
+
+ profile = next((p for p in diffusion_profiles() if p.name == name), None)
+ candidates = (
+ getattr(base, "_unsloth_mask_token_id", None),
+ getattr(base, "_unsloth_diffusion_mask_token_id", None),
+ getattr(getattr(base, "config", None), "mask_token_id", None),
+ getattr(profile, "default_mask_token_id", None),
+ (getattr(profile, "defaults", None) or {}).get("mask_token_id"),
+ getattr(_text_tokenizer(tokenizer), "mask_token_id", None),
+ )
+ for value in candidates:
+ if value is not None:
+ return int(value)
+ if name in _UNIFORM_NOISE:
+ return None # uniform-noise families (DiffusionGemma) have no [MASK] token to strip
+ raise ValueError(f"Unsloth: no [MASK] token id for the {name} sampler.")
+
+
+def _eos_ids(base, tokenizer):
+ ids = []
+ for source in (
+ getattr(base, "generation_config", None),
+ getattr(base, "config", None),
+ _text_tokenizer(tokenizer),
+ ):
+ value = getattr(source, "eos_token_id", None)
+ for item in value if isinstance(value, (list, tuple)) else (value,):
+ if item is not None and int(item) not in ids:
+ ids.append(int(item))
+ return ids
+
+
+_TURN_PROBE = "Unsloth turn probe"
+
+
+def _turn_end_ids(tokenizer):
+ """The special token closing an assistant turn (Dream: <|im_end|>, while its eos_token_id is <|endoftext|>)."""
+ text_tokenizer = _text_tokenizer(tokenizer)
+ try:
+ rendered = text_tokenizer.apply_chat_template(
+ [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": _TURN_PROBE}],
+ tokenize = False,
+ )
+ ids = text_tokenizer(rendered.split(_TURN_PROBE, 1)[1], add_special_tokens = False)[
+ "input_ids"
+ ]
+ except Exception:
+ return []
+ special = set(getattr(text_tokenizer, "all_special_ids", None) or ())
+ decoder = getattr(text_tokenizer, "added_tokens_decoder", None) or {}
+ special.update(i for i, token in decoder.items() if getattr(token, "special", False))
+ return [int(ids[0])] if ids and ids[0] in special else []
+
+
+def _device(base):
+ try:
+ return next(base.parameters()).device
+ except StopIteration:
+ return torch.device("cpu")
+
+
+def _encode(tokenizer, prompt, messages, device):
+ if messages is not None:
+ text = None
+ for source in (tokenizer, _text_tokenizer(tokenizer)):
+ try:
+ text = source.apply_chat_template(
+ messages, add_generation_prompt = True, tokenize = False
+ )
+ break
+ except Exception:
+ continue
+ if text is None:
+ raise ValueError(
+ "Unsloth: this tokenizer has no usable chat template; pass prompt= instead."
+ )
+ add_special_tokens = False
+ elif prompt is not None:
+ text, add_special_tokens = prompt, True
+ else:
+ raise ValueError("Unsloth: generate_text needs prompt= or messages=.")
+ encoded = _text_tokenizer(tokenizer)(
+ text, return_tensors = "pt", add_special_tokens = add_special_tokens
+ )
+ return encoded["input_ids"].to(device), encoded.get("attention_mask")
+
+
+def _round_up(value, multiple):
+ return int(math.ceil(value / multiple) * multiple)
+
+
+def _contains_any(tokens, ids):
+ return bool(ids) and bool(torch.isin(tokens, torch.tensor(ids, device = tokens.device)).any())
+
+
+@_register("llada", "llada_moe")
+def _llada_sample(
+ base,
+ tokenizer,
+ input_ids,
+ attention_mask,
+ *,
+ max_new_tokens,
+ steps,
+ block_length,
+ temperature,
+ mask_id,
+ eos_ids,
+ remasking = "low_confidence",
+ eos_early_stop = True,
+ **unused,
+):
+ """Semi-autoregressive low-confidence remasking (LLaDA generate.py, also LLaDA-MoE's model card):
+ blocks of `block_length` masks, a linear schedule unmasking the most confident predictions each step."""
+ block_length = int(block_length or 32)
+ gen_length = _round_up(max_new_tokens, block_length)
+ num_blocks = gen_length // block_length
+ steps = _round_up(int(steps or gen_length), num_blocks)
+ steps_per_block = steps // num_blocks
+ batch, prompt_length = input_ids.shape
+ x = torch.full(
+ (batch, prompt_length + gen_length), mask_id, dtype = torch.long, device = input_ids.device
+ )
+ x[:, :prompt_length] = input_ids
+ for block in range(num_blocks):
+ start = prompt_length + block * block_length
+ end = start + block_length
+ remaining = (x[:, start:end] == mask_id).sum(dim = 1, keepdim = True)
+ schedule = remaining // steps_per_block + (
+ torch.arange(steps_per_block, device = x.device)[None, :] < remaining % steps_per_block
+ )
+ for step in range(steps_per_block):
+ mask_index = x == mask_id
+ logits = base(x).logits
+ if temperature > 0:
+ # Gumbel-max in float64: low precision noise hurts MDM generation quality (arXiv:2409.02908).
+ noisy = logits.to(torch.float64)
+ noisy = noisy.exp() / (-torch.log(torch.rand_like(noisy))) ** temperature
+ else:
+ noisy = logits
+ x0 = noisy.argmax(dim = -1)
+ if remasking == "low_confidence":
+ probs = F.softmax(logits, dim = -1)
+ x0_p = probs.gather(-1, x0.unsqueeze(-1)).squeeze(-1)
+ elif remasking == "random":
+ x0_p = torch.rand(x0.shape, device = x0.device)
+ else:
+ raise ValueError(f"Unsloth: unknown LLaDA remasking {remasking!r}.")
+ x0_p[:, end:] = -math.inf
+ x0 = torch.where(mask_index, x0, x)
+ confidence = torch.where(mask_index, x0_p, -math.inf)
+ transfer = torch.zeros_like(x0, dtype = torch.bool)
+ for row in range(batch):
+ _, index = torch.topk(confidence[row], k = int(schedule[row, step]))
+ transfer[row, index] = True
+ x[transfer] = x0[transfer]
+ if eos_early_stop and _contains_any(x[:, prompt_length:end], eos_ids):
+ # Later blocks never change earlier ones, so stopping here only drops text after the EOS.
+ return x[:, prompt_length:end]
+ return x[:, prompt_length:]
+
+
+def _block_sample(
+ base,
+ tokenizer,
+ input_ids,
+ attention_mask,
+ *,
+ max_new_tokens,
+ steps,
+ block_length,
+ temperature,
+ mask_id,
+ eos_ids,
+ mask_format,
+ confidence_threshold,
+ top_k = 0,
+ top_p = 1.0,
+ **unused,
+):
+ """Block diffusion decoding shared by SDAR (JetAstra/SDAR generate.py, MIT, low_confidence_dynamic) and
+ LLaDA2 (its modeling generate, Apache-2.0): blocks aligned to absolute positions, block-causal attention,
+ every token above `confidence_threshold` accepted at once, else the scheduled most confident ones. The
+ visible prefix is recomputed each step (as LLaDA2 does) instead of SDAR's custom KV cache. temperature 0
+ is greedy."""
+ prompt_length = input_ids.shape[1]
+ num_blocks = (prompt_length + max_new_tokens + block_length - 1) // block_length
+ total_length = num_blocks * block_length
+ device = input_ids.device
+ block_causal = torch.tril(torch.ones(num_blocks, num_blocks, device = device, dtype = torch.bool))
+ allowed = block_causal.repeat_interleave(block_length, 0).repeat_interleave(block_length, 1)[
+ None
+ ]
+ if mask_format == "additive":
+ dtype = next(base.parameters()).dtype
+ attention = torch.zeros(allowed.shape, dtype = dtype, device = device)
+ attention = attention.masked_fill(~allowed, torch.finfo(dtype).min)[:, None]
+ else:
+ attention = allowed
+ position_ids = torch.arange(total_length, device = device)[None]
+ x = torch.full((input_ids.shape[0], total_length), mask_id, dtype = torch.long, device = device)
+ x[:, :prompt_length] = input_ids
+ schedule = torch.full((steps,), block_length // steps, dtype = torch.long)
+ schedule[: block_length % steps] += 1
+ for block in range(prompt_length // block_length, num_blocks):
+ start, end = block * block_length, (block + 1) * block_length
+ for step in range(steps + 1):
+ mask_index = x[:, start:end] == mask_id
+ if not bool(mask_index.any()):
+ break
+ logits = (
+ base(
+ x[:, :end],
+ attention_mask = attention[..., :end, :end],
+ position_ids = position_ids[:, :end],
+ )
+ .logits[:, start:end]
+ .float()
+ )
+ if temperature > 0:
+ logits = logits / temperature
+ if top_k and top_k > 0:
+ kth = torch.topk(logits, top_k).values[..., -1, None]
+ logits = logits.masked_fill(logits < kth, -math.inf)
+ if top_p is not None and top_p < 1.0:
+ sorted_logits, order = torch.sort(logits, descending = True)
+ drop = F.softmax(sorted_logits, dim = -1).cumsum(-1) > top_p
+ drop[..., 1:] = drop[..., :-1].clone()
+ drop[..., 0] = False
+ logits = logits.masked_fill(drop.scatter(-1, order, drop), -math.inf)
+ probs = F.softmax(logits, dim = -1)
+ x0 = torch.multinomial(probs.flatten(0, 1), 1).view(probs.shape[:-1])
+ else:
+ probs = F.softmax(logits, dim = -1)
+ x0 = probs.argmax(dim = -1)
+ x0_p = probs.gather(-1, x0.unsqueeze(-1)).squeeze(-1)
+ confidence = torch.where(mask_index, x0_p, -math.inf)
+ count = int(schedule[min(step, steps - 1)])
+ transfer = torch.zeros_like(mask_index)
+ for row in range(x.shape[0]):
+ high = confidence[row] > confidence_threshold
+ if int(high.sum()) >= count:
+ transfer[row] = high
+ else:
+ # Never more than the masks left: the official top-k could overwrite prompt tokens sharing
+ # the first decoded block.
+ k = min(count, int(mask_index[row].sum()))
+ transfer[row, torch.topk(confidence[row], k).indices] = True
+ window = x[:, start:end]
+ window[transfer] = x0[transfer]
+ if _contains_any(x[:, prompt_length:end], eos_ids):
+ return x[:, prompt_length:end]
+ return x[:, prompt_length:]
+
+
+@_register("sdar")
+def _sdar_sample(base, tokenizer, input_ids, attention_mask, *, steps, block_length, **kwargs):
+ """SDAR model card / generate.py defaults: block 4, steps = block length, threshold 0.85."""
+ block_length = int(block_length or getattr(base.config, "block_size", None) or 4)
+ kwargs.setdefault("confidence_threshold", 0.85)
+ return _block_sample(
+ base,
+ tokenizer,
+ input_ids,
+ attention_mask,
+ steps = int(steps or block_length),
+ block_length = block_length,
+ mask_format = "bool",
+ **kwargs,
+ )
+
+
+@_register("llada2")
+def _llada2_sample(
+ base,
+ tokenizer,
+ input_ids,
+ attention_mask,
+ *,
+ steps,
+ block_length,
+ threshold = 0.95,
+ **kwargs,
+):
+ """LLaDA2 model card defaults: block 32, steps 32, threshold 0.95. Its own generate builds a bfloat16 mask
+ (fails on fp16 / fp32 weights) and samples even at temperature 0, so the shared block sampler runs instead."""
+ kwargs.setdefault("confidence_threshold", threshold)
+ return _block_sample(
+ base,
+ tokenizer,
+ input_ids,
+ attention_mask,
+ steps = int(steps or 32),
+ block_length = int(block_length or 32),
+ mask_format = "additive",
+ **kwargs,
+ )
+
+
+@_register("dream")
+def _dream_sample(
+ base,
+ tokenizer,
+ input_ids,
+ attention_mask,
+ *,
+ max_new_tokens,
+ steps,
+ block_length,
+ temperature,
+ mask_id,
+ eos_ids,
+ top_p = None,
+ alg = "entropy",
+ alg_temp = 0.0,
+ **unused,
+):
+ """Dream's own diffusion_generate (model card: steps = max_new_tokens, alg "entropy", alg_temp 0)."""
+ if top_p is None and temperature > 0:
+ top_p = 0.95
+ out = base.diffusion_generate(
+ input_ids,
+ attention_mask = None if attention_mask is None else attention_mask.to(input_ids.device),
+ max_new_tokens = max_new_tokens,
+ steps = int(steps or max_new_tokens),
+ temperature = temperature,
+ top_p = top_p,
+ alg = alg,
+ alg_temp = alg_temp,
+ output_history = False,
+ return_dict_in_generate = True,
+ )
+ return out.sequences[:, input_ids.shape[1] :]
+
+
+@_register("nemotron_labs_diffusion")
+def _nemotron_sample(
+ base,
+ tokenizer,
+ input_ids,
+ attention_mask,
+ *,
+ max_new_tokens,
+ steps,
+ block_length,
+ temperature,
+ mask_id,
+ eos_ids,
+ threshold = None,
+ **unused,
+):
+ """Nemotron-Labs-Diffusion's own generate: block diffusion with a causal context between blocks."""
+ block_length = int(block_length or getattr(base.config, "block_size", None) or 32)
+ output_ids, _nfe = base.generate(
+ input_ids,
+ max_new_tokens = _round_up(max_new_tokens, block_length),
+ block_length = block_length,
+ threshold = threshold,
+ temperature = temperature,
+ eos_token_id = eos_ids[0] if eos_ids else None,
+ )
+ return output_ids[:, input_ids.shape[1] :]
+
+
+@_register("diffusion_gemma")
+def _diffusion_gemma_sample(
+ base,
+ tokenizer,
+ input_ids,
+ attention_mask,
+ *,
+ max_new_tokens,
+ steps,
+ block_length,
+ temperature,
+ mask_id,
+ eos_ids,
+ **unused,
+):
+ """transformers' DiffusionGemma generate (entropy-bound sampler, its own temperature schedule)."""
+ kwargs = {"max_new_tokens": max_new_tokens}
+ if steps is not None:
+ kwargs["max_denoising_steps"] = int(steps)
+ if attention_mask is not None:
+ kwargs["attention_mask"] = attention_mask.to(input_ids.device)
+ out = base.generate(input_ids, **kwargs)
+ return getattr(out, "sequences", out)[:, input_ids.shape[1] :]
+
+
+def _cancellation(base, stop_callback):
+ if stop_callback is None:
+ yield
+ return
+
+ def check(
+ module,
+ args,
+ kwargs = None,
+ ):
+ if stop_callback():
+ raise DiffusionGenerationCancelled("Unsloth: diffusion generation cancelled.")
+
+ # Every embedding table: DiffusionGemma's generate calls encoder and decoder directly, bypassing forward.
+ targets = [m for m in base.modules() if isinstance(m, torch.nn.Embedding)] or [base]
+ handles = [m.register_forward_pre_hook(check, with_kwargs = True) for m in targets]
+ try:
+ yield
+ finally:
+ for handle in handles:
+ handle.remove()
+
+
+def _restore_attention_modes(base):
+ """Nemotron-Labs-Diffusion's generate flips each layer's `self_attn.diffusion_lm` for its causal prefill
+ and flips it back at the end; a cancelled generate would leave the model causal. Put them back."""
+ saved = [
+ (module, module.diffusion_lm)
+ for module in base.modules()
+ if isinstance(getattr(module, "diffusion_lm", None), bool)
+ ]
+ try:
+ yield
+ finally:
+ for module, value in saved:
+ module.diffusion_lm = value
+
+
+def _eval_mode(module):
+ was_training = module.training
+ module.eval()
+ try:
+ yield
+ finally:
+ module.train(was_training)
+
+
+def generate_text(
+ model,
+ tokenizer,
+ prompt = None,
+ messages = None,
+ max_new_tokens = 256,
+ steps = None,
+ block_length = None,
+ temperature = 0.0,
+ stop_callback = None,
+ seed = None,
+ **kwargs,
+):
+ """Decode one completion with the model family's diffusion sampler and return it as text (prompt
+ removed, cut at the first EOS, special tokens skipped). stop_callback() is checked before every forward;
+ returning True raises DiffusionGenerationCancelled. `steps` / `block_length` default to each family's
+ published settings; extra kwargs go to the family sampler (e.g. remasking, threshold, top_p, alg)."""
+ name = diffusion_profile_name(model)
+ sampler = _SAMPLERS.get(name)
+ if sampler is None:
+ raise ValueError(
+ f"Unsloth: no diffusion sampler for {name!r}; supported: {sorted(_SAMPLERS)}."
+ )
+ base = unwrap_diffusion_model(model)
+ device = _device(base)
+ input_ids, attention_mask = _encode(tokenizer, prompt, messages, device)
+ eos_ids = _eos_ids(base, tokenizer)
+ if messages is not None:
+ eos_ids += [i for i in _turn_end_ids(tokenizer) if i not in eos_ids]
+ mask_id = _profile_mask_id(base, tokenizer, name)
+ # manual_seed reseeds every CUDA device, so snapshot every device a sharded model uses.
+ cuda_devices = sorted({p.device.index for p in base.parameters() if p.device.type == "cuda"})
+ rng = (
+ torch.random.fork_rng(devices = cuda_devices)
+ if seed is not None
+ else contextlib.nullcontext()
+ )
+ with rng, _eval_mode(base), _restore_attention_modes(base), _cancellation(base, stop_callback):
+ if seed is not None:
+ torch.manual_seed(int(seed))
+ generated = sampler(
+ base,
+ tokenizer,
+ input_ids,
+ attention_mask,
+ max_new_tokens = int(max_new_tokens),
+ steps = steps,
+ block_length = block_length,
+ temperature = float(temperature or 0.0),
+ mask_id = mask_id,
+ eos_ids = eos_ids,
+ **kwargs,
+ )
+ tokens = generated[0][: int(max_new_tokens)].tolist()
+ for index, token in enumerate(tokens):
+ if token in eos_ids:
+ tokens = tokens[:index]
+ break
+ if mask_id is not None:
+ tokens = [token for token in tokens if token != mask_id]
+ return _text_tokenizer(tokenizer).decode(tokens, skip_special_tokens = True).strip()
diff --git a/unsloth/models/diffusion_trainer.py b/unsloth/models/diffusion_trainer.py
index 4a56423fb87..2a89effb2a7 100644
--- a/unsloth/models/diffusion_trainer.py
+++ b/unsloth/models/diffusion_trainer.py
@@ -8,9 +8,11 @@ from dataclasses import dataclass, field, fields
from typing import Optional
import torch
-from trl import SFTConfig, SFTTrainer
+from trl import SFTConfig
-from .diffusion_profiles import resolve_diffusion_profile
+from ..trainer import UnslothTrainer
+from .diffusion_loss import compile_backbone_layers, diffusion_compile_mode
+from .diffusion_profiles import resolve_diffusion_profile, unwrap_diffusion_model
__all__ = ["DiffusionConfig", "DiffusionTrainer"]
@@ -47,6 +49,19 @@ class DiffusionConfig(SFTConfig):
default = None,
metadata = {"help": "Family-specific prediction parameterisation (e.g. 'mean', 'mean_loo')."},
)
+ embedding_learning_rate: Optional[float] = field(
+ default = None,
+ metadata = {
+ "help": "Learning rate for embed_tokens / lm_head in modules_to_save (continued pretraining)."
+ },
+ )
+ diffusion_compile: Optional[str] = field(
+ default = "auto",
+ metadata = {
+ "help": "'auto' compiles the chunked loss on CUDA; 'loss' always; 'full' (or True) also compiles "
+ "each backbone layer; False / 'eager' disables. UNSLOTH_COMPILE_DISABLE=1 forces eager."
+ },
+ )
def __post_init__(self):
# Packed rows share one bidirectional attention window, so documents would see each other.
@@ -66,7 +81,7 @@ def _unwrap_config(model):
return None
-class DiffusionTrainer(SFTTrainer):
+class DiffusionTrainer(UnslothTrainer):
def __init__(
self,
model = None,
@@ -97,9 +112,12 @@ class DiffusionTrainer(SFTTrainer):
)
self.diffusion_profile = profile
self._diffusion_metrics = defaultdict(list)
+ diffusion_compile_mode(args) # reject a bad value before any work
super().__init__(model, args, *pos, **kwargs)
# The loss is ours, so the Trainer must scale it for gradient accumulation itself.
self.model_accepts_loss_kwargs = False
+ if diffusion_compile_mode(args) == "full" and profile.compile_backbone:
+ compile_backbone_layers(unwrap_diffusion_model(self.model))
def compute_loss(
self,
@@ -112,8 +130,11 @@ class DiffusionTrainer(SFTTrainer):
model, inputs, self.args, num_items_in_batch = num_items_in_batch
)
mode = "train" if model.training else "eval"
+ # Kept on device; one host sync per log() instead of one per metric per micro-batch.
for key, value in (metrics or {}).items():
- self._diffusion_metrics[f"{mode}/{key}"].append(float(value))
+ if isinstance(value, torch.Tensor):
+ value = value.detach().float().reshape(())
+ self._diffusion_metrics[f"{mode}/{key}"].append(value)
return (loss, outputs) if return_outputs else loss
def prediction_step(
@@ -134,5 +155,10 @@ class DiffusionTrainer(SFTTrainer):
for key in [k for k in self._diffusion_metrics if k.startswith(mode + "/")]:
values = self._diffusion_metrics.pop(key)
if values:
- logs[prefix + key.split("/", 1)[1]] = sum(values) / len(values)
+ tensors = [v for v in values if isinstance(v, torch.Tensor)]
+ total = sum(float(v) for v in values if not isinstance(v, torch.Tensor))
+ if tensors:
+ device = tensors[0].device
+ total += float(torch.stack([v.to(device) for v in tensors]).sum())
+ logs[prefix + key.split("/", 1)[1]] = total / len(values)
return super().log(logs, *args, **kwargs)
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 1b2f350f294..995476ce1fe 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -62,7 +62,7 @@ from .loader_utils import (
_is_offline_related_error,
raise_if_fast_inference_under_fsdp2,
)
-import os, contextlib, sys
+import os, contextlib, functools, sys
try:
from huggingface_hub import get_token
@@ -972,6 +972,26 @@ def _vllm_unavailable_error():
)
+def _diffusion_peft_kwargs(
+ model,
+ args,
+ kwargs,
+ signature_of = None,
+):
+ """The caller's get_peft_model arguments, bound against its own signature (FastLanguageModel's by default)
+ so positional ones keep their meaning, for FastDiffusionModel.get_peft_model. Omitted ones stay omitted
+ (profile defaults apply)."""
+ import inspect
+
+ bound = inspect.signature(signature_of or FastLlamaModel.get_peft_model).bind(
+ model, *args, **kwargs
+ )
+ arguments = dict(bound.arguments)
+ arguments.pop("model", None)
+ arguments.update(arguments.pop("kwargs", {}))
+ return arguments
+
+
# load_vllm-only kwargs; transformers' from_pretrained rejects them.
_VLLM_ENGINE_KWARGS = (
"max_loras",
@@ -995,6 +1015,32 @@ def _drop_vllm_engine_kwargs(kwargs):
class FastLanguageModel(FastLlamaModel):
+ # Text-diffusion models load through FastModel's slow path; these route them there for the PEFT / mode helpers.
+ @staticmethod
+ @functools.wraps(FastLlamaModel.get_peft_model)
+ def get_peft_model(model, *args, **kwargs):
+ if getattr(model, "_unsloth_slow_diffusion", False):
+ return FastDiffusionModel.get_peft_model(
+ model, **_diffusion_peft_kwargs(model, args, kwargs)
+ )
+ return FastLlamaModel.get_peft_model(model, *args, **kwargs)
+
+ @staticmethod
+ @functools.wraps(FastLlamaModel.for_inference)
+ def for_inference(model):
+ if getattr(model, "_unsloth_slow_diffusion", False):
+ return FastDiffusionModel.for_inference(model)
+ return FastLlamaModel.for_inference(model)
+
+ @staticmethod
+ @functools.wraps(FastLlamaModel.for_training)
+ def for_training(model, use_gradient_checkpointing = None):
+ if getattr(model, "_unsloth_slow_diffusion", False):
+ if use_gradient_checkpointing is None:
+ use_gradient_checkpointing = getattr(model, "_unsloth_gradient_checkpointing", True)
+ return FastDiffusionModel.for_training(model, use_gradient_checkpointing)
+ return FastLlamaModel.for_training(model, use_gradient_checkpointing)
+
@staticmethod
@tokenizers_v1_on_return
@_offline_aware_load
@@ -1872,7 +1918,10 @@ class FastModel(FastBaseModel):
def get_peft_model(model, *args, **kwargs):
# Route text-diffusion models (slow path) to the transformers-only PEFT helper.
if getattr(model, "_unsloth_slow_diffusion", False):
- return FastDiffusionModel.get_peft_model(model, *args, **kwargs)
+ return FastDiffusionModel.get_peft_model(
+ model,
+ **_diffusion_peft_kwargs(model, args, kwargs, FastBaseModel.get_peft_model),
+ )
return FastBaseModel.get_peft_model(model, *args, **kwargs)
@staticmethod