Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Train text-diffusion LMs: DiffusionTrainer with recipes for DiffusionGemma, LLaDA, LLaDA-MoE, LLaDA2, Dream, Nemotron and SDAR (#13157)

Daniel Hancd14635e2cc6
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.json
index fb5a1d610de..42f9d51234c 100644
--- a/scripts/risky_loaders_baseline.json
+++ b/scripts/risky_loaders_baseline.json
@@ -586,6 +586,14 @@
"count": 1,
"reason": "Loads the sibling decision_gguf.py by a fixed name beside this file for GGUF export on MLX, without importing unsloth.models; nothing caller-supplied picks the file."
},
+ {
+ "file": "unsloth/models/diffusion_masked.py",
+ "rule": "dynamic-import",
+ "sink": "get_class_from_dynamic_module",
+ "digest": "1900c2baf63f723f",
+ "count": 1,
+ "reason": "Falls back to the auto class unless the caller passed trust_remote_code; revision pinned to the config commit hash."
+ },
{
"file": "unsloth/models/loader.py",
"rule": "dynamic-import",
diff --git a/tests/diffusion/test_diffusion_block.py b/tests/diffusion/test_diffusion_block.py
new file mode 100644
index 00000000000..0e0964f87c7
--- /dev/null
+++ b/tests/diffusion/test_diffusion_block.py
@@ -0,0 +1,302 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Block-diffusion profiles (LLaDA2.x, SDAR): mask layout, batch building, noise, loss vs a naive reference."""
+
+import re
+import sys
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+MASK_ID = 30
+EOS_ID = 31
+VOCAB = 32
+
+
+class _Decoder(nn.Module):
+ def __init__(self, hidden = 16):
+ super().__init__()
+ self.embed = nn.Embedding(VOCAB, hidden)
+ self.pos = nn.Embedding(256, hidden)
+ self.qkv = nn.Linear(hidden, 3 * hidden)
+ self.out = nn.Linear(hidden, hidden)
+
+ def forward(
+ self,
+ input_ids,
+ attention_mask,
+ position_ids,
+ use_cache = False,
+ return_dict = True,
+ ):
+ x = self.embed(input_ids) + self.pos(position_ids)
+ q, k, v = self.qkv(x)[:, None].chunk(3, dim = -1)
+ h = F.scaled_dot_product_attention(q, k, v, attn_mask = attention_mask)[:, 0]
+ return (x + self.out(h),)
+
+
+class _LM(nn.Module):
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.model = _Decoder()
+ self.lm_head = nn.Linear(16, VOCAB)
+ self.config = SimpleNamespace(
+ model_type = "llada2_moe", eos_token_id = EOS_ID, mask_token_id = MASK_ID
+ )
+
+ def get_decoder(self):
+ return self.model
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+
+def _profiles():
+ from unsloth.models.diffusion_block import LLADA2_PROFILE, SDAR_PROFILE
+ return LLADA2_PROFILE, SDAR_PROFILE
+
+
+def _batch(prompt_lengths, real_lengths, length):
+ torch.manual_seed(1)
+ batch = len(real_lengths)
+ ids = torch.randint(0, 29, (batch, length))
+ attention = torch.zeros(batch, length, dtype = torch.long)
+ labels = torch.full((batch, length), -100)
+ for i, (p, r) in enumerate(zip(prompt_lengths, real_lengths)):
+ attention[i, :r] = 1
+ labels[i, p:r] = ids[i, p:r]
+ ids[i, r:] = EOS_ID
+ return {"input_ids": ids, "attention_mask": attention, "labels": labels}
+
+
+def test_block_mask_small_layout():
+ from unsloth.models.diffusion_block import block_diffusion_attention_mask
+
+ # L = 4, block 2: rows/cols 0-3 noisy, 4-7 clean.
+ expected = torch.tensor(
+ [
+ [1, 1, 0, 0, 0, 0, 0, 0],
+ [1, 1, 0, 0, 0, 0, 0, 0],
+ [0, 0, 1, 1, 1, 1, 0, 0],
+ [0, 0, 1, 1, 1, 1, 0, 0],
+ [0, 0, 0, 0, 1, 1, 0, 0],
+ [0, 0, 0, 0, 1, 1, 0, 0],
+ [0, 0, 0, 0, 1, 1, 1, 1],
+ [0, 0, 0, 0, 1, 1, 1, 1],
+ ],
+ dtype = torch.bool,
+ )
+ assert torch.equal(block_diffusion_attention_mask(4, 2), expected)
+
+
+def test_profiles_resolve_and_route():
+ from unsloth.models.diffusion import is_diffusion_model_type
+ from unsloth.models.diffusion_profiles import resolve_diffusion_profile
+
+ llada2, sdar = _profiles()
+ assert resolve_diffusion_profile(SimpleNamespace(model_type = "llada2_moe")) is llada2
+ assert (
+ resolve_diffusion_profile(
+ SimpleNamespace(model_type = "x", architectures = ["SDARForCausalLM"])
+ )
+ is sdar
+ )
+ assert is_diffusion_model_type("llada2_moe") and is_diffusion_model_type("sdar")
+ assert not is_diffusion_model_type("qwen3")
+
+
+def test_llada2_lora_targets_skip_routed_experts_and_router():
+ llada2, _ = _profiles()
+ pattern = re.compile(llada2.lora_target_modules)
+ hit = [
+ "model.layers.0.attention.query_key_value",
+ "model.layers.0.attention.dense",
+ "model.layers.0.mlp.gate_proj",
+ "model.layers.3.mlp.shared_experts.down_proj",
+ ]
+ miss = [
+ "model.layers.3.mlp.experts.7.up_proj",
+ "model.layers.3.mlp.gate",
+ "lm_head",
+ "model.word_embeddings",
+ ]
+ assert all(pattern.fullmatch(name) for name in hit)
+ assert not any(pattern.fullmatch(name) for name in miss)
+
+
+def test_llada2_eos_fill_is_block_aligned_and_supervised():
+ llada2, _ = _profiles()
+ model, args = _LM(), SimpleNamespace()
+ inputs = _batch([3, 5, 2], [40, 20, 9], 40)
+ inputs["labels"][2] = -100 # a row with nothing to learn gets no EOS target
+ clean, maskable, valid = llada2.build_batch(model, inputs, args)
+ assert clean.shape[1] == 64 # next multiple of 32 past 40
+ assert torch.equal(
+ clean[:, :40][inputs["attention_mask"].bool()],
+ inputs["input_ids"][inputs["attention_mask"].bool()],
+ )
+ assert (clean[0, 40:] == EOS_ID).all() and maskable[0, 40:].all()
+ assert (clean[1, 20:] == EOS_ID).all() and maskable[1, 20:].all() and not maskable[1, :5].any()
+ assert not maskable[2].any()
+
+
+def test_llada2_row_clipped_at_max_length_gets_no_eos_tail():
+ llada2, _ = _profiles()
+ inputs = _batch([3, 5], [40, 20], 40)
+ clean, maskable, valid = llada2.build_batch(_LM(), inputs, SimpleNamespace(max_length = 40))
+ assert not maskable[0, 40:].any() and valid[0, 40:].all() and (clean[0, 40:] == EOS_ID).all()
+ assert (clean[1, 20:] == EOS_ID).all() and maskable[1, 20:].all()
+
+
+def test_left_padding_rejected():
+ llada2, _ = _profiles()
+ inputs = _batch([2], [10], 12)
+ inputs["attention_mask"] = inputs["attention_mask"].flip(1)
+ with pytest.raises(ValueError, match = "right padding"):
+ llada2.build_batch(_LM(), inputs, SimpleNamespace())
+
+
+def _naive_loss(profile, model, clean, noisy, masked, maskable, p, valid, block):
+ from unsloth.models.diffusion_block import block_diffusion_attention_mask
+
+ batch, length = clean.shape
+ allowed = (
+ block_diffusion_attention_mask(length, block)[None, None].expand(batch, 1, -1, -1).clone()
+ )
+ if profile.defaults["key_padding"]:
+ keys = torch.cat([valid, valid], 1)[:, None, None, :]
+ allowed = (allowed & keys) | torch.eye(2 * length, dtype = torch.bool)
+ positions = torch.arange(length)
+ hidden = model.model(
+ torch.cat([noisy, clean], 1),
+ allowed,
+ torch.cat([positions, positions])[None].expand(batch, -1),
+ )[0]
+ logits = model.lm_head(hidden)[:, :length]
+ nll = F.cross_entropy(logits.transpose(1, 2), clean, reduction = "none")
+ if profile.defaults["diffusion_time_weighting"] == "inverse_t":
+ nll = nll / p[:, None]
+ nll = nll * masked
+ denominator = maskable.sum() if profile.defaults["normalize"] == "supervised" else masked.sum()
+ return nll.sum() / denominator
+
+
[email protected]("which", ["llada2", "sdar"])
+def test_loss_matches_naive_full_forward(which):
+ llada2, sdar = _profiles()
+ profile = llada2 if which == "llada2" else sdar
+ model = _LM()
+ args = SimpleNamespace(diffusion_block_size = 4)
+ clean, maskable, valid = profile.build_batch(model, _batch([3, 6], [22, 13], 22), args)
+ torch.manual_seed(2)
+ noisy, masked, p = profile.sample_noise(clean, maskable, MASK_ID, args)
+ ours = profile.loss_from_noise(model, clean, noisy, masked, maskable, p, valid, args)
+ ref = _naive_loss(profile, model, clean, noisy, masked, maskable, p, valid, 4)
+ torch.testing.assert_close(ours, ref, rtol = 1e-5, atol = 1e-6)
+
+
+def test_noisy_block_never_sees_its_own_or_later_clean_tokens():
+ llada2, _ = _profiles()
+ model, args = _LM(), SimpleNamespace(diffusion_block_size = 4)
+ clean = torch.randint(0, 29, (1, 12))
+ noisy = clean.clone()
+ noisy[:, 4:] = MASK_ID
+ valid = torch.ones_like(clean, dtype = torch.bool)
+ base, _ = llada2.noisy_hidden_and_head(model, noisy, clean, valid, 4)
+ changed = clean.clone()
+ changed[:, 4:] = (changed[:, 4:] + 1) % 29
+ moved, _ = llada2.noisy_hidden_and_head(model, noisy, changed, valid, 4)
+ torch.testing.assert_close(base[:, :8], moved[:, :8]) # block 1 sees clean block 0 only
+ assert not torch.allclose(base[:, 8:], moved[:, 8:]) # block 2 sees the changed clean block 1
+
+
+def test_sdar_padding_does_not_leak_into_real_tokens():
+ _, sdar = _profiles()
+ model, args = _LM(), SimpleNamespace(diffusion_block_size = 4)
+ short = _batch([2], [9], 9)
+ padded = _batch([2], [9], 16)
+ padded["input_ids"][:, :9] = short["input_ids"]
+ padded["labels"][:, :9] = short["labels"]
+ outs = []
+ for inputs in (short, padded):
+ clean, maskable, valid = sdar.build_batch(model, inputs, args)
+ noisy = torch.where(maskable, MASK_ID, clean)
+ hidden, _ = sdar.noisy_hidden_and_head(model, noisy, clean, valid, 4)
+ outs.append(hidden[:, :9])
+ torch.testing.assert_close(outs[0], outs[1], rtol = 1e-5, atol = 1e-6)
+
+
+def test_noise_ranges_and_prompt_untouched():
+ llada2, sdar = _profiles()
+ clean = torch.randint(0, 29, (2048, 24))
+ maskable = torch.ones_like(clean, dtype = torch.bool)
+ maskable[:, :5] = False
+ for profile, (low, high) in ((llada2, (0.3, 0.8)), (sdar, (1e-3, 1.0))):
+ torch.manual_seed(0)
+ noisy, masked, p = profile.sample_noise(clean, maskable, MASK_ID, SimpleNamespace())
+ assert p.min() >= low and p.max() <= high
+ assert not masked[:, :5].any() and torch.equal(noisy == MASK_ID, masked)
+ ratio = masked[:, 5:].float().mean().item()
+ assert abs(ratio - (low + high) / 2) < 0.02
+ _, masked, _ = sdar.sample_noise(clean[:, :6], maskable[:, :6], MASK_ID, SimpleNamespace())
+ assert masked.any(dim = 1).all() # SDAR masks at least one supervised token per row
+ _, _, p = llada2.sample_noise(clean, maskable, MASK_ID, SimpleNamespace(diffusion_eps = 0.1))
+ assert p.min() >= 0.1 and p.max() <= 0.9
+
+
+def test_sdar_import_shim_is_scoped():
+ from unsloth.models.diffusion_block import _sdar_remote_code_imports
+ import importlib.util
+
+ if importlib.util.find_spec("flash_attn") is not None:
+ pytest.skip("flash-attn installed: the shim is a no-op")
+ with _sdar_remote_code_imports():
+ from flash_attn.ops.triton.layer_norm import rms_norm_fn
+ x = torch.randn(3, 8)
+ torch.testing.assert_close(
+ rms_norm_fn(x, torch.ones(8), eps = 1e-6),
+ x * torch.rsqrt(x.pow(2).mean(-1, keepdim = True) + 1e-6),
+ )
+ assert "flash_attn" not in sys.modules
+
+
+def test_restore_rotary_buffers_refills_zeroed_inv_freq():
+ from unsloth.models.diffusion_profiles import restore_rotary_buffers
+
+ def init_fn(config, device = None):
+ dim = config.head_dim
+ return 1.0 / (
+ config.rope_theta ** (torch.arange(0, dim, 2, dtype = torch.float, device = device) / dim)
+ ), 1.0
+
+ class Rotary(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(head_dim = 8, rope_theta = 10000.0)
+ self.rope_init_fn = init_fn
+ self.register_buffer("inv_freq", torch.zeros(4), persistent = False)
+ self.original_inv_freq = self.inv_freq
+
+ holder = nn.Module()
+ holder.rotary_emb = Rotary()
+ assert restore_rotary_buffers(holder) == 1
+ torch.testing.assert_close(holder.rotary_emb.inv_freq, init_fn(holder.rotary_emb.config)[0])
+ assert restore_rotary_buffers(holder) == 0 # idempotent
+
+
+def test_deepspeed_engine_keeps_its_own_forward():
+ from unsloth.models.diffusion_block import _is_distributed_wrapper
+ engine = type("DeepSpeedEngine", (nn.Module,), {})()
+ assert _is_distributed_wrapper(engine) and not _is_distributed_wrapper(_LM())
+
+
+def test_unknown_time_weighting_is_rejected():
+ llada2, _ = _profiles()
+ inputs = _batch([3], [20], 20)
+ with pytest.raises(ValueError, match = "diffusion_time_weighting"):
+ llada2.compute_loss(_LM(), inputs, SimpleNamespace(diffusion_time_weighting = "inverse-t"))
diff --git a/tests/diffusion/test_diffusion_gemma.py b/tests/diffusion/test_diffusion_gemma.py
new file mode 100644
index 00000000000..85a28393f4e
--- /dev/null
+++ b/tests/diffusion/test_diffusion_gemma.py
@@ -0,0 +1,279 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+import os
+import types
+
+import pytest
+from real_accelerator import has_real_cuda
+
+torch = pytest.importorskip("torch")
+transformers = pytest.importorskip("transformers")
+if not hasattr(transformers, "DiffusionGemmaForBlockDiffusion"):
+ pytest.skip("transformers without DiffusionGemma", allow_module_level = True)
+
+from unsloth.models.diffusion_gemma_objective import DIFFUSION_GEMMA_PROFILE
+from unsloth.models.diffusion_profiles import resolve_diffusion_profile
+
+TINY = "trl-internal-testing/tiny-DiffusionGemmaForBlockDiffusion"
+TRL_EXAMPLE = os.environ.get("UNSLOTH_TRL_DIFFUSION_GEMMA_EXAMPLE")
+
+
[email protected](scope = "module")
+def model():
+ m = transformers.DiffusionGemmaForBlockDiffusion.from_pretrained(TINY, dtype = torch.float32)
+ m._unsloth_diffusion_eos_token_id = 1
+ return m.train()
+
+
+def _batch(
+ vocab,
+ canvas,
+ max_length = 80,
+ rows = None,
+):
+ g = torch.Generator().manual_seed(0)
+ if rows is None:
+ rows = [(5, 10), (7, canvas * 2 + 3), (4, max_length - 4)]
+ input_ids = torch.zeros(len(rows), max_length, dtype = torch.long)
+ attention_mask = torch.zeros_like(input_ids)
+ labels = torch.full_like(input_ids, -100)
+ for i, (p, r) in enumerate(rows):
+ n = p + r
+ input_ids[i, :n] = torch.randint(10, vocab, (n,), generator = g)
+ attention_mask[i, :n] = 1
+ labels[i, p:n] = input_ids[i, p:n]
+ return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}
+
+
+def _args(**kw):
+ base = dict(
+ max_length = 80,
+ diffusion_eps = None,
+ diffusion_self_conditioning_p = None,
+ diffusion_encoder_ar_weight = None,
+ diffusion_prediction_type = None,
+ )
+ base.update(kw)
+ return types.SimpleNamespace(**base)
+
+
+def test_profile_resolves(model):
+ assert resolve_diffusion_profile(model.config) is DIFFUSION_GEMMA_PROFILE
+
+
+def test_canvas_reaches_the_decoder(model):
+ # The notebook passed canvas_ids, which forward() silently swallows; decoder_input_ids must change the logits.
+ cfg = model.config
+ ids = torch.randint(10, 1000, (1, 8))
+ a = torch.randint(10, 1000, (1, cfg.canvas_length))
+ b = torch.randint(10, 1000, (1, cfg.canvas_length))
+ with torch.no_grad():
+ la = model(input_ids = ids, decoder_input_ids = a).logits
+ lb = model(input_ids = ids, decoder_input_ids = b).logits
+ assert not torch.allclose(la, lb)
+
+
+def test_loss_finite_and_metrics(model):
+ batch = _batch(model.config.text_config.vocab_size, model.config.canvas_length)
+ torch.manual_seed(0)
+ loss, _, metrics = DIFFUSION_GEMMA_PROFILE.compute_loss(model, batch, _args())
+ assert torch.isfinite(loss)
+ assert set(metrics) == {"diffusion_loss", "encoder_ar_loss", "mean_t"}
+
+
+ TRL_EXAMPLE is None, reason = "set UNSLOTH_TRL_DIFFUSION_GEMMA_EXAMPLE to TRL's example file"
+)
[email protected]("prediction_type", ["mean", "mean_loo"])
+def test_parity_with_trl_reference(model, prediction_type):
+ source = open(TRL_EXAMPLE, encoding = "utf-8").read()
+ source = source.replace(
+ "from trl.trainer.utils import maybe_gather_lm_head_ctx",
+ "import contextlib\nmaybe_gather_lm_head_ctx = lambda *a: contextlib.nullcontext()",
+ )
+ namespace = {"__name__": "trl_example"}
+ exec(compile(source, TRL_EXAMPLE, "exec"), namespace)
+ reference = namespace["DiffusionGemmaSFTTrainer"]
+ cfg = model.config
+ fake = types.SimpleNamespace(
+ args = types.SimpleNamespace(max_length = 80),
+ canvas_length = cfg.canvas_length,
+ vocab_size = cfg.text_config.vocab_size,
+ final_logit_softcapping = cfg.text_config.final_logit_softcapping,
+ eos_token_id = 1,
+ model_prediction_type = prediction_type,
+ model = model,
+ eps = reference.eps,
+ self_conditioning_p = reference.self_conditioning_p,
+ )
+ batch = _batch(cfg.text_config.vocab_size, cfg.canvas_length)
+ params = [p for p in model.parameters() if p.requires_grad]
+
+ def run(fn):
+ model.zero_grad(set_to_none = True)
+ torch.manual_seed(1234)
+ loss = fn()
+ loss.backward()
+ return loss.detach(), [None if p.grad is None else p.grad.clone() for p in params]
+
+ ref_loss, ref_grads = run(lambda: reference.compute_loss(fake, model, batch))
+ our_loss, our_grads = run(
+ lambda: DIFFUSION_GEMMA_PROFILE.compute_loss(
+ model, batch, _args(diffusion_prediction_type = prediction_type)
+ )[0]
+ )
+ torch.testing.assert_close(our_loss, ref_loss, rtol = 0, atol = 1e-6)
+ for a, b in zip(our_grads, ref_grads):
+ if a is None or b is None:
+ assert a is None and b is None
+ else:
+ torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-7)
+
+
+def test_tied_encoder_decoder_lora_merge_is_refused():
+ from peft import LoraConfig, get_peft_model
+
+ from unsloth.models.diffusion import _refuse_tied_lora_merge
+
+ m = transformers.DiffusionGemmaForBlockDiffusion.from_pretrained(TINY, dtype = torch.float32)
+ enc = m.model.encoder.language_model.layers[0].self_attn.q_proj
+ dec = m.model.decoder.layers[0].self_attn.q_proj
+ assert enc is not dec and enc.weight.data_ptr() == dec.weight.data_ptr()
+ m = get_peft_model(
+ m, LoraConfig(r = 4, target_modules = DIFFUSION_GEMMA_PROFILE.lora_target_modules)
+ )
+ m = _refuse_tied_lora_merge(m)
+ with pytest.raises(RuntimeError, match = "share base weights"):
+ m.merge_and_unload()
+
+
+def test_untied_lora_merge_still_allowed():
+ from peft import LoraConfig, get_peft_model
+
+ from unsloth.models.diffusion import _refuse_tied_lora_merge
+
+ m = transformers.AutoModelForCausalLM.from_pretrained(
+ "trl-internal-testing/tiny-Qwen3ForCausalLM"
+ )
+ m = _refuse_tied_lora_merge(
+ get_peft_model(m, LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))
+ )
+ assert "merge_and_unload" not in vars(m)
+ m.merge_and_unload()
+
+
+class _DistributedLike(torch.nn.Module):
+ """DDP / FSDP expose neither config nor heads; only forward reaches the module."""
+
+ def __init__(self, inner):
+ super().__init__()
+ self.module = inner
+
+ def forward(self, **kwargs):
+ return self.module(**kwargs)
+
+
+def test_loss_through_distributed_wrapper(model):
+ batch = _batch(model.config.text_config.vocab_size, model.config.canvas_length)
+ torch.manual_seed(0)
+ plain = DIFFUSION_GEMMA_PROFILE.compute_loss(model, batch, _args())[0]
+ torch.manual_seed(0)
+ wrapped = DIFFUSION_GEMMA_PROFILE.compute_loss(_DistributedLike(model), batch, _args())[0]
+ torch.testing.assert_close(wrapped, plain)
+
+
+def test_row_without_supervised_tokens_adds_no_canvas_loss(model):
+ batch = _batch(model.config.text_config.vocab_size, model.config.canvas_length, rows = [(6, 0)])
+ torch.manual_seed(0)
+ _, _, metrics = DIFFUSION_GEMMA_PROFILE.compute_loss(model, batch, _args())
+ assert metrics["diffusion_loss"] == 0.0
+
+
+def test_reloaded_adapter_still_refuses_merge(tmp_path):
+ from peft import LoraConfig, get_peft_model
+
+ from unsloth import FastModel
+
+ m = transformers.DiffusionGemmaForBlockDiffusion.from_pretrained(TINY, dtype = torch.float32)
+ m = get_peft_model(
+ m, LoraConfig(r = 4, target_modules = DIFFUSION_GEMMA_PROFILE.lora_target_modules)
+ )
+ m.save_pretrained(tmp_path)
+ reloaded, _ = FastModel.from_pretrained(str(tmp_path), dtype = torch.float32, device_map = "cpu")
+ with pytest.raises(RuntimeError, match = "share base weights"):
+ reloaded.merge_and_unload()
+ # The objective reads its token bookkeeping from the unwrapped module.
+ from unsloth.models.diffusion_profiles import unwrap_diffusion_model
+
+ assert "_unsloth_diffusion_eos_token_id" in vars(unwrap_diffusion_model(reloaded))
+
+
+def test_reload_uses_the_adapters_saved_tokenizer(tmp_path):
+ from peft import LoraConfig, get_peft_model
+
+ from unsloth import FastModel
+
+ m = transformers.DiffusionGemmaForBlockDiffusion.from_pretrained(TINY, dtype = torch.float32)
+ m = get_peft_model(
+ m, LoraConfig(r = 4, target_modules = DIFFUSION_GEMMA_PROFILE.lora_target_modules)
+ )
+ m.save_pretrained(tmp_path)
+ tokenizer = transformers.AutoTokenizer.from_pretrained(TINY)
+ tokenizer.chat_template = "{{ 'adapter-template-marker' }}"
+ tokenizer.save_pretrained(tmp_path)
+ _, reloaded = FastModel.from_pretrained(str(tmp_path), dtype = torch.float32, device_map = "cpu")
+ reloaded = getattr(reloaded, "tokenizer", reloaded)
+ assert "adapter-template-marker" in (reloaded.chat_template or "")
+
+
+def test_diffusion_load_restores_progress_bars():
+ from huggingface_hub.utils import are_progress_bars_disabled, enable_progress_bars
+
+ from unsloth import FastModel
+
+ enable_progress_bars()
+ FastModel.from_pretrained(TINY, dtype = torch.float32, device_map = "cpu")
+ assert not are_progress_bars_disabled()
+
+
[email protected](not has_real_cuda(), reason = "bitsandbytes needs a GPU")
+def test_caller_quantization_config_is_kept():
+ from transformers import BitsAndBytesConfig
+
+ from unsloth import FastModel
+
+ qcfg = BitsAndBytesConfig(load_in_4bit = True, bnb_4bit_quant_type = "fp4")
+ m, _ = FastModel.from_pretrained(TINY, quantization_config = qcfg, load_in_4bit = True)
+ assert m.config.quantization_config.bnb_4bit_quant_type == "fp4"
+
+
+def test_self_conditioning_off_skips_the_conditioning_pass(model):
+ batch = _batch(model.config.text_config.vocab_size, model.config.canvas_length)
+ calls = []
+ hook = model.register_forward_pre_hook(lambda *a: calls.append(1))
+ try:
+ torch.manual_seed(0)
+ DIFFUSION_GEMMA_PROFILE.compute_loss(model, batch, _args(diffusion_self_conditioning_p = 0.0))
+ finally:
+ hook.remove()
+ assert len(calls) == 1
+
+
+def test_full_finetuning_skips_lora():
+ from unsloth import FastModel
+
+ m, _ = FastModel.from_pretrained(
+ TINY, dtype = torch.float32, device_map = "cpu", full_finetuning = True
+ )
+ assert FastModel.get_peft_model(m, r = 4) is m
+ assert not any("lora_" in name for name, _ in m.named_parameters())
+
+
+def test_caller_config_is_the_one_loaded():
+ from unsloth import FastModel
+
+ config = transformers.AutoConfig.from_pretrained(TINY)
+ config.unsloth_test_marker = 7
+ m, _ = FastModel.from_pretrained(TINY, config = config, dtype = torch.float32, device_map = "cpu")
+ assert getattr(m.config, "unsloth_test_marker", None) == 7
diff --git a/tests/diffusion/test_diffusion_masked.py b/tests/diffusion/test_diffusion_masked.py
new file mode 100644
index 00000000000..313e5f38495
--- /dev/null
+++ b/tests/diffusion/test_diffusion_masked.py
@@ -0,0 +1,475 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Masked-diffusion recipes reproduce each family's published loss code on the same batch and seed."""
+
+import os
+import re
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+VOCAB = 97
+MASK_ID = 90
+EOS_ID = 3
+PAD_ID = 5
+
+
+class _Bidirectional(torch.nn.Module):
+ """Every position sees the whole row, so a wrong mask or fill changes every logit."""
+
+ def __init__(self, mask_token_id = MASK_ID):
+ super().__init__()
+ torch.manual_seed(0)
+ self.embed = torch.nn.Embedding(VOCAB, 16)
+ self.mix = torch.nn.Linear(16, 16)
+ self.head = torch.nn.Linear(16, VOCAB)
+ self.config = SimpleNamespace(mask_token_id = mask_token_id, eos_token_id = EOS_ID)
+ self.seen = []
+
+ def forward(
+ self,
+ input_ids,
+ attention_mask = None,
+ ):
+ self.seen.append((input_ids.clone(), attention_mask))
+ h = self.embed(input_ids)
+ h = h + self.mix(h.mean(dim = 1, keepdim = True))
+ return SimpleNamespace(logits = self.head(torch.tanh(h)))
+
+
+def _batch():
+ # (prompt, response) lengths; right padded with a pad id distinct from EOS.
+ shapes = [(4, 7), (6, 3), (3, 5)]
+ length = max(p + r for p, r in shapes) + 2
+ gen = torch.Generator().manual_seed(1)
+ input_ids = torch.full((len(shapes), length), PAD_ID)
+ attention_mask = torch.zeros_like(input_ids)
+ labels = torch.full_like(input_ids, -100)
+ prompt_lengths = []
+ for i, (p, r) in enumerate(shapes):
+ row = torch.randint(10, 80, (p + r,), generator = gen)
+ row[-1] = EOS_ID
+ input_ids[i, : p + r] = row
+ attention_mask[i, : p + r] = 1
+ labels[i, p : p + r] = row[p:]
+ prompt_lengths.append(p)
+ return {
+ "input_ids": input_ids,
+ "attention_mask": attention_mask,
+ "labels": labels,
+ }, torch.tensor(prompt_lengths)
+
+
+def _eos_filled(inputs):
+ return inputs["input_ids"].masked_fill(inputs["attention_mask"] == 0, EOS_ID)
+
+
+def _profile(name):
+ from unsloth.models.diffusion_profiles import diffusion_profiles
+ return {p.name: p for p in diffusion_profiles()}[name]
+
+
+# LLaDA GUIDELINES.md, "SFT" section, verbatim apart from names.
+def _llada_reference(
+ model,
+ input_ids,
+ prompt_length,
+ mask_id,
+ eps = 1e-3,
+):
+ b, l = input_ids.shape
+ t = torch.rand(b, device = input_ids.device)
+ p_mask = (1 - eps) * t + eps
+ p_mask = p_mask[:, None].repeat(1, l)
+ masked_indices = torch.rand((b, l), device = input_ids.device) < p_mask
+ noisy_batch = torch.where(masked_indices, mask_id, input_ids)
+ token_positions = torch.arange(noisy_batch.shape[1]).expand(
+ noisy_batch.size(0), noisy_batch.size(1)
+ )
+ prompt_mask = token_positions < prompt_length.unsqueeze(1)
+ noisy_batch[prompt_mask] = input_ids[prompt_mask]
+ prompt_mask = prompt_mask.to(torch.int64)
+ answer_lengths = torch.sum((1 - prompt_mask), dim = -1, keepdim = True)
+ answer_lengths = answer_lengths.repeat(1, noisy_batch.shape[1])
+ masked_indices = noisy_batch == mask_id
+ logits = model(input_ids = noisy_batch).logits
+ token_loss = (
+ F.cross_entropy(logits[masked_indices], input_ids[masked_indices], reduction = "none")
+ / p_mask[masked_indices]
+ )
+ return torch.sum(token_loss / answer_lengths[masked_indices]) / input_ids.shape[0]
+
+
+# Dream src/trainer/fsdp_sft_trainer.py _compute_loss_and_backward + src/diffllm/gen_utils.q_sample,
+# with the default config (time_reweighting "original", no token reweighting, treat_eos_as_one off).
+def _dream_reference(model, input_ids, loss_mask, mask_id):
+ t = torch.rand((input_ids.shape[0],), dtype = torch.float)
+ u = torch.rand_like(input_ids, dtype = torch.float)
+ t_mask = (u < t[:, None]) & loss_mask
+ masked_input_ids = input_ids.masked_fill(t_mask, mask_id)
+ logits = model(input_ids = masked_input_ids).logits
+ shift_logits = torch.cat([logits[:, 0:1], logits[:, :-1]], dim = 1)
+ loss = F.cross_entropy(shift_logits.reshape(-1, VOCAB), input_ids.reshape(-1), reduction = "none")
+ loss = loss.masked_fill(~t_mask.reshape(-1), 0)
+ loss = loss * (1 / t[:, None].float().expand(input_ids.size())).reshape(-1)
+ return torch.sum(loss) / torch.sum(t_mask)
+
+
+# Nemotron-Labs-Diffusion modeling_nemotron_labs_diffusion.py forward_process + bidirectional loss branch;
+# the (sum, count) pair it returns is divided by the trainer, as global_loss_avg does on one rank.
+def _nemotron_reference(
+ model,
+ input_ids,
+ loss_mask,
+ mask_id,
+ eps = 1e-3,
+):
+ b, l = input_ids.shape
+ t = torch.rand(b)
+ p_mask = ((1 - eps) * t + eps)[:, None].expand(-1, l)
+ masked_indices = torch.rand((b, l)) < p_mask
+ masked_indices[loss_mask == 0] = 0
+ noisy = torch.where(masked_indices, mask_id, input_ids)
+ logits = model(input_ids = noisy).logits
+ token_loss = (
+ F.cross_entropy(logits[masked_indices], input_ids[masked_indices], reduction = "none")
+ / p_mask[masked_indices]
+ )
+ return token_loss.sum() / masked_indices.sum()
+
+
+def _ours(name, model, inputs, seed):
+ torch.manual_seed(seed)
+ loss, _, metrics = _profile(name).compute_loss(model, inputs, None)
+ return loss, metrics
+
+
[email protected]("name", ["llada", "llada_moe"])
[email protected]("seed", [0, 1, 2])
+def test_llada_matches_guidelines_sft_loss(name, seed):
+ inputs, prompt_lengths = _batch()
+ model = _Bidirectional()
+ torch.manual_seed(seed)
+ reference = _llada_reference(model, _eos_filled(inputs), prompt_lengths, MASK_ID)
+ loss, _ = _ours(name, model, inputs, seed)
+ assert torch.allclose(loss, reference, atol = 0, rtol = 1e-6), (
+ loss.item(),
+ reference.item(),
+ (loss - reference).item(),
+ )
+
+
[email protected]("seed", [0, 1, 2])
+def test_dream_matches_official_sft_loss(seed):
+ inputs, _ = _batch()
+ model = _Bidirectional()
+ loss_mask = (inputs["labels"] != -100) | (inputs["attention_mask"] == 0)
+ torch.manual_seed(seed)
+ reference = _dream_reference(model, _eos_filled(inputs), loss_mask, MASK_ID)
+ loss, _ = _ours("dream", model, inputs, seed)
+ assert torch.allclose(loss, reference, atol = 0, rtol = 1e-6), (
+ loss.item(),
+ reference.item(),
+ (loss - reference).item(),
+ )
+
+
[email protected]("seed", [0, 1, 2])
+def test_nemotron_matches_remote_forward_loss(seed):
+ inputs, _ = _batch()
+ model = _Bidirectional()
+ loss_mask = (inputs["labels"] != -100) | (inputs["attention_mask"] == 0)
+ torch.manual_seed(seed)
+ reference = _nemotron_reference(model, _eos_filled(inputs), loss_mask, MASK_ID)
+ loss, _ = _ours("nemotron_labs_diffusion", model, inputs, seed)
+ assert torch.allclose(loss, reference, atol = 0, rtol = 1e-6), (
+ loss.item(),
+ reference.item(),
+ (loss - reference).item(),
+ )
+
+
[email protected]("name", ["llada", "llada_moe", "dream", "nemotron_labs_diffusion"])
+def test_prompt_stays_clean_and_padding_becomes_attended_eos(name):
+ inputs, prompt_lengths = _batch()
+ model = _Bidirectional()
+ _ours(name, model, inputs, 0)
+ noisy, attention_mask = model.seen[-1]
+ assert attention_mask is None
+ for i, p in enumerate(prompt_lengths.tolist()):
+ assert torch.equal(noisy[i, :p], inputs["input_ids"][i, :p])
+ padding = inputs["attention_mask"] == 0
+ assert set(noisy[padding].tolist()) <= {EOS_ID, MASK_ID}
+ assert PAD_ID not in noisy.tolist()
+
+
+def test_unsupervised_batch_masks_every_attended_token_and_no_padding():
+ inputs, _ = _batch()
+ inputs.pop("labels")
+ model = _Bidirectional()
+ torch.manual_seed(0)
+ _profile("llada").compute_loss(model, inputs, None)
+ noisy, _ = model.seen[-1]
+ assert not (noisy[inputs["attention_mask"] == 0] == MASK_ID).any()
+
+
+class _DistributedLike(torch.nn.Module):
+ """DDP / FSDP expose neither config nor Unsloth attributes; only forward reaches the module."""
+
+ def __init__(self, inner):
+ super().__init__()
+ self.module = inner
+
+ def forward(self, **kwargs):
+ return self.module(**kwargs)
+
+
[email protected]("name", ["llada", "dream", "nemotron_labs_diffusion"])
+def test_loss_through_distributed_wrapper(name):
+ inputs, _ = _batch()
+ model = _Bidirectional()
+ plain, _ = _ours(name, model, inputs, 0)
+ wrapped, _ = _ours(name, _DistributedLike(model), inputs, 0)
+ torch.testing.assert_close(wrapped, plain)
+
+
+def test_row_without_response_gets_no_eos_targets():
+ inputs, _ = _batch()
+ inputs["labels"][1] = -100
+ model = _Bidirectional()
+ torch.manual_seed(0)
+ _profile("llada").compute_loss(model, inputs, None)
+ noisy, _ = model.seen[-1]
+ assert not (noisy[1] == MASK_ID).any()
+
+
+def test_mask_token_falls_back_to_family_default():
+ inputs, _ = _batch()
+ model = _Bidirectional(mask_token_id = None)
+ _profile("llada_moe").prepare_model(model, None)
+ assert model._unsloth_mask_token_id == 156895
+ model = _Bidirectional(mask_token_id = None)
+ _profile("llada").prepare_model(model, None)
+ assert model._unsloth_mask_token_id == 126336
+
+
+def test_all_masked_rows_absent_gives_zero_loss_with_grad():
+ inputs, _ = _batch()
+ inputs["labels"].fill_(-100)
+ inputs["attention_mask"].fill_(1)
+ model = _Bidirectional()
+ loss, _ = _ours("dream", model, inputs, 0)
+ assert loss.item() == 0.0 and loss.requires_grad
+
+
+ "config, expected",
+ [
+ (dict(model_type = "llada", architectures = ["LLaDAModelLM"]), "llada"),
+ (dict(model_type = "llada", architectures = ["LLaDAMoEModel"]), "llada_moe"),
+ (dict(model_type = "Dream", architectures = ["DreamModel"]), "dream"),
+ (
+ dict(
+ model_type = "nemotron_labs_diffusion", architectures = ["NemotronLabsDiffusionModel"]
+ ),
+ "nemotron_labs_diffusion",
+ ),
+ (dict(model_type = "nemotron", architectures = ["NemotronForCausalLM"]), None),
+ (dict(model_type = "qwen3", architectures = ["Qwen3ForCausalLM"]), None),
+ ],
+)
+def test_profile_routing(config, expected):
+ from unsloth.models.diffusion_profiles import resolve_diffusion_profile
+ profile = resolve_diffusion_profile(SimpleNamespace(**config))
+ assert (profile.name if profile else None) == expected
+
+
+def test_lora_targets_skip_output_head_router_and_experts():
+ llada = _profile("llada").lora_target_modules
+ assert re.fullmatch(llada, "model.transformer.blocks.3.ff_out")
+ assert re.fullmatch(llada, "model.transformer.blocks.3.attn_out")
+ assert not re.fullmatch(llada, "model.transformer.ff_out")
+ moe = _profile("llada_moe").lora_target_modules
+ assert re.fullmatch(moe, "model.layers.0.self_attn.o_proj")
+ assert not re.fullmatch(moe, "model.layers.0.mlp.gate")
+ assert not re.fullmatch(moe, "model.layers.0.mlp.experts.7.down_proj")
+
+
+def test_rotary_buffers_rebuilt_from_rope_init_fn():
+ from unsloth.models.diffusion_profiles import restore_rotary_buffers as _rebuild_rotary_buffers
+
+ def init(config, device, **kwargs):
+ return 1.0 / (config.rope_theta ** (torch.arange(0, 8, 2, dtype = torch.float32) / 8)), 1.0
+
+ class Rotary(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(rope_theta = 10000.0)
+ self.rope_init_fn = init
+ self.rope_kwargs = {}
+ self.register_buffer("inv_freq", torch.zeros(4), persistent = False)
+ self.original_inv_freq = torch.empty(4, device = "meta")
+
+ model = torch.nn.Module()
+ model.config = SimpleNamespace()
+ model.rotary = Rotary()
+ expected = init(model.rotary.config, "cpu")[0]
+ assert _rebuild_rotary_buffers(model) == 1
+ assert torch.equal(model.rotary.inv_freq, expected)
+ assert torch.equal(model.rotary.original_inv_freq, expected)
+ assert _rebuild_rotary_buffers(model) == 0 and torch.equal(model.rotary.inv_freq, expected)
+
+
+def test_dream_generate_defaults_to_filled_model_generation_config():
+ from unsloth.models.diffusion_masked import _default_diffusion_generate_config
+
+ seen = {}
+
+ class Dream:
+ config = SimpleNamespace(
+ mask_token_id = 151666, eos_token_id = 151643, pad_token_id = 151643, bos_token_id = 151643
+ )
+ generation_config = SimpleNamespace(
+ mask_token_id = None, eos_token_id = 151643, pad_token_id = None, bos_token_id = None
+ )
+
+ def diffusion_generate(
+ self,
+ inputs = None,
+ generation_config = None,
+ **kwargs,
+ ):
+ seen["config"] = generation_config
+ return inputs
+
+ model = Dream()
+ _default_diffusion_generate_config(model)
+ _default_diffusion_generate_config(model)
+ model.diffusion_generate("x", steps = 4)
+ assert seen["config"].mask_token_id == 151666 and seen["config"].pad_token_id == 151643
+ assert model.generation_config.mask_token_id is None
+ explicit = SimpleNamespace(mask_token_id = 7)
+ model.diffusion_generate("x", generation_config = explicit)
+ assert seen["config"] is explicit
+
+
+def test_mask_builder_signature_exposed_from_wrapper_closure():
+ import inspect
+
+ from unsloth.models import diffusion_masked
+
+ def original(
+ config,
+ inputs_embeds,
+ attention_mask,
+ past_key_values,
+ position_ids = None,
+ ):
+ return None
+
+ def make(signature):
+ def return_attention_mask(*args, **kwargs):
+ return signature
+
+ return return_attention_mask
+
+ wrapper = make(inspect.signature(original))
+ fake = SimpleNamespace(create_causal_mask = wrapper, create_sliding_window_causal_mask = None)
+ import sys
+
+ real = sys.modules.get("transformers.masking_utils")
+ sys.modules["transformers.masking_utils"] = fake
+ try:
+ import transformers
+
+ saved = getattr(transformers, "masking_utils", None)
+ transformers.masking_utils = fake
+ diffusion_masked._expose_mask_signatures()
+ finally:
+ sys.modules["transformers.masking_utils"] = real
+ transformers.masking_utils = saved
+ assert list(inspect.signature(wrapper).parameters)[:2] == ["config", "inputs_embeds"]
+
+
+_TINY_TRAIN = r"""
+import math, os, sys, torch
+import unsloth
+from datasets import Dataset
+from unsloth import DiffusionConfig, DiffusionTrainer, FastModel
+
+path, out = sys.argv[1], sys.argv[2]
+device = os.environ.get("TINY_DEVICE", "cpu")
+model, processor = FastModel.from_pretrained(path, trust_remote_code = True, dtype = torch.float32, device_map = {"": device})
+assert getattr(model, "_unsloth_slow_diffusion", False), type(model)
+tokenizer = getattr(processor, "tokenizer", processor)
+for module in model.modules():
+ if hasattr(module, "rope_init_fn") and isinstance(getattr(module, "inv_freq", None), torch.Tensor):
+ assert module.inv_freq.abs().sum() > 0, "RoPE inv_freq left uninitialised"
+model = FastModel.get_peft_model(model, r = 16, lora_alpha = 32, use_gradient_checkpointing = True)
+before = {k: v.detach().clone() for k, v in model.named_parameters() if "lora_B" in k}
+rows = [{"prompt": f"Question {i}: what is {i} plus {i}?", "completion": f" The answer is {2 * i}."} for i in range(4)] * 16
+args = DiffusionConfig(
+ output_dir = out, max_steps = 20, per_device_train_batch_size = 4, learning_rate = 2e-2, logging_steps = 1,
+ report_to = [], save_strategy = "no", seed = 3407, use_cpu = device == "cpu", max_length = 64,
+ bf16 = False, fp16 = False, dataset_num_proc = 1, gradient_checkpointing = True,
+)
+trainer = DiffusionTrainer(model = model, args = args, train_dataset = Dataset.from_list(rows), processing_class = tokenizer)
+fixed = trainer._prepare_inputs(next(iter(trainer.get_train_dataloader())))
+
+
+def fixed_noise_ce():
+ # Per-step losses are too noisy over 20 steps (1 / t weight, fresh noise); score one batch under one draw.
+ torch.manual_seed(0)
+ with torch.no_grad():
+ return trainer.diffusion_profile.compute_loss(model, fixed, args)[2]["masked_ce"]
+
+
+first = fixed_noise_ce()
+trainer.train()
+losses = [h["loss"] for h in trainer.state.log_history if "loss" in h]
+assert len(losses) == 20 and all(math.isfinite(x) for x in losses), losses
+last = fixed_noise_ce()
+assert last < first, (first, last)
+after = {k: v for k, v in model.named_parameters() if k in before}
+assert any(not torch.equal(before[k], after[k].detach()) for k in before)
+model.save_pretrained(os.path.join(out, "adapter"))
+tokenizer.save_pretrained(os.path.join(out, "adapter"))
+reloaded, _ = FastModel.from_pretrained(os.path.join(out, "adapter"), trust_remote_code = True, dtype = torch.float32, device_map = {"": device})
+saved = {k.replace(".default", ""): v for k, v in model.named_parameters() if "lora_B" in k}
+loaded = {k.replace(".default", ""): v for k, v in reloaded.named_parameters() if "lora_B" in k}
+assert saved.keys() == loaded.keys() and saved, (list(saved)[:3], list(loaded)[:3])
+assert all(torch.equal(saved[k].detach().cpu(), loaded[k].detach().cpu()) for k in saved)
+print("TINY_OK", trainer.diffusion_profile.name, round(first, 4), round(last, 4))
+"""
+
+
+# Real remote code, tiny random weights: UNSLOTH_DIFFUSION_TINY_DIR/<name> built from the published config and code.
+_TINY = os.environ.get("UNSLOTH_DIFFUSION_TINY_DIR")
+
+
[email protected](not _TINY, reason = "set UNSLOTH_DIFFUSION_TINY_DIR to tiny remote-code copies")
+ "name",
+ [
+ "LLaDA-8B-Instruct",
+ "LLaDA-MoE-7B-A1B-Instruct",
+ "Dream-v0-Instruct-7B",
+ "Nemotron-Labs-Diffusion-3B",
+ ],
+)
+def test_tiny_remote_checkpoint_trains_and_reloads(name, tmp_path):
+ import subprocess
+ import sys
+
+ # One process per family: LLaDA's remote config registers "llada" globally, shadowing LLaDA-MoE's.
+ out = subprocess.run(
+ [sys.executable, "-c", _TINY_TRAIN, os.path.join(_TINY, name), str(tmp_path)],
+ capture_output = True,
+ text = True,
+ timeout = 900,
+ )
+ assert out.returncode == 0, out.stdout[-3000:] + out.stderr[-3000:]
+ assert "TINY_OK" in out.stdout
diff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.py
index 2ef7fec7dfa..53ec1d02849 100644
--- a/unsloth/models/__init__.py
+++ b/unsloth/models/__init__.py
@@ -18,6 +18,8 @@ from .qwen3_moe import FastQwen3MoeModel
from .granite import FastGraniteModel
from .sentence_transformer import FastSentenceTransformer
from .decision import *
+from .diffusion import *
+from .diffusion_trainer import *
try:
from .falcon_h1 import FastFalconH1Model
diff --git a/unsloth/models/diffusion.py b/unsloth/models/diffusion.py
index 259e0e02c8b..e7e51ccd817 100644
--- a/unsloth/models/diffusion.py
+++ b/unsloth/models/diffusion.py
@@ -18,11 +18,18 @@ bit-identical to transformers), keeping only the safe conveniences: 4bit/8bit lo
"""
import os
+import re
import torch
from transformers import AutoConfig, AutoProcessor, AutoTokenizer
from unsloth_zoo.hf_utils import add_dtype_kwargs
from ._utils import is_bfloat16_supported, maybe_prefetch_hf_snapshot
+from .diffusion_profiles import (
+ diffusion_model_types,
+ resolve_diffusion_profile,
+ restore_rotary_buffers,
+ unwrap_diffusion_model,
+)
from .llama import logger
from .loader_utils import (
planner_config_overrides,
@@ -58,7 +65,33 @@ def is_diffusion_model_type(model_types):
"""model_types: str or iterable -> True if any is a known diffusion model_type."""
if isinstance(model_types, str):
model_types = (model_types,)
- return any(mt in DIFFUSION_MODEL_TYPES for mt in model_types)
+ known = DIFFUSION_MODEL_TYPES + diffusion_model_types()
+ return any(mt in known for mt in model_types)
+
+
+def _refuse_tied_lora_merge(model):
+ """DiffusionGemma's encoder and decoder layers share one base tensor but get separate adapters, so a merge
+ adds both deltas to both. Refuse it instead of writing a silently wrong checkpoint."""
+ owners = {}
+ for name, module in model.named_modules():
+ weight = getattr(getattr(module, "base_layer", None), "weight", None)
+ if weight is None or not hasattr(module, "lora_A") or weight.is_meta:
+ continue
+ if owners.setdefault(weight.data_ptr(), name) != name:
+ break
+ else:
+ return model
+
+ def refuse(*args, **kwargs):
+ raise RuntimeError(
+ "Unsloth: this diffusion model's encoder and decoder share base weights but train separate LoRA "
+ "adapters, so merging would add both deltas to both. Save the adapter with "
+ "model.save_pretrained(...) and load it on top of the base model unmerged."
+ )
+
+ for attr in ("merge_and_unload", "merge_adapter"):
+ setattr(model, attr, refuse)
+ return model
def _resolve_diffusion_model_class(config):
@@ -153,7 +186,7 @@ class FastDiffusionModel:
# Planner hints for device_map = "unsloth"; see resolve_unsloth_device_map.
device_map_planner_kwargs = None,
trust_remote_code = False,
- attn_implementation = "eager", # exact match with the reference golden logits
+ attn_implementation = None, # family default: eager for DiffusionGemma (reference golden logits)
revision = None,
return_tokenizer = True,
**kwargs,
@@ -176,22 +209,63 @@ class FastDiffusionModel:
cache_dir = kwargs.get("cache_dir")
- config = _load_diffusion_config(
- model_name,
- token,
- trust_remote_code,
- revision,
- local_files_only,
- cache_dir = cache_dir,
- )
+ adapter_name = None
+ adapter_revision = None
+ try:
+ from peft import PeftConfig
+ adapter_config = PeftConfig.from_pretrained(
+ model_name,
+ token = token,
+ revision = revision,
+ local_files_only = local_files_only,
+ )
+ adapter_name, adapter_revision, model_name, revision = (
+ model_name,
+ revision,
+ adapter_config.base_model_name_or_path,
+ adapter_config.revision,
+ )
+ except Exception:
+ pass
+
+ config = kwargs.pop("config", None)
+ if config is None:
+ config = _load_diffusion_config(
+ model_name,
+ token,
+ trust_remote_code,
+ revision,
+ local_files_only,
+ cache_dir = cache_dir,
+ )
model_type = getattr(config, "model_type", None)
- if not is_diffusion_model_type(model_type):
+ profile = resolve_diffusion_profile(config)
+ if profile is None and not is_diffusion_model_type(model_type):
raise RuntimeError(
- f"Unsloth: FastDiffusionModel only supports diffusion model_types {DIFFUSION_MODEL_TYPES}, "
+ f"Unsloth: FastDiffusionModel only supports diffusion model_types "
+ f"{tuple(dict.fromkeys(DIFFUSION_MODEL_TYPES + diffusion_model_types()))}, "
f"got '{model_type}'. Use FastModel/FastLanguageModel for autoregressive models."
)
-
- model_cls = _resolve_diffusion_model_class(config)
+ if profile is not None and profile.requires_remote_code and not trust_remote_code:
+ raise RuntimeError(
+ f"Unsloth: {model_name} ships its own modeling code. Pass trust_remote_code = True."
+ )
+ if attn_implementation is None:
+ attn_implementation = (
+ profile.defaults.get("attn_implementation") if profile else None
+ ) or "eager"
+
+ if profile is not None and model_type not in DIFFUSION_MODEL_TYPES:
+ model_cls = profile.model_class(
+ config,
+ trust_remote_code,
+ token = token,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ code_revision = kwargs.get("code_revision"),
+ )
+ else:
+ model_cls = _resolve_diffusion_model_class(config)
# Prefetch the whole repo root so the weight load is a cache hit. No subfolder: the pipeline loads
# every component subfolder, so narrowing would leave unet/vae/text_encoder to Xet.
@@ -212,8 +286,9 @@ class FastDiffusionModel:
# them, and lm_head/embeddings stay full precision. Before the plan, since the skip list becomes
# modules_to_not_convert and sizing those at 4 bits while they load in compute dtype OOMs a tight
# map.
- qcfg = None
- if load_in_4bit or load_in_8bit:
+ # A caller's own quantization_config (NF4 vs FP4, skip modules, non-bnb quantizers) wins.
+ qcfg = kwargs.get("quantization_config")
+ if qcfg is None and (load_in_4bit or load_in_8bit):
from transformers import BitsAndBytesConfig
if load_in_4bit:
qcfg = BitsAndBytesConfig(
@@ -221,13 +296,11 @@ class FastDiffusionModel:
bnb_4bit_use_double_quant = True,
bnb_4bit_quant_type = "nf4",
bnb_4bit_compute_dtype = dtype,
- llm_int8_skip_modules = [
- "lm_head",
- "embed_tokens",
- "experts",
- "self_conditioning",
- "router",
- ],
+ llm_int8_skip_modules = (
+ profile.quant_skip_modules
+ if profile is not None and profile.quant_skip_modules is not None
+ else ["lm_head", "embed_tokens", "experts", "self_conditioning", "router"]
+ ),
)
else:
qcfg = BitsAndBytesConfig(load_in_8bit = True)
@@ -292,39 +365,98 @@ class FastDiffusionModel:
load_kwargs["quantization_config"] = qcfg
print(f"==(( Unsloth: FastDiffusionModel (slow / transformers-only path) ))==")
- print(f" Model: {model_name} | class: {model_cls.__name__} | model_type: {model_type}")
+ print(
+ f" Model: {model_name} | class: {model_cls.__name__} | model_type: {model_type}"
+ f" | recipe: {profile.name if profile else 'none'}"
+ )
print(
f" dtype: {dtype} | 4bit: {load_in_4bit} | 8bit: {load_in_8bit} | attn: {attn_implementation}"
)
- model = model_cls.from_pretrained(model_name, **load_kwargs).eval()
+ model = model_cls.from_pretrained(model_name, config = config, **load_kwargs).eval()
+ model._unsloth_full_finetuning = bool(full_finetuning)
+ if trust_remote_code:
+ # 4.x remote code builds RoPE inv_freq in __init__; transformers 5 leaves it uninitialised.
+ from ._remote_code_buffers import restore_remote_code_non_persistent_buffers
+
+ restore_remote_code_non_persistent_buffers(model)
+ # That helper misses rotary modules whose class overrides _init_weights (LLaDA, LLaDA2, SDAR, Dream).
+ restore_rotary_buffers(model)
+ # Remote classes without GenerationMixin (LLaDA-MoE) have none; TRL 1.x SFTTrainer reads it.
+ if getattr(model, "generation_config", None) is None:
+ from transformers import GenerationConfig
+ try:
+ model.generation_config = GenerationConfig.from_model_config(model.config)
+ except Exception:
+ model.generation_config = GenerationConfig()
# Mark before any early return so get_peft_model/for_* route to the slow path.
model._unsloth_slow_diffusion = True
+ if profile is not None:
+ model._unsloth_diffusion_profile = profile.name
+
+ if adapter_name is not None:
+ from peft import PeftModel
+
+ model = PeftModel.from_pretrained(
+ model,
+ adapter_name,
+ is_trainable = True,
+ token = token,
+ revision = adapter_revision,
+ local_files_only = local_files_only,
+ )
+ model = _refuse_tied_lora_merge(model)
+ model._unsloth_slow_diffusion = True
if not return_tokenizer:
+ if profile is not None:
+ profile.prepare_model(unwrap_diffusion_model(model), None)
return model, None
# Prefer the processor (chat template plus tokenizer), falling back to a bare tokenizer, returned as
# "tokenizer" to match the Unsloth (model, tokenizer) contract.
+ tokenizer_source, tokenizer_revision = model_name, revision
+ if adapter_name is not None:
+ from transformers.utils import cached_file
+
+ # A local directory or a Hub repo; the adapter's own tokenizer carries its chat template / tokens.
+ try:
+ saved = cached_file(
+ adapter_name,
+ "tokenizer_config.json",
+ token = token,
+ revision = adapter_revision,
+ local_files_only = local_files_only,
+ cache_dir = cache_dir,
+ _raise_exceptions_for_missing_entries = False,
+ _raise_exceptions_for_connection_errors = False,
+ )
+ except Exception:
+ saved = None
+ if saved is not None:
+ tokenizer_source, tokenizer_revision = adapter_name, adapter_revision
try:
tokenizer = AutoProcessor.from_pretrained(
- model_name,
+ tokenizer_source,
token = token,
trust_remote_code = trust_remote_code,
- revision = revision,
+ revision = tokenizer_revision,
local_files_only = local_files_only,
cache_dir = cache_dir,
)
except Exception:
tokenizer = AutoTokenizer.from_pretrained(
- model_name,
+ tokenizer_source,
token = token,
trust_remote_code = trust_remote_code,
- revision = revision,
+ revision = tokenizer_revision,
local_files_only = local_files_only,
cache_dir = cache_dir,
)
+ if profile is not None:
+ # On the base module: the objectives read these attributes after unwrapping PEFT / DDP.
+ profile.prepare_model(unwrap_diffusion_model(model), tokenizer)
return model, tokenizer
@staticmethod
@@ -341,10 +473,16 @@ class FastDiffusionModel:
**kwargs,
):
"""Attach a PEFT LoRA to the diffusion backbone (attention + dense MLP). No fused kernels."""
+ if getattr(model, "_unsloth_full_finetuning", False):
+ print("Unsloth: Full finetuning is enabled, so .get_peft_model has no effect")
+ return model
from peft import LoraConfig, get_peft_model as peft_get_peft_model
+ profile = resolve_diffusion_profile(getattr(model, "config", None))
if target_modules is None:
- target_modules = DIFFUSION_LORA_TARGETS
+ target_modules = (
+ profile.lora_target_modules if profile is not None else DIFFUSION_LORA_TARGETS
+ )
# use_dora, and any other LoraConfig kwarg outside this allowlist, is silently dropped: Unsloth does
# not reach this path today, so it is untested on diffusion models.
@@ -358,9 +496,18 @@ class FastDiffusionModel:
**{k: v for k, v in kwargs.items() if k in ("modules_to_save", "init_lora_weights")},
)
# Exclude the vision tower's custom (non-Linear) modules that share suffix names.
- exclude = kwargs.get("exclude_modules", DIFFUSION_LORA_EXCLUDE)
+ default_exclude = None
+ if profile is not None and profile.lora_exclude_modules is not None:
+ default_exclude = profile.lora_exclude_modules
+ elif any(re.match(DIFFUSION_LORA_EXCLUDE, name) for name, _ in model.named_modules()):
+ default_exclude = DIFFUSION_LORA_EXCLUDE
+ exclude = kwargs.get("exclude_modules", default_exclude)
try:
- lora_config = LoraConfig(exclude_modules = exclude, **lora_kwargs)
+ lora_config = (
+ LoraConfig(exclude_modules = exclude, **lora_kwargs)
+ if exclude is not None
+ else LoraConfig(**lora_kwargs)
+ )
except TypeError:
# Older PEFT without exclude_modules: scope the target to the text decoder by regex.
lora_kwargs["target_modules"] = (
@@ -373,6 +520,7 @@ class FastDiffusionModel:
model.enable_input_require_grads()
model = peft_get_peft_model(model, lora_config)
+ model = _refuse_tied_lora_merge(model)
model._unsloth_slow_diffusion = True
model._unsloth_gradient_checkpointing = use_gradient_checkpointing
try:
diff --git a/unsloth/models/diffusion_block.py b/unsloth/models/diffusion_block.py
new file mode 100644
index 00000000000..29b5df60360
--- /dev/null
+++ b/unsloth/models/diffusion_block.py
@@ -0,0 +1,489 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Block-diffusion families (LLaDA2.x, SDAR).
+
+Training follows BD3-LM (https://arxiv.org/abs/2503.09573): each row is fed as [x_t ; x_0] with position ids
+restarting at the clean copy. A noisy token sees its own noisy block and the clean blocks before it; clean tokens
+see clean blocks causally. The loss reads the noisy half at the same position (no shift).
+"""
+
+import contextlib
+import functools
+import importlib.machinery
+import inspect
+import sys
+import types
+
+import torch
+import torch.nn.functional as F
+
+from .diffusion_profiles import (
+ DiffusionProfile,
+ register_diffusion_profile,
+ response_mask,
+ unwrap_diffusion_model,
+)
+
+__all__ = [
+ "BlockDiffusionProfile",
+ "block_diffusion_attention_mask",
+ "LLADA2_PROFILE",
+ "SDAR_PROFILE",
+]
+
+
+def block_diffusion_attention_mask(
+ length,
+ block_size,
+ device = None,
+):
+ """Bool [2L, 2L], True = attend. Rows/cols 0..L-1 are x_t, L..2L-1 are x_0."""
+ idx = torch.arange(2 * length, device = device)
+ q, kv = idx[:, None], idx[None, :]
+ q_clean, kv_clean = q >= length, kv >= length
+ q_block = torch.where(q_clean, q - length, q) // block_size
+ kv_block = torch.where(kv_clean, kv - length, kv) // block_size
+ same_noisy_block = (q_block == kv_block) & (q_clean == kv_clean)
+ noisy_to_past_clean = (q_block > kv_block) & kv_clean & ~q_clean
+ clean_causal = (q_block >= kv_block) & kv_clean & q_clean
+ return same_noisy_block | noisy_to_past_clean | clean_causal
+
+
+_unwrap = unwrap_diffusion_model
+
+
+def _is_distributed_wrapper(model):
+ return type(model).__name__ in (
+ "DistributedDataParallel",
+ "FullyShardedDataParallel",
+ "DeepSpeedEngine",
+ )
+
+
+class BlockDiffusionProfile(DiffusionProfile):
+ # defaults keys: diffusion_block_size, noise_low, noise_high (mask probability range), time_weighting
+ # ("none" | "inverse_t"), normalize ("masked" | "supervised"), eos_fill, key_padding, ensure_masked,
+ # mask_token_id, mask_token_names, mask_format ("additive" | "bool").
+
+ def prepare_model(self, model, tokenizer):
+ mask_id = self._mask_token_from_tokenizer(tokenizer)
+ if mask_id is not None:
+ model._unsloth_diffusion_mask_token_id = mask_id
+ eos = None
+ if tokenizer is not None:
+ tok = getattr(tokenizer, "tokenizer", tokenizer)
+ eos = tok.eos_token_id if tok.eos_token_id is not None else tok.pad_token_id
+ if eos is not None:
+ model._unsloth_diffusion_eos_token_id = eos
+ return model
+
+ def _mask_token_from_tokenizer(self, tokenizer):
+ if tokenizer is None:
+ return None
+ tok = getattr(tokenizer, "tokenizer", tokenizer)
+ if getattr(tok, "mask_token_id", None) is not None:
+ return tok.mask_token_id
+ unk = getattr(tok, "unk_token_id", None)
+ for name in self.defaults.get("mask_token_names", ()):
+ token_id = tok.convert_tokens_to_ids(name)
+ if isinstance(token_id, int) and token_id != unk:
+ return token_id
+ return None
+
+ def mask_token_id(self, model, args):
+ base = _unwrap(model)
+ for value in (
+ getattr(args, "diffusion_mask_token_id", None),
+ getattr(base, "_unsloth_diffusion_mask_token_id", None),
+ getattr(getattr(base, "config", None), "mask_token_id", None),
+ self.defaults.get("mask_token_id"),
+ ):
+ if value is not None:
+ return int(value)
+ raise ValueError(
+ f"Unsloth: no [MASK] token id for {self.name}; set DiffusionConfig.diffusion_mask_token_id."
+ )
+
+ def eos_token_id(self, model):
+ base = _unwrap(model)
+ config = getattr(base, "config", None)
+ for value in (
+ getattr(base, "_unsloth_diffusion_eos_token_id", None),
+ getattr(config, "eos_token_id", None),
+ getattr(config, "pad_token_id", None),
+ ):
+ if isinstance(value, (list, tuple)):
+ value = value[0] if value else None
+ if value is not None:
+ return int(value)
+ raise ValueError(f"Unsloth: no EOS token id for {self.name}.")
+
+ def block_size(self, model, args):
+ value = getattr(args, "diffusion_block_size", None)
+ if value is None:
+ value = getattr(getattr(_unwrap(model), "config", None), "block_size", None)
+ if value is None:
+ value = self.defaults["diffusion_block_size"]
+ return int(value)
+
+ def build_batch(self, model, inputs, args):
+ """-> clean ids [B, L], maskable [B, L], valid [B, L] (attended positions)."""
+ input_ids = inputs["input_ids"]
+ attention_mask = inputs.get("attention_mask")
+ valid = (
+ torch.ones_like(input_ids, dtype = torch.bool)
+ if attention_mask is None
+ else attention_mask.bool()
+ )
+ maskable = response_mask(inputs)
+ if not self.defaults.get("eos_fill", False):
+ return input_ids, maskable, valid
+ if (valid[:, 1:] & ~valid[:, :-1]).any():
+ raise ValueError("Unsloth: block-diffusion training expects right padding.")
+ # Turns end with a terminator, not EOS, so stopping is learnt only from a supervised EOS tail (as the
+ # reference's max-length EOS padding); pad to the next block past the longest row.
+ block = self.block_size(model, args)
+ batch, length = input_ids.shape
+ new_length = -(-(length + 1) // block) * block
+ pad = new_length - length
+ eos = self.eos_token_id(model)
+ input_ids = F.pad(input_ids, (0, pad), value = eos)
+ maskable = F.pad(maskable, (0, pad), value = False)
+ valid = F.pad(valid, (0, pad), value = False)
+ real_length = valid.sum(dim = 1, keepdim = True)
+ positions = torch.arange(new_length, device = input_ids.device)
+ # A row clipped at max_length has no real end, so it gets no EOS tail (as for DiffusionGemma).
+ max_length = getattr(args, "max_length", None)
+ truncated = torch.zeros_like(real_length, dtype = torch.bool)
+ if max_length is not None:
+ last = torch.where(maskable, positions[None, :], -1).amax(dim = 1, keepdim = True)
+ truncated = (real_length >= max_length) & (last == real_length - 1)
+ fill = (positions[None, :] >= real_length) & maskable.any(dim = 1, keepdim = True)
+ input_ids = torch.where(fill, eos, input_ids)
+ # Clipped rows still see an EOS tail rather than pad ids, but it is never a target.
+ return input_ids, maskable | (fill & ~truncated), valid | fill
+
+ def noise_range(self, args):
+ eps = getattr(args, "diffusion_eps", None)
+ if eps is not None:
+ return float(eps), 1.0 - float(eps)
+ return float(self.defaults["noise_low"]), float(self.defaults["noise_high"])
+
+ def sample_noise(self, clean, maskable, mask_token_id, args):
+ """-> noisy ids, masked bool [B, L], p [B] mask probability per row."""
+ batch, length = clean.shape
+ low, high = self.noise_range(args)
+ p = low + (high - low) * torch.rand(batch, device = clean.device)
+ masked = (torch.rand(batch, length, device = clean.device) < p[:, None]) & maskable
+ if self.defaults.get("ensure_masked", False):
+ empty = maskable.any(dim = 1) & ~masked.any(dim = 1)
+ if empty.any():
+ scores = torch.rand(batch, length, device = clean.device).masked_fill(~maskable, -1.0)
+ pick = F.one_hot(scores.argmax(dim = 1), length).bool()
+ masked = masked | (pick & empty[:, None])
+ return torch.where(masked, mask_token_id, clean), masked, p
+
+ def attention_mask(self, length, block_size, valid, dtype, device):
+ mask = block_diffusion_attention_mask(length, block_size, device)[None, None]
+ batch = valid.shape[0]
+ if self.defaults.get("key_padding", False) and not bool(valid.all()):
+ keys = torch.cat([valid, valid], dim = 1)[:, None, None, :]
+ # Padding is its own segment in the reference; a padding query keeps its diagonal so no row is empty.
+ mask = (mask & keys) | torch.eye(2 * length, dtype = torch.bool, device = device)[
+ None, None
+ ]
+ mask = mask.expand(batch, 1, 2 * length, 2 * length)
+ if self.defaults.get("mask_format", "additive") == "bool":
+ return mask
+ additive = torch.zeros(mask.shape, dtype = dtype, device = device)
+ return additive.masked_fill(~mask, float("-inf"))
+
+ def noisy_hidden_and_head(self, model, noisy, clean, valid, block_size):
+ """Hidden states of the noisy half [B, L, H] and the output head."""
+ batch, length = clean.shape
+ base = _unwrap(model)
+ head = base.get_output_embeddings()
+ concat = torch.cat([noisy, clean], dim = 1)
+ positions = torch.arange(length, device = clean.device)
+ position_ids = torch.cat([positions, positions])[None].expand(batch, -1)
+ dtype = head.weight.dtype if head.weight.dtype.is_floating_point else torch.float32
+ mask = self.attention_mask(length, block_size, valid, dtype, clean.device)
+ if _is_distributed_wrapper(model):
+ # DDP / FSDP / DeepSpeed must see their own forward; this path materialises logits for both halves.
+ out = model(
+ input_ids = concat, attention_mask = mask, position_ids = position_ids, use_cache = False
+ )
+ return out.logits[:, :length], None
+ out = base.get_decoder()(
+ input_ids = concat,
+ attention_mask = mask,
+ position_ids = position_ids,
+ use_cache = False,
+ return_dict = True,
+ )
+ return out[0][:, :length], head
+
+ def loss_from_noise(self, model, clean, noisy, masked, maskable, p, valid, args):
+ block_size = self.block_size(model, args)
+ hidden, head = self.noisy_hidden_and_head(model, noisy, clean, valid, block_size)
+ if head is None:
+ logits = hidden[masked].float()
+ else:
+ # Project only the masked positions: the vocab-sized logits are the memory peak otherwise.
+ logits = head(hidden[masked]).float()
+ nll = F.cross_entropy(logits, clean[masked], reduction = "none")
+ weighting = self.option(args, "diffusion_time_weighting", "none")
+ if weighting not in ("inverse_t", "none"):
+ raise ValueError(
+ f"Unsloth: {self.name} does not support diffusion_time_weighting={weighting!r}."
+ )
+ if weighting == "inverse_t":
+ nll = nll / p[:, None].expand_as(masked)[masked].float()
+ if self.defaults.get("normalize", "masked") == "supervised":
+ denominator = maskable.sum()
+ else:
+ denominator = masked.sum()
+ loss = nll.sum() / denominator.clamp_min(1)
+ if not masked.any():
+ loss = loss + hidden.sum() * 0.0
+ return loss
+
+ def compute_loss(
+ self,
+ model,
+ inputs,
+ args,
+ num_items_in_batch = None,
+ ):
+ clean, maskable, valid = self.build_batch(model, inputs, args)
+ mask_token_id = self.mask_token_id(model, args)
+ noisy, masked, p = self.sample_noise(clean, maskable, mask_token_id, args)
+ loss = self.loss_from_noise(model, clean, noisy, masked, maskable, p, valid, args)
+ metrics = {
+ "mask_ratio": (masked.sum() / maskable.sum().clamp_min(1)).item(),
+ "masked_tokens": masked.sum().item(),
+ }
+ return loss, None, metrics
+
+
+class _LLaDA2Profile(BlockDiffusionProfile):
+ pass
+
+
+# Reference: inclusionAI/dFactory tasks/train_llada2_bd.py and configs/sft/llada2_mini_bd_sft.yaml.
+LLADA2_PROFILE = register_diffusion_profile(
+ _LLaDA2Profile(
+ name = "llada2",
+ model_types = ("llada2_moe",),
+ architectures = ("LLaDA2MoeModelLM",),
+ noise = "mask",
+ requires_remote_code = True,
+ # Attention, the dense first-layer MLP and the shared expert; routed experts and the router stay frozen.
+ lora_target_modules = (
+ r".*\.layers\.\d+\.(attention\.(query_key_value|dense)"
+ r"|mlp\.(shared_experts\.)?(gate_proj|up_proj|down_proj))"
+ ),
+ defaults = dict(
+ diffusion_block_size = 32,
+ noise_low = 0.3,
+ noise_high = 0.8,
+ diffusion_time_weighting = "none",
+ normalize = "masked",
+ eos_fill = True,
+ key_padding = False,
+ ensure_masked = False,
+ mask_token_id = 156895,
+ mask_token_names = ("<|mask|>",),
+ mask_format = "additive",
+ attn_implementation = "sdpa",
+ ),
+ )
+)
+
+
+def _rms_norm_fn(
+ x,
+ weight,
+ bias = None,
+ eps = 1e-6,
+ **kwargs,
+):
+ dtype = x.dtype
+ x = x.float()
+ x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim = True) + eps)
+ out = weight * x.to(dtype) if weight is not None else x.to(dtype)
+ return out + bias if bias is not None else out
+
+
+def _flash_attn_func(
+ q,
+ k,
+ v,
+ dropout_p = 0.0,
+ softmax_scale = None,
+ causal = False,
+ **kwargs,
+):
+ # [B, S, H, D] layout like flash-attn; SDPA handles GQA via enable_gqa.
+ out = F.scaled_dot_product_attention(
+ q.transpose(1, 2),
+ k.transpose(1, 2),
+ v.transpose(1, 2),
+ is_causal = causal,
+ scale = softmax_scale,
+ enable_gqa = k.shape[2] != q.shape[2],
+ )
+ return out.transpose(1, 2)
+
+
+def _module(name, **attributes):
+ module = types.ModuleType(name)
+ module.__spec__ = importlib.machinery.ModuleSpec(name, None)
+ module.__path__ = []
+ module.__dict__.update(attributes)
+ return module
+
+
+def _sdar_remote_code_imports():
+ """SDAR's modeling file imports flash-attn's RMSNorm and 4.x `LossKwargs` at module level. Provide
+ torch stand-ins only while it imports when flash-attn is absent; the module keeps its references."""
+ import importlib.util
+ import transformers.utils as transformers_utils
+
+ if not hasattr(transformers_utils, "LossKwargs"):
+ from typing import Optional, TypedDict
+ class LossKwargs(TypedDict, total = False):
+ num_items_in_batch: Optional[torch.Tensor]
+
+ transformers_utils.LossKwargs = LossKwargs
+ try:
+ has_flash = importlib.util.find_spec("flash_attn") is not None
+ except (ImportError, ValueError):
+ has_flash = False
+ if has_flash:
+ yield
+ return
+ names = (
+ "flash_attn",
+ "flash_attn.ops",
+ "flash_attn.ops.triton",
+ "flash_attn.ops.triton.layer_norm",
+ )
+ stubs = {
+ "flash_attn": _module("flash_attn", flash_attn_func = _flash_attn_func),
+ "flash_attn.ops": _module("flash_attn.ops"),
+ "flash_attn.ops.triton": _module("flash_attn.ops.triton"),
+ "flash_attn.ops.triton.layer_norm": _module(
+ "flash_attn.ops.triton.layer_norm", rms_norm_fn = _rms_norm_fn
+ ),
+ }
+ from transformers import dynamic_module_utils
+
+ original_check_imports = dynamic_module_utils.check_imports
+
+ def check_imports(filename):
+ try:
+ return original_check_imports(filename)
+ except ImportError as error:
+ if "flash_attn" not in str(error):
+ raise
+ return dynamic_module_utils.get_relative_imports(filename)
+
+ saved = {name: sys.modules.get(name) for name in names}
+ sys.modules.update(stubs)
+ dynamic_module_utils.check_imports = check_imports
+ try:
+ yield
+ finally:
+ dynamic_module_utils.check_imports = original_check_imports
+ for name, module in saved.items():
+ if module is None:
+ sys.modules.pop(name, None)
+ else:
+ sys.modules[name] = module
+
+
+def _accept_cache_objects(model):
+ """SDAR's attention reads `past_key_value[layer]` (4.x tuple API); transformers 5 caches have no
+ __getitem__. Hand it a read view while `update` / `len` still reach the real cache."""
+ from .remote_code_shims import _LegacyCacheView, _needs_legacy_view
+
+ wrapped = 0
+ for module in model.modules():
+ cls = type(module)
+ forward = cls.__dict__.get("forward")
+ if forward is None or getattr(forward, "_unsloth_cache_view", False):
+ continue
+ try:
+ source = inspect.getsource(forward)
+ except (OSError, TypeError):
+ continue
+ if (
+ "past_key_value[" not in source
+ or "past_key_value" not in inspect.signature(forward).parameters
+ ):
+ continue
+
+ def make(original):
+ @functools.wraps(original)
+ def forward(self, *args, **kwargs):
+ cache = kwargs.get("past_key_value")
+ if _needs_legacy_view(cache):
+ kwargs["past_key_value"] = _LegacyCacheView(cache)
+ return original(self, *args, **kwargs)
+
+ forward._unsloth_cache_view = True
+ return forward
+
+ cls.forward = make(forward)
+ wrapped += 1
+ return wrapped
+
+
+class _SDARProfile(BlockDiffusionProfile):
+ def prepare_model(self, model, tokenizer):
+ _accept_cache_objects(model)
+ # The released config fuses the head into its training loss, so a train-mode forward returns no logits.
+ config = getattr(model, "config", None)
+ if getattr(config, "fuse_cross_entropy", False):
+ config.fuse_cross_entropy = False
+ return super().prepare_model(model, tokenizer)
+
+ def model_class(self, config, trust_remote_code, **hub_kwargs):
+ from transformers import AutoModelForCausalLM
+ class _Loader:
+ @staticmethod
+ def from_pretrained(*args, **kwargs):
+ with _sdar_remote_code_imports():
+ return AutoModelForCausalLM.from_pretrained(*args, **kwargs)
+
+ return _Loader
+
+
+# Reference: JetAstra/SDAR training/model/SDAR-8B-Chat/modeling_sdar.py (forward_add_noise_packed,
+# block_attn_mask, FusedLinearDiffusionCrossEntropyLoss); block length 4 from the model card.
+SDAR_PROFILE = register_diffusion_profile(
+ _SDARProfile(
+ name = "sdar",
+ model_types = ("sdar",),
+ architectures = ("SDARForCausalLM",),
+ noise = "mask",
+ requires_remote_code = True,
+ defaults = dict(
+ diffusion_block_size = 4,
+ noise_low = 1e-3,
+ noise_high = 1.0,
+ diffusion_time_weighting = "inverse_t",
+ normalize = "supervised",
+ eos_fill = False,
+ key_padding = True,
+ ensure_masked = True,
+ mask_token_id = 151669,
+ mask_token_names = ("<MASK>", "<|MASK|>"),
+ mask_format = "bool",
+ attn_implementation = "sdpa",
+ ),
+ )
+)
diff --git a/unsloth/models/diffusion_gemma_objective.py b/unsloth/models/diffusion_gemma_objective.py
new file mode 100644
index 00000000000..bff1c8fb1a5
--- /dev/null
+++ b/unsloth/models/diffusion_gemma_objective.py
@@ -0,0 +1,213 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""DiffusionGemma block-diffusion SFT objective, matching the reference fine-tune of the released checkpoint
+(the recipe in TRL's examples/sft_diffusion_gemma/sft_diffusion_gemma.py)."""
+
+import contextlib
+
+import torch
+import torch.nn.functional as F
+
+from .diffusion_profiles import DiffusionProfile, register_diffusion_profile, unwrap_diffusion_model
+
+__all__ = ["DiffusionGemmaProfile"]
+
+try:
+ from trl.trainer.utils import maybe_gather_lm_head_ctx as _gather_lm_head
+except ImportError: # trl < 1.x: no ZeRO-3 aware helper, plain access is fine off ZeRO-3
+
+ def _gather_lm_head(*params):
+ return contextlib.nullcontext()
+
+
+def _find_attr(model, name):
+ seen = set()
+ stack = [model]
+ while stack:
+ m = stack.pop()
+ if m is None or id(m) in seen:
+ continue
+ seen.add(id(m))
+ value = m.__dict__.get(name) if hasattr(m, "__dict__") else None
+ if value is not None:
+ return value
+ stack += [
+ getattr(m, "base_model", None),
+ getattr(m, "model", None),
+ getattr(m, "module", None),
+ ]
+ return None
+
+
+def _eos_token_id(model):
+ eos = _find_attr(model, "_unsloth_diffusion_eos_token_id")
+ if eos is None:
+ config = model.config
+ eos = getattr(getattr(config, "text_config", config), "eos_token_id", None)
+ if eos is None:
+ eos = getattr(getattr(model, "generation_config", None), "eos_token_id", None)
+ if isinstance(eos, (list, tuple)):
+ eos = eos[0]
+ if eos is None:
+ raise ValueError(
+ "Unsloth: DiffusionGemma training needs an eos_token_id to fill the canvas."
+ )
+ return int(eos)
+
+
+class DiffusionGemmaProfile(DiffusionProfile):
+ def prepare_model(self, model, tokenizer):
+ tok = getattr(tokenizer, "tokenizer", tokenizer)
+ eos = getattr(tok, "eos_token_id", None)
+ if eos is not None:
+ model._unsloth_diffusion_eos_token_id = int(eos)
+ return model
+
+ def compute_loss(
+ self,
+ model,
+ inputs,
+ args,
+ num_items_in_batch = None,
+ ):
+ eps = self.option(args, "diffusion_eps")
+ sc_p = self.option(args, "diffusion_self_conditioning_p")
+ ar_weight = self.option(args, "diffusion_encoder_ar_weight")
+ prediction_type = self.option(args, "diffusion_prediction_type")
+ if prediction_type not in ("mean", "mean_loo"):
+ raise ValueError(
+ f"Unsloth: diffusion_prediction_type must be 'mean' or 'mean_loo', got {prediction_type!r}."
+ )
+
+ base = unwrap_diffusion_model(model)
+ config = base.config
+ text_config = config.text_config
+ block_size = config.canvas_length
+ vocab_size = text_config.vocab_size
+ softcap = text_config.final_logit_softcapping
+ eos_token_id = _eos_token_id(base)
+ max_length = getattr(args, "max_length", None)
+
+ input_ids = inputs["input_ids"]
+ attention_mask = inputs.get("attention_mask")
+ if attention_mask is None:
+ attention_mask = torch.ones_like(input_ids)
+ labels = inputs.get("labels")
+ if labels is None:
+ labels = input_ids.masked_fill(~attention_mask.bool(), -100)
+ device = input_ids.device
+ batch_size, seq_len = input_ids.shape
+
+ # The canvas covers the last supervised span only: the denoising loss needs one contiguous response.
+ supervised = labels != -100
+ positions = torch.arange(seq_len, device = device)
+ span_starts = supervised & ~F.pad(supervised, (1, 0))[:, :-1]
+ span_end = torch.where(supervised, positions, -1).amax(dim = 1)
+ prefix_len = torch.where(span_starts, positions, -1).amax(dim = 1).clamp(min = 0)
+ response_len = (span_end - prefix_len + 1) * (span_end >= 0)
+
+ # A clipped response has no real end, so its EOS fill must not teach the model to stop there.
+ if max_length is None:
+ truncated = torch.zeros(batch_size, dtype = torch.bool, device = device)
+ else:
+ real_len = attention_mask.sum(dim = 1)
+ truncated = (real_len >= max_length) & (span_end == real_len - 1)
+
+ # One response block per example; the decoder sees the prompt plus clean blocks before it, nothing later.
+ num_blocks = (response_len - 1).clamp(min = 0) // block_size + 1
+ block_idx = (torch.rand(batch_size, device = device) * num_blocks).long()
+ encoder_len = prefix_len + block_idx * block_size
+
+ offsets = torch.arange(block_size, device = device)
+ abs_idx = (encoder_len[:, None] + offsets).clamp(max = seq_len - 1)
+ in_response = offsets < (response_len - block_idx * block_size)[:, None]
+ canvas_target = torch.where(in_response, input_ids.gather(1, abs_idx), eos_token_id)
+
+ t = eps + (1 - 2 * eps) * torch.rand(batch_size, 1, device = device)
+ corrupt = torch.rand(batch_size, block_size, device = device) < t
+ random_tokens = torch.randint(vocab_size, (batch_size, block_size), device = device)
+ canvas_ids = torch.where(corrupt, random_tokens, canvas_target)
+
+ cache_mask = positions < encoder_len[:, None]
+ canvas_mask = torch.ones(batch_size, block_size, dtype = torch.bool, device = device)
+ model_kwargs = {
+ "input_ids": input_ids,
+ "attention_mask": attention_mask,
+ "decoder_input_ids": canvas_ids,
+ "decoder_attention_mask": torch.cat([cache_mask, canvas_mask], dim = 1),
+ "decoder_position_ids": encoder_len[:, None] + offsets,
+ }
+ # Two-pass self-conditioning: a no-grad first pass supplies the logits, kept per example with prob p.
+ if sc_p > 0:
+ with torch.no_grad():
+ model_kwargs["self_conditioning_logits"] = model(**model_kwargs).logits
+ model_kwargs["self_conditioning_mask"] = torch.rand(batch_size, device = device) < sc_p
+ outputs = model(**model_kwargs)
+
+ # Flat CE over the whole canvas, corrupted and clean alike: the uniform kernel's ELBO has no 1/t weight.
+ diffusion_target = canvas_target.masked_fill(truncated[:, None] & ~in_response, -100)
+ # A row with no supervised token (completion truncated away) would otherwise train an all-EOS canvas.
+ diffusion_target = diffusion_target.masked_fill((span_end < 0)[:, None], -100)
+ diffusion_logits = outputs.logits
+ if prediction_type == "mean_loo":
+ # The softmax parameterises the leave-one-out posterior (arXiv 2605.22765): convert in fp32, ~log(K) nats.
+ alpha = (1 - t).float()
+ bump = torch.log1p(diffusion_logits.shape[-1] * alpha / (1 - alpha))
+ diffusion_logits = diffusion_logits.float().scatter_add(
+ 2, canvas_ids.unsqueeze(-1), bump[:, :, None].expand(-1, block_size, 1)
+ )
+ if (diffusion_target != -100).any():
+ diffusion_loss = F.cross_entropy(
+ diffusion_logits.flatten(0, 1), diffusion_target.flatten(), ignore_index = -100
+ )
+ else:
+ diffusion_loss = outputs.logits.sum() * 0.0
+
+ lm_head = base.get_output_embeddings()
+ hidden_states = outputs.encoder_last_hidden_state.to(lm_head.weight.dtype)
+ with _gather_lm_head(lm_head.weight, lm_head.bias):
+ encoder_logits = hidden_states @ lm_head.weight.t()
+ if lm_head.bias is not None:
+ encoder_logits = encoder_logits + lm_head.bias
+ encoder_logits = encoder_logits.float()
+ encoder_logits = torch.tanh(encoder_logits / softcap) * softcap
+ ar_mask = attention_mask[:, :-1].bool() & attention_mask[:, 1:].bool()
+ ar_targets = torch.where(ar_mask, input_ids[:, 1:], -100)
+ ar_loss = F.cross_entropy(
+ encoder_logits[:, :-1].flatten(0, 1), ar_targets.flatten(), ignore_index = -100
+ )
+
+ loss = diffusion_loss + ar_weight * ar_loss
+ metrics = {
+ "diffusion_loss": diffusion_loss.detach().item(),
+ "encoder_ar_loss": ar_loss.detach().item(),
+ "mean_t": t.mean().item(),
+ }
+ return loss, outputs, metrics
+
+
+DIFFUSION_GEMMA_PROFILE = register_diffusion_profile(
+ DiffusionGemmaProfile(
+ name = "diffusion_gemma",
+ model_types = ("diffusion_gemma", "diffusion_gemma4"),
+ architectures = (
+ "DiffusionGemmaForBlockDiffusion",
+ "DiffusionGemma4ModelForBlockDiffusion",
+ "DiffusionGemma4ForBlockDiffusion",
+ ),
+ noise = "uniform",
+ # Encoder + decoder attention / dense MLP only; suffix targets would also wrap the self-conditioning block.
+ lora_target_modules = (
+ r".*model\.(encoder\.language_model|decoder)\.layers\.\d+\."
+ r"(self_attn\.[qkvo]_proj|mlp\.(gate|up|down)_proj)"
+ ),
+ defaults = {
+ "diffusion_eps": 1e-4,
+ "diffusion_self_conditioning_p": 0.5,
+ "diffusion_encoder_ar_weight": 1.0,
+ "diffusion_prediction_type": "mean",
+ "attn_implementation": "eager",
+ },
+ )
+)
diff --git a/unsloth/models/diffusion_masked.py b/unsloth/models/diffusion_masked.py
new file mode 100644
index 00000000000..f1928b6a2f7
--- /dev/null
+++ b/unsloth/models/diffusion_masked.py
@@ -0,0 +1,341 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Absorbing-mask diffusion LMs over one bidirectional sequence: LLaDA, LLaDA-MoE, Dream, Nemotron-Labs-Diffusion.
+
+Each response token is replaced by [MASK] with probability p, the clean prompt stays visible, and the
+loss is cross-entropy at the masked positions weighted by 1 / p. The families differ only in the knobs
+below, each taken from that family's published training code.
+"""
+
+import functools
+from dataclasses import dataclass
+
+import torch
+import torch.nn.functional as F
+
+from .diffusion_profiles import (
+ DiffusionProfile,
+ register_diffusion_profile,
+ response_mask,
+ unwrap_diffusion_model,
+)
+
+__all__ = ["MaskedDiffusionProfile"]
+
+
+def _ensure_post_init(cls):
+ """4.x remote code that never calls post_init() loads on transformers 5 only once it does."""
+ if getattr(cls, "_unsloth_post_init_guard", False):
+ return cls
+ original_init = cls.__init__
+
+ @functools.wraps(original_init)
+ def __init__(self, *args, **kwargs):
+ original_init(self, *args, **kwargs)
+ if not hasattr(self, "all_tied_weights_keys") and hasattr(self, "post_init"):
+ self.post_init()
+
+ cls.__init__ = __init__
+ cls._unsloth_post_init_guard = True
+ return cls
+
+
+def _accept_new_validate_kwargs(cls):
+ """Dream's GenerationConfig.validate(self, is_init) predates the keywords transformers 5 passes it."""
+ import inspect
+ import sys
+
+ from transformers import GenerationConfig
+
+ module = sys.modules.get(cls.__module__)
+ for value in list(vars(module).values()) if module is not None else ():
+ if (
+ not (isinstance(value, type) and issubclass(value, GenerationConfig))
+ or value is GenerationConfig
+ ):
+ continue
+ validate = value.__dict__.get("validate")
+ if validate is None or getattr(validate, "_unsloth_tolerant", False):
+ continue
+ parameters = inspect.signature(validate).parameters
+ if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters.values()):
+ continue
+ accepted = set(parameters)
+
+ @functools.wraps(validate)
+ def tolerant(
+ self,
+ *args,
+ _validate = validate,
+ _accepted = accepted,
+ **kwargs,
+ ):
+ return _validate(self, *args, **{k: v for k, v in kwargs.items() if k in _accepted})
+
+ tolerant._unsloth_tolerant = True
+ value.validate = tolerant
+
+
+def _default_diffusion_generate_config(model):
+ """Dream's diffusion_generate() without a generation_config rebuilds one through
+ GenerationConfig.from_model_config, which transformers 5 breaks on Dream-only fields (eps)."""
+ generate = getattr(model, "diffusion_generate", None)
+ if generate is None or getattr(generate, "_unsloth_default_config", False):
+ return
+
+ @functools.wraps(generate)
+ def diffusion_generate(
+ inputs = None,
+ generation_config = None,
+ **kwargs,
+ ):
+ if generation_config is None:
+ import copy
+ generation_config = copy.deepcopy(model.generation_config)
+ # from_model_config used to fill these from the model config; Dream ships mask_token_id null.
+ for name in ("mask_token_id", "eos_token_id", "pad_token_id", "bos_token_id"):
+ if getattr(generation_config, name, None) is None:
+ setattr(generation_config, name, getattr(model.config, name, None))
+ return generate(inputs, generation_config = generation_config, **kwargs)
+
+ diffusion_generate._unsloth_default_config = True
+ model.diffusion_generate = diffusion_generate
+
+
+def _expose_mask_signatures():
+ """unsloth_zoo wraps the transformers mask builders as (*args, **kwargs); remote code that filters its
+ arguments by inspect.signature (Nemotron-Labs-Diffusion's causal prefill) then passes nothing."""
+ import inspect
+
+ try:
+ import transformers.masking_utils as masking_utils
+ except ImportError:
+ return
+ for name in ("create_causal_mask", "create_sliding_window_causal_mask"):
+ function = getattr(masking_utils, name, None)
+ if (
+ function is None
+ or "__signature__" in vars(function)
+ or hasattr(function, "__wrapped__")
+ ):
+ continue
+ for cell_name, cell in zip(function.__code__.co_freevars, function.__closure__ or ()):
+ if cell_name == "signature" and isinstance(cell.cell_contents, inspect.Signature):
+ function.__signature__ = cell.cell_contents
+ break
+
+
+def _bridge_native_checkpointing(model):
+ """LLaDA checkpoints through its own set_activation_checkpointing, not the transformers API."""
+ inner = getattr(model, "model", None)
+ if getattr(model, "supports_gradient_checkpointing", False) or not hasattr(
+ inner, "set_activation_checkpointing"
+ ):
+ return
+
+ def gradient_checkpointing_enable(gradient_checkpointing_kwargs = None, **kwargs):
+ inner.set_activation_checkpointing("whole_layer")
+
+ def gradient_checkpointing_disable():
+ inner.set_activation_checkpointing(None)
+
+ model.gradient_checkpointing_enable = gradient_checkpointing_enable
+ model.gradient_checkpointing_disable = gradient_checkpointing_disable
+ if hasattr(model, "enable_input_require_grads"):
+ model.enable_input_require_grads = lambda: None
+
+
+@dataclass
+class MaskedDiffusionProfile(DiffusionProfile):
+ # Dream is adapted from an AR model: position i's logits predict token i + 1 (Dream src/trainer).
+ shift_logits: bool = False
+ # "answer_length": LLaDA GUIDELINES.md SFT, each example / its answer length, mean over the batch.
+ # "masked_tokens": Dream / Nemotron, summed then divided by the number of masked tokens.
+ loss_normalization: str = "answer_length"
+ # LLaDA and Dream train on EOS-filled, attended padding so the model learns where answers stop.
+ pad_with_eos: bool = True
+ auto_class: str = "AutoModelForCausalLM"
+ default_mask_token_id: int = None
+
+ def __post_init__(self):
+ self.noise = "mask"
+ self.defaults = {
+ "diffusion_eps": 1e-3,
+ "diffusion_time_weighting": "inverse_t",
+ **self.defaults,
+ }
+
+ def model_class(self, config, trust_remote_code, **hub_kwargs):
+ from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+ auto_map = getattr(config, "auto_map", None) or {}
+ class_ref = (
+ auto_map.get(self.auto_class)
+ or auto_map.get("AutoModelForCausalLM")
+ or auto_map.get("AutoModel")
+ )
+ if class_ref is None or not trust_remote_code:
+ return super().model_class(config, trust_remote_code, **hub_kwargs)
+ cls = get_class_from_dynamic_module(
+ class_ref,
+ config._name_or_path,
+ revision = getattr(config, "_commit_hash", None),
+ **hub_kwargs,
+ )
+ _accept_new_validate_kwargs(cls)
+ return _ensure_post_init(cls)
+
+ def prepare_model(self, model, tokenizer):
+ config = model.config
+ # transformers 5 dropped the PretrainedConfig.use_cache default that LLaDA's forward reads.
+ if getattr(config, "use_cache", None) is None:
+ config.use_cache = False
+ mask_id = getattr(config, "mask_token_id", None)
+ if mask_id is None and tokenizer is not None:
+ mask_id = getattr(getattr(tokenizer, "tokenizer", tokenizer), "mask_token_id", None)
+ if mask_id is None:
+ mask_id = self.default_mask_token_id
+ eos_id = getattr(config, "eos_token_id", None)
+ if isinstance(eos_id, (list, tuple)):
+ eos_id = eos_id[0] if eos_id else None
+ if eos_id is None and tokenizer is not None:
+ eos_id = getattr(getattr(tokenizer, "tokenizer", tokenizer), "eos_token_id", None)
+ model._unsloth_mask_token_id = mask_id
+ model._unsloth_eos_token_id = eos_id
+ _bridge_native_checkpointing(model)
+ _default_diffusion_generate_config(model)
+ _expose_mask_signatures()
+ return model
+
+ def _token_ids(self, model, args):
+ model = unwrap_diffusion_model(model)
+ mask_id = self.option(args, "diffusion_mask_token_id")
+ if mask_id is None:
+ mask_id = getattr(model, "_unsloth_mask_token_id", None)
+ if mask_id is None:
+ mask_id = getattr(model.config, "mask_token_id", None) or self.default_mask_token_id
+ if mask_id is None:
+ raise ValueError(
+ f"Unsloth: {self.name} needs a [MASK] token id. Set DiffusionConfig(diffusion_mask_token_id = ...)."
+ )
+ eos_id = getattr(model, "_unsloth_eos_token_id", None)
+ if eos_id is None:
+ eos_id = getattr(model.config, "eos_token_id", None)
+ if isinstance(eos_id, (list, tuple)):
+ eos_id = eos_id[0]
+ return mask_id, eos_id
+
+ def compute_loss(
+ self,
+ model,
+ inputs,
+ args,
+ num_items_in_batch = None,
+ ):
+ input_ids = inputs["input_ids"]
+ attention_mask = inputs.get("attention_mask")
+ batch, length = input_ids.shape
+ device = input_ids.device
+ mask_id, eos_id = self._token_ids(model, args)
+ eps = float(self.option(args, "diffusion_eps"))
+ weighting = self.option(args, "diffusion_time_weighting")
+
+ maskable = response_mask(inputs)
+ forward_mask = attention_mask
+ if self.pad_with_eos and attention_mask is not None and eos_id is not None:
+ padding = ~attention_mask.bool()
+ input_ids = input_ids.masked_fill(padding, eos_id)
+ if inputs.get("labels") is not None:
+ # Only rows with a response get an EOS tail; a fully truncated row contributes nothing.
+ maskable = maskable | (padding & maskable.any(dim = 1, keepdim = True))
+ forward_mask = None
+
+ # Same draw order as the references (t, then the per-token draw) so a seed reproduces their batch.
+ t = torch.rand(batch, device = device)
+ p_mask = ((1 - eps) * t + eps)[:, None].expand(batch, length)
+ masked = (torch.rand((batch, length), device = device) < p_mask) & maskable
+ noisy = torch.where(masked, mask_id, input_ids)
+
+ model_inputs = {"input_ids": noisy}
+ if forward_mask is not None and not bool(forward_mask.all()):
+ model_inputs["attention_mask"] = forward_mask
+ logits = model(**model_inputs).logits
+ if self.shift_logits:
+ logits = torch.cat([logits[:, :1], logits[:, :-1]], dim = 1)
+
+ if not bool(masked.any()):
+ zero = logits.sum() * 0.0
+ return zero, None, {"masked_fraction": 0.0}
+
+ token_loss = F.cross_entropy(logits[masked].float(), input_ids[masked], reduction = "none")
+ if weighting == "inverse_t":
+ token_loss = token_loss / p_mask[masked]
+ elif weighting not in ("none", None):
+ raise ValueError(
+ f"Unsloth: {self.name} does not support diffusion_time_weighting={weighting!r}."
+ )
+
+ if self.loss_normalization == "answer_length":
+ answer_length = maskable.sum(dim = 1, keepdim = True).clamp_min(1).expand(batch, length)
+ loss = (token_loss / answer_length[masked]).sum() / batch
+ else:
+ loss = token_loss.sum() / masked.sum()
+
+ metrics = {
+ "masked_fraction": (masked.sum() / maskable.sum().clamp_min(1)).item(),
+ "masked_ce": F.cross_entropy(logits[masked].float(), input_ids[masked]).item(),
+ }
+ return loss, None, metrics
+
+
+_ATTENTION_AND_MLP = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+
+# LLaDA-MoE shares model_type "llada" with the dense model, so it is matched first by architecture.
+register_diffusion_profile(
+ MaskedDiffusionProfile(
+ name = "llada_moe",
+ architectures = ("LLaDAMoEModel", "LLaDAMoEModelLM"),
+ requires_remote_code = True,
+ # Every layer is MoE: adapt attention only, never the router or the experts.
+ lora_target_modules = r".*\.layers\.\d+\.self_attn\.(q_proj|k_proj|v_proj|o_proj)",
+ default_mask_token_id = 156895, # model card generate(mask_id = 156895); absent from config.json
+ )
+)
+register_diffusion_profile(
+ MaskedDiffusionProfile(
+ name = "llada",
+ model_types = ("llada",),
+ architectures = ("LLaDAModelLM",),
+ requires_remote_code = True,
+ # OLMo-style names; anchored to blocks so the output head (transformer.ff_out) stays frozen.
+ lora_target_modules = r".*\.blocks\.\d+\.(q_proj|k_proj|v_proj|attn_out|ff_proj|up_proj|ff_out)",
+ quant_skip_modules = ["transformer.ff_out", "transformer.wte", "lm_head"],
+ default_mask_token_id = 126336,
+ )
+)
+register_diffusion_profile(
+ MaskedDiffusionProfile(
+ name = "dream",
+ model_types = ("Dream", "dream"),
+ architectures = ("DreamModel",),
+ requires_remote_code = True,
+ lora_target_modules = list(_ATTENTION_AND_MLP),
+ shift_logits = True,
+ loss_normalization = "masked_tokens",
+ auto_class = "AutoModel",
+ # Dream q_sample draws t ~ U(0, 1) with no floor and weights by 1 / t (time_reweighting "original").
+ defaults = {"diffusion_eps": 0.0},
+ )
+)
+register_diffusion_profile(
+ MaskedDiffusionProfile(
+ name = "nemotron_labs_diffusion",
+ model_types = ("nemotron_labs_diffusion",),
+ architectures = ("NemotronLabsDiffusionModel",),
+ requires_remote_code = True,
+ lora_target_modules = list(_ATTENTION_AND_MLP),
+ loss_normalization = "masked_tokens",
+ auto_class = "AutoModel",
+ )
+)
diff --git a/unsloth/models/diffusion_profiles.py b/unsloth/models/diffusion_profiles.py
new file mode 100644
index 00000000000..0667da7b513
--- /dev/null
+++ b/unsloth/models/diffusion_profiles.py
@@ -0,0 +1,209 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Per-family training recipes for text-diffusion language models.
+
+A profile names the checkpoints it handles (``model_types`` / ``architectures``), how they load, which
+LoRA targets are safe, and the training objective. ``DiffusionTrainer`` calls ``profile.compute_loss``
+on ordinary SFT batches (``input_ids``, ``attention_mask``, ``labels`` with -100 outside the response).
+"""
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+import torch
+
+__all__ = [
+ "DiffusionProfile",
+ "register_diffusion_profile",
+ "resolve_diffusion_profile",
+ "diffusion_profiles",
+ "diffusion_model_types",
+ "response_mask",
+ "sample_noise_level",
+ "restore_rotary_buffers",
+ "unwrap_diffusion_model",
+]
+
+
+@dataclass
+class DiffusionProfile:
+ name: str
+ model_types: tuple = ()
+ architectures: tuple = ()
+ # "mask" = absorbing [MASK] noise (LLaDA, Dream, Nemotron); "uniform" = random-token noise (DiffusionGemma).
+ noise: str = "mask"
+ requires_remote_code: bool = False
+ lora_target_modules: object = field(
+ default_factory = lambda: [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "gate_proj",
+ "up_proj",
+ "down_proj",
+ ]
+ )
+ lora_exclude_modules: Optional[str] = None
+ # bitsandbytes keeps these in full precision; None = the shared default list.
+ quant_skip_modules: Optional[list] = None
+ # Profile-specific defaults a DiffusionConfig field left at None falls back to.
+ defaults: dict = field(default_factory = dict)
+
+ def matches(self, config) -> bool:
+ model_type = getattr(config, "model_type", None)
+ if model_type in self.model_types:
+ return True
+ archs = getattr(config, "architectures", None) or ()
+ return any(arch in self.architectures for arch in archs)
+
+ def model_class(self, config, trust_remote_code, **hub_kwargs):
+ """The class ``from_pretrained`` is called on. Remote-code families use the auto class."""
+ from transformers import AutoModel, AutoModelForCausalLM
+
+ auto_map = getattr(config, "auto_map", None) or {}
+ if "AutoModelForCausalLM" in auto_map or not auto_map:
+ return AutoModelForCausalLM
+ return AutoModel
+
+ def prepare_model(self, model, tokenizer):
+ """Post-load fixes (remote-code compat, mask-token bookkeeping). Must be idempotent."""
+ return model
+
+ def option(
+ self,
+ args,
+ name,
+ default = None,
+ ):
+ value = getattr(args, name, None) if args is not None else None
+ if value is None:
+ value = self.defaults.get(name, default)
+ return value
+
+ def compute_loss(
+ self,
+ model,
+ inputs,
+ args,
+ num_items_in_batch = None,
+ ):
+ """Returns (loss, outputs_or_None, metrics: dict[str, float])."""
+ raise NotImplementedError(
+ f"Unsloth: diffusion profile {self.name} has no training objective."
+ )
+
+
+_PROFILES = []
+
+
+def register_diffusion_profile(profile):
+ for i, existing in enumerate(_PROFILES):
+ if existing.name == profile.name:
+ _PROFILES[i] = profile
+ return profile
+ _PROFILES.append(profile)
+ return profile
+
+
+def diffusion_profiles():
+ _load_builtin_profiles()
+ return tuple(_PROFILES)
+
+
+def diffusion_model_types():
+ return tuple(mt for p in diffusion_profiles() for mt in p.model_types)
+
+
+def resolve_diffusion_profile(config):
+ if config is None:
+ return None
+ for profile in diffusion_profiles():
+ if profile.matches(config):
+ return profile
+ return None
+
+
+_BUILTINS_LOADED = False
+
+
+def _load_builtin_profiles():
+ global _BUILTINS_LOADED
+ if _BUILTINS_LOADED:
+ return
+ _BUILTINS_LOADED = True
+ # Each module registers its profiles on import.
+ from . import diffusion_block, diffusion_gemma_objective, diffusion_masked # noqa: F401
+
+
+def unwrap_diffusion_model(model):
+ """The module holding config / heads / Unsloth attributes under DDP, FSDP and PEFT wrappers. Forwards
+ still go through the wrapper so gradients synchronise."""
+ while hasattr(model, "module") and isinstance(model.module, torch.nn.Module):
+ model = model.module
+ if hasattr(model, "get_base_model"):
+ model = model.get_base_model()
+ return model
+
+
+def response_mask(inputs):
+ """Bool [B, L]: supervised positions. No labels -> every attended token (pretraining-style)."""
+ labels = inputs.get("labels")
+ attention_mask = inputs.get("attention_mask")
+ if labels is not None:
+ mask = labels != -100
+ if attention_mask is not None:
+ mask = mask & attention_mask.bool()
+ return mask
+ if attention_mask is not None:
+ return attention_mask.bool()
+ return torch.ones_like(inputs["input_ids"], dtype = torch.bool)
+
+
+def sample_noise_level(
+ batch_size,
+ eps,
+ device,
+ low = None,
+ high = None,
+):
+ """Per-example t ~ U(low, high), default U(eps, 1 - eps) as in the published recipes."""
+ low = eps if low is None else low
+ high = 1.0 - eps if high is None else high
+ return low + (high - low) * torch.rand(batch_size, device = device)
+
+
+def restore_rotary_buffers(model):
+ """4.x remote code keeps RoPE frequencies in a non-persistent buffer filled in __init__; transformers 5 builds
+ on the meta device and only re-fills it through `_init_weights`, which these classes override, so `inv_freq`
+ comes back zero (GPU) or uninitialised (CPU) and positions are silently ignored. Recompute it."""
+ repaired = 0
+ for module in model.modules():
+ init_fn = getattr(module, "rope_init_fn", None)
+ inv_freq = getattr(module, "inv_freq", None)
+ config = getattr(module, "config", None) or getattr(model, "config", None)
+ if (
+ init_fn is None
+ or config is None
+ or not isinstance(inv_freq, torch.Tensor)
+ or inv_freq.is_meta
+ ):
+ continue
+ try:
+ expected, scaling = init_fn(
+ config, inv_freq.device, **getattr(module, "rope_kwargs", {})
+ )
+ except Exception:
+ continue
+ expected = expected.to(device = inv_freq.device, dtype = inv_freq.dtype)
+ if expected.shape != inv_freq.shape or torch.equal(inv_freq, expected):
+ continue
+ with torch.no_grad():
+ inv_freq.copy_(expected)
+ if isinstance(getattr(module, "original_inv_freq", None), torch.Tensor):
+ module.original_inv_freq = inv_freq.clone()
+ if hasattr(module, "attention_scaling"):
+ module.attention_scaling = scaling
+ repaired += 1
+ return repaired
diff --git a/unsloth/models/diffusion_trainer.py b/unsloth/models/diffusion_trainer.py
new file mode 100644
index 00000000000..e611babe470
--- /dev/null
+++ b/unsloth/models/diffusion_trainer.py
@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""SFT for text-diffusion language models: SFTTrainer data handling, the family's own denoising loss."""
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Optional
+
+import torch
+from trl import SFTConfig, SFTTrainer
+
+from .diffusion_profiles import resolve_diffusion_profile
+
+__all__ = ["DiffusionConfig", "DiffusionTrainer"]
+
+
+@dataclass
+class DiffusionConfig(SFTConfig):
+ """``SFTConfig`` plus the diffusion objective knobs. ``None`` = the model family's reference value."""
+
+ diffusion_eps: Optional[float] = field(
+ default = None, metadata = {"help": "Noise level t is drawn from U(eps, 1 - eps)."}
+ )
+ diffusion_time_weighting: Optional[str] = field(
+ default = None,
+ metadata = {
+ "help": "'inverse_t' (masked-diffusion ELBO), 'none', or a family-specific scheme."
+ },
+ )
+ diffusion_mask_token_id: Optional[int] = field(
+ default = None, metadata = {"help": "Override the [MASK] token for absorbing-noise families."}
+ )
+ diffusion_block_size: Optional[int] = field(
+ default = None, metadata = {"help": "Block length for block-diffusion families."}
+ )
+ diffusion_self_conditioning_p: Optional[float] = field(
+ default = None, metadata = {"help": "Probability of the two-pass self-conditioned forward."}
+ )
+ diffusion_encoder_ar_weight: Optional[float] = field(
+ default = None,
+ metadata = {
+ "help": "Weight of the encoder autoregressive co-loss (encoder/decoder families)."
+ },
+ )
+ diffusion_prediction_type: Optional[str] = field(
+ default = None,
+ metadata = {"help": "Family-specific prediction parameterisation (e.g. 'mean', 'mean_loo')."},
+ )
+
+ def __post_init__(self):
+ # Packed rows share one bidirectional attention window, so documents would see each other.
+ self.packing = False
+ self.padding_free = False
+ # TRL 1.x enables MoE router logits from this coef; no diffusion objective adds the aux term.
+ if hasattr(self, "router_aux_loss_coef"):
+ self.router_aux_loss_coef = 0.0
+ super().__post_init__()
+
+
+def _unwrap_config(model):
+ for candidate in (model, getattr(model, "base_model", None), getattr(model, "module", None)):
+ config = getattr(candidate, "config", None)
+ if config is not None:
+ return config
+ return None
+
+
+class DiffusionTrainer(SFTTrainer):
+ def __init__(
+ self,
+ model = None,
+ args = None,
+ *pos,
+ diffusion_profile = None,
+ **kwargs,
+ ):
+ if args is None:
+ args = DiffusionConfig(output_dir = "outputs")
+ elif not isinstance(args, DiffusionConfig):
+ args = DiffusionConfig(
+ **{
+ k: v
+ for k, v in args.to_dict().items()
+ if k in DiffusionConfig.__dataclass_fields__
+ }
+ )
+ args.packing = False
+ args.padding_free = False
+ profile = diffusion_profile or resolve_diffusion_profile(_unwrap_config(model))
+ if profile is None:
+ raise ValueError(
+ "Unsloth: DiffusionTrainer needs a text-diffusion model. "
+ f"Got model_type={getattr(_unwrap_config(model), 'model_type', None)!r}."
+ )
+ self.diffusion_profile = profile
+ self._diffusion_metrics = defaultdict(list)
+ super().__init__(model, args, *pos, **kwargs)
+ # The loss is ours, so the Trainer must scale it for gradient accumulation itself.
+ self.model_accepts_loss_kwargs = False
+
+ def compute_loss(
+ self,
+ model,
+ inputs,
+ return_outputs = False,
+ num_items_in_batch = None,
+ ):
+ loss, outputs, metrics = self.diffusion_profile.compute_loss(
+ model, inputs, self.args, num_items_in_batch = num_items_in_batch
+ )
+ mode = "train" if model.training else "eval"
+ for key, value in (metrics or {}).items():
+ self._diffusion_metrics[f"{mode}/{key}"].append(float(value))
+ return (loss, outputs) if return_outputs else loss
+
+ def prediction_step(
+ self,
+ model,
+ inputs,
+ prediction_loss_only,
+ ignore_keys = None,
+ ):
+ inputs = self._prepare_inputs(inputs)
+ with torch.no_grad(), self.compute_loss_context_manager():
+ loss = self.compute_loss(model, inputs)
+ return (loss.detach(), None, None)
+
+ def log(self, logs, *args, **kwargs):
+ mode = "eval" if any(k.startswith("eval_") for k in logs) else "train"
+ prefix = "eval_" if mode == "eval" else ""
+ for key in [k for k in self._diffusion_metrics if k.startswith(mode + "/")]:
+ values = self._diffusion_metrics.pop(key)
+ if values:
+ logs[prefix + key.split("/", 1)[1]] = sum(values) / len(values)
+ return super().log(logs, *args, **kwargs)
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index aed68d94888..f3fd14cdc77 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -1790,6 +1790,7 @@ from ..kernels import (
)
from .vision import FastBaseModel, _is_text_seq2seq_config
from .diffusion import FastDiffusionModel, is_diffusion_model_type
+from .diffusion_profiles import resolve_diffusion_profile
from transformers import (
AutoModelForCausalLM,
)
@@ -2131,11 +2132,15 @@ class FastModel(FastBaseModel):
device_map_planner_kwargs = device_map_planner_kwargs,
trust_remote_code = trust_remote_code,
revision = base_revision,
+ **({"config": user_config} if user_config is not None else {}),
**kwargs,
)
# Returns before the FORCE_FLOAT32 scan and no diffusion type is on that list, so False. Stamped, not left unset, or the trainer reads whatever an earlier load wrote.
model = _mark_forced_float32(model, False)
model = _mark_full_finetuning(model, full_finetuning)
+ # This early return skips the re-enable at the end of from_pretrained.
+ if not was_disabled:
+ enable_progress_bars()
return _mark_requested_float32(model, user_float32), tokenizer
try:
@@ -2253,7 +2258,22 @@ class FastModel(FastBaseModel):
fast_inference = False
# Text-diffusion models (DiffusionGemma) take a transformers-only slow path: a custom block-diffusion generate over a novel backbone, so Unsloth's autoregressive kernel/compile patching is skipped and the unmodified HF model is loaded, keeping 4bit/8bit and PEFT LoRA.
- if is_diffusion_model_type(model_types):
+ # Remote-code checkpoints can report their backbone's type (Nemotron-Labs-Diffusion says "nemotron"), so also match the raw config.
+ diffusion_config = model_config
+ if diffusion_config is None and peft_config is not None:
+ try:
+ diffusion_config = AutoConfig.from_pretrained(
+ peft_config.base_model_name_or_path,
+ token = token,
+ trust_remote_code = trust_remote_code,
+ local_files_only = local_files_only,
+ )
+ except Exception:
+ diffusion_config = None
+ if (
+ is_diffusion_model_type(model_types)
+ or resolve_diffusion_profile(diffusion_config) is not None
+ ):
return _dispatch_diffusion()
lowered_model_name = model_name.lower()