Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Train text-diffusion LMs: DiffusionTrainer with recipes for DiffusionGemma, LLaDA, LLaDA-MoE, LLaDA2, Dream, Nemotron and SDAR (#13157)
Daniel Hancd14635e2cc6
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.jsonindex fb5a1d610de..42f9d51234c 100644--- a/scripts/risky_loaders_baseline.json+++ b/scripts/risky_loaders_baseline.json@@ -586,6 +586,14 @@"count": 1,"reason": "Loads the sibling decision_gguf.py by a fixed name beside this file for GGUF export on MLX, without importing unsloth.models; nothing caller-supplied picks the file."},+ {+ "file": "unsloth/models/diffusion_masked.py",+ "rule": "dynamic-import",+ "sink": "get_class_from_dynamic_module",+ "digest": "1900c2baf63f723f",+ "count": 1,+ "reason": "Falls back to the auto class unless the caller passed trust_remote_code; revision pinned to the config commit hash."+ },{"file": "unsloth/models/loader.py","rule": "dynamic-import",diff --git a/tests/diffusion/test_diffusion_block.py b/tests/diffusion/test_diffusion_block.pynew file mode 100644index 00000000000..0e0964f87c7--- /dev/null+++ b/tests/diffusion/test_diffusion_block.py@@ -0,0 +1,302 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Block-diffusion profiles (LLaDA2.x, SDAR): mask layout, batch building, noise, loss vs a naive reference."""++import re+import sys+from types import SimpleNamespace++import pytest+import torch+import torch.nn as nn+import torch.nn.functional as F++MASK_ID = 30+EOS_ID = 31+VOCAB = 32+++class _Decoder(nn.Module):+ def __init__(self, hidden = 16):+ super().__init__()+ self.embed = nn.Embedding(VOCAB, hidden)+ self.pos = nn.Embedding(256, hidden)+ self.qkv = nn.Linear(hidden, 3 * hidden)+ self.out = nn.Linear(hidden, hidden)++ def forward(+ self,+ input_ids,+ attention_mask,+ position_ids,+ use_cache = False,+ return_dict = True,+ ):+ x = self.embed(input_ids) + self.pos(position_ids)+ q, k, v = self.qkv(x)[:, None].chunk(3, dim = -1)+ h = F.scaled_dot_product_attention(q, k, v, attn_mask = attention_mask)[:, 0]+ return (x + self.out(h),)+++class _LM(nn.Module):+ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ self.model = _Decoder()+ self.lm_head = nn.Linear(16, VOCAB)+ self.config = SimpleNamespace(+ model_type = "llada2_moe", eos_token_id = EOS_ID, mask_token_id = MASK_ID+ )++ def get_decoder(self):+ return self.model++ def get_output_embeddings(self):+ return self.lm_head+++def _profiles():+ from unsloth.models.diffusion_block import LLADA2_PROFILE, SDAR_PROFILE+ return LLADA2_PROFILE, SDAR_PROFILE+++def _batch(prompt_lengths, real_lengths, length):+ torch.manual_seed(1)+ batch = len(real_lengths)+ ids = torch.randint(0, 29, (batch, length))+ attention = torch.zeros(batch, length, dtype = torch.long)+ labels = torch.full((batch, length), -100)+ for i, (p, r) in enumerate(zip(prompt_lengths, real_lengths)):+ attention[i, :r] = 1+ labels[i, p:r] = ids[i, p:r]+ ids[i, r:] = EOS_ID+ return {"input_ids": ids, "attention_mask": attention, "labels": labels}+++def test_block_mask_small_layout():+ from unsloth.models.diffusion_block import block_diffusion_attention_mask++ # L = 4, block 2: rows/cols 0-3 noisy, 4-7 clean.+ expected = torch.tensor(+ [+ [1, 1, 0, 0, 0, 0, 0, 0],+ [1, 1, 0, 0, 0, 0, 0, 0],+ [0, 0, 1, 1, 1, 1, 0, 0],+ [0, 0, 1, 1, 1, 1, 0, 0],+ [0, 0, 0, 0, 1, 1, 0, 0],+ [0, 0, 0, 0, 1, 1, 0, 0],+ [0, 0, 0, 0, 1, 1, 1, 1],+ [0, 0, 0, 0, 1, 1, 1, 1],+ ],+ dtype = torch.bool,+ )+ assert torch.equal(block_diffusion_attention_mask(4, 2), expected)+++def test_profiles_resolve_and_route():+ from unsloth.models.diffusion import is_diffusion_model_type+ from unsloth.models.diffusion_profiles import resolve_diffusion_profile++ llada2, sdar = _profiles()+ assert resolve_diffusion_profile(SimpleNamespace(model_type = "llada2_moe")) is llada2+ assert (+ resolve_diffusion_profile(+ SimpleNamespace(model_type = "x", architectures = ["SDARForCausalLM"])+ )+ is sdar+ )+ assert is_diffusion_model_type("llada2_moe") and is_diffusion_model_type("sdar")+ assert not is_diffusion_model_type("qwen3")+++def test_llada2_lora_targets_skip_routed_experts_and_router():+ llada2, _ = _profiles()+ pattern = re.compile(llada2.lora_target_modules)+ hit = [+ "model.layers.0.attention.query_key_value",+ "model.layers.0.attention.dense",+ "model.layers.0.mlp.gate_proj",+ "model.layers.3.mlp.shared_experts.down_proj",+ ]+ miss = [+ "model.layers.3.mlp.experts.7.up_proj",+ "model.layers.3.mlp.gate",+ "lm_head",+ "model.word_embeddings",+ ]+ assert all(pattern.fullmatch(name) for name in hit)+ assert not any(pattern.fullmatch(name) for name in miss)+++def test_llada2_eos_fill_is_block_aligned_and_supervised():+ llada2, _ = _profiles()+ model, args = _LM(), SimpleNamespace()+ inputs = _batch([3, 5, 2], [40, 20, 9], 40)+ inputs["labels"][2] = -100 # a row with nothing to learn gets no EOS target+ clean, maskable, valid = llada2.build_batch(model, inputs, args)+ assert clean.shape[1] == 64 # next multiple of 32 past 40+ assert torch.equal(+ clean[:, :40][inputs["attention_mask"].bool()],+ inputs["input_ids"][inputs["attention_mask"].bool()],+ )+ assert (clean[0, 40:] == EOS_ID).all() and maskable[0, 40:].all()+ assert (clean[1, 20:] == EOS_ID).all() and maskable[1, 20:].all() and not maskable[1, :5].any()+ assert not maskable[2].any()+++def test_llada2_row_clipped_at_max_length_gets_no_eos_tail():+ llada2, _ = _profiles()+ inputs = _batch([3, 5], [40, 20], 40)+ clean, maskable, valid = llada2.build_batch(_LM(), inputs, SimpleNamespace(max_length = 40))+ assert not maskable[0, 40:].any() and valid[0, 40:].all() and (clean[0, 40:] == EOS_ID).all()+ assert (clean[1, 20:] == EOS_ID).all() and maskable[1, 20:].all()+++def test_left_padding_rejected():+ llada2, _ = _profiles()+ inputs = _batch([2], [10], 12)+ inputs["attention_mask"] = inputs["attention_mask"].flip(1)+ with pytest.raises(ValueError, match = "right padding"):+ llada2.build_batch(_LM(), inputs, SimpleNamespace())+++def _naive_loss(profile, model, clean, noisy, masked, maskable, p, valid, block):+ from unsloth.models.diffusion_block import block_diffusion_attention_mask++ batch, length = clean.shape+ allowed = (+ block_diffusion_attention_mask(length, block)[None, None].expand(batch, 1, -1, -1).clone()+ )+ if profile.defaults["key_padding"]:+ keys = torch.cat([valid, valid], 1)[:, None, None, :]+ allowed = (allowed & keys) | torch.eye(2 * length, dtype = torch.bool)+ positions = torch.arange(length)+ hidden = model.model(+ torch.cat([noisy, clean], 1),+ allowed,+ torch.cat([positions, positions])[None].expand(batch, -1),+ )[0]+ logits = model.lm_head(hidden)[:, :length]+ nll = F.cross_entropy(logits.transpose(1, 2), clean, reduction = "none")+ if profile.defaults["diffusion_time_weighting"] == "inverse_t":+ nll = nll / p[:, None]+ nll = nll * masked+ denominator = maskable.sum() if profile.defaults["normalize"] == "supervised" else masked.sum()+ return nll.sum() / denominator++[email protected]("which", ["llada2", "sdar"])+def test_loss_matches_naive_full_forward(which):+ llada2, sdar = _profiles()+ profile = llada2 if which == "llada2" else sdar+ model = _LM()+ args = SimpleNamespace(diffusion_block_size = 4)+ clean, maskable, valid = profile.build_batch(model, _batch([3, 6], [22, 13], 22), args)+ torch.manual_seed(2)+ noisy, masked, p = profile.sample_noise(clean, maskable, MASK_ID, args)+ ours = profile.loss_from_noise(model, clean, noisy, masked, maskable, p, valid, args)+ ref = _naive_loss(profile, model, clean, noisy, masked, maskable, p, valid, 4)+ torch.testing.assert_close(ours, ref, rtol = 1e-5, atol = 1e-6)+++def test_noisy_block_never_sees_its_own_or_later_clean_tokens():+ llada2, _ = _profiles()+ model, args = _LM(), SimpleNamespace(diffusion_block_size = 4)+ clean = torch.randint(0, 29, (1, 12))+ noisy = clean.clone()+ noisy[:, 4:] = MASK_ID+ valid = torch.ones_like(clean, dtype = torch.bool)+ base, _ = llada2.noisy_hidden_and_head(model, noisy, clean, valid, 4)+ changed = clean.clone()+ changed[:, 4:] = (changed[:, 4:] + 1) % 29+ moved, _ = llada2.noisy_hidden_and_head(model, noisy, changed, valid, 4)+ torch.testing.assert_close(base[:, :8], moved[:, :8]) # block 1 sees clean block 0 only+ assert not torch.allclose(base[:, 8:], moved[:, 8:]) # block 2 sees the changed clean block 1+++def test_sdar_padding_does_not_leak_into_real_tokens():+ _, sdar = _profiles()+ model, args = _LM(), SimpleNamespace(diffusion_block_size = 4)+ short = _batch([2], [9], 9)+ padded = _batch([2], [9], 16)+ padded["input_ids"][:, :9] = short["input_ids"]+ padded["labels"][:, :9] = short["labels"]+ outs = []+ for inputs in (short, padded):+ clean, maskable, valid = sdar.build_batch(model, inputs, args)+ noisy = torch.where(maskable, MASK_ID, clean)+ hidden, _ = sdar.noisy_hidden_and_head(model, noisy, clean, valid, 4)+ outs.append(hidden[:, :9])+ torch.testing.assert_close(outs[0], outs[1], rtol = 1e-5, atol = 1e-6)+++def test_noise_ranges_and_prompt_untouched():+ llada2, sdar = _profiles()+ clean = torch.randint(0, 29, (2048, 24))+ maskable = torch.ones_like(clean, dtype = torch.bool)+ maskable[:, :5] = False+ for profile, (low, high) in ((llada2, (0.3, 0.8)), (sdar, (1e-3, 1.0))):+ torch.manual_seed(0)+ noisy, masked, p = profile.sample_noise(clean, maskable, MASK_ID, SimpleNamespace())+ assert p.min() >= low and p.max() <= high+ assert not masked[:, :5].any() and torch.equal(noisy == MASK_ID, masked)+ ratio = masked[:, 5:].float().mean().item()+ assert abs(ratio - (low + high) / 2) < 0.02+ _, masked, _ = sdar.sample_noise(clean[:, :6], maskable[:, :6], MASK_ID, SimpleNamespace())+ assert masked.any(dim = 1).all() # SDAR masks at least one supervised token per row+ _, _, p = llada2.sample_noise(clean, maskable, MASK_ID, SimpleNamespace(diffusion_eps = 0.1))+ assert p.min() >= 0.1 and p.max() <= 0.9+++def test_sdar_import_shim_is_scoped():+ from unsloth.models.diffusion_block import _sdar_remote_code_imports+ import importlib.util++ if importlib.util.find_spec("flash_attn") is not None:+ pytest.skip("flash-attn installed: the shim is a no-op")+ with _sdar_remote_code_imports():+ from flash_attn.ops.triton.layer_norm import rms_norm_fn+ x = torch.randn(3, 8)+ torch.testing.assert_close(+ rms_norm_fn(x, torch.ones(8), eps = 1e-6),+ x * torch.rsqrt(x.pow(2).mean(-1, keepdim = True) + 1e-6),+ )+ assert "flash_attn" not in sys.modules+++def test_restore_rotary_buffers_refills_zeroed_inv_freq():+ from unsloth.models.diffusion_profiles import restore_rotary_buffers++ def init_fn(config, device = None):+ dim = config.head_dim+ return 1.0 / (+ config.rope_theta ** (torch.arange(0, dim, 2, dtype = torch.float, device = device) / dim)+ ), 1.0++ class Rotary(nn.Module):+ def __init__(self):+ super().__init__()+ self.config = SimpleNamespace(head_dim = 8, rope_theta = 10000.0)+ self.rope_init_fn = init_fn+ self.register_buffer("inv_freq", torch.zeros(4), persistent = False)+ self.original_inv_freq = self.inv_freq++ holder = nn.Module()+ holder.rotary_emb = Rotary()+ assert restore_rotary_buffers(holder) == 1+ torch.testing.assert_close(holder.rotary_emb.inv_freq, init_fn(holder.rotary_emb.config)[0])+ assert restore_rotary_buffers(holder) == 0 # idempotent+++def test_deepspeed_engine_keeps_its_own_forward():+ from unsloth.models.diffusion_block import _is_distributed_wrapper+ engine = type("DeepSpeedEngine", (nn.Module,), {})()+ assert _is_distributed_wrapper(engine) and not _is_distributed_wrapper(_LM())+++def test_unknown_time_weighting_is_rejected():+ llada2, _ = _profiles()+ inputs = _batch([3], [20], 20)+ with pytest.raises(ValueError, match = "diffusion_time_weighting"):+ llada2.compute_loss(_LM(), inputs, SimpleNamespace(diffusion_time_weighting = "inverse-t"))diff --git a/tests/diffusion/test_diffusion_gemma.py b/tests/diffusion/test_diffusion_gemma.pynew file mode 100644index 00000000000..85a28393f4e--- /dev/null+++ b/tests/diffusion/test_diffusion_gemma.py@@ -0,0 +1,279 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++import os+import types++import pytest+from real_accelerator import has_real_cuda++torch = pytest.importorskip("torch")+transformers = pytest.importorskip("transformers")+if not hasattr(transformers, "DiffusionGemmaForBlockDiffusion"):+ pytest.skip("transformers without DiffusionGemma", allow_module_level = True)++from unsloth.models.diffusion_gemma_objective import DIFFUSION_GEMMA_PROFILE+from unsloth.models.diffusion_profiles import resolve_diffusion_profile++TINY = "trl-internal-testing/tiny-DiffusionGemmaForBlockDiffusion"+TRL_EXAMPLE = os.environ.get("UNSLOTH_TRL_DIFFUSION_GEMMA_EXAMPLE")++[email protected](scope = "module")+def model():+ m = transformers.DiffusionGemmaForBlockDiffusion.from_pretrained(TINY, dtype = torch.float32)+ m._unsloth_diffusion_eos_token_id = 1+ return m.train()+++def _batch(+ vocab,+ canvas,+ max_length = 80,+ rows = None,+):+ g = torch.Generator().manual_seed(0)+ if rows is None:+ rows = [(5, 10), (7, canvas * 2 + 3), (4, max_length - 4)]+ input_ids = torch.zeros(len(rows), max_length, dtype = torch.long)+ attention_mask = torch.zeros_like(input_ids)+ labels = torch.full_like(input_ids, -100)+ for i, (p, r) in enumerate(rows):+ n = p + r+ input_ids[i, :n] = torch.randint(10, vocab, (n,), generator = g)+ attention_mask[i, :n] = 1+ labels[i, p:n] = input_ids[i, p:n]+ return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}+++def _args(**kw):+ base = dict(+ max_length = 80,+ diffusion_eps = None,+ diffusion_self_conditioning_p = None,+ diffusion_encoder_ar_weight = None,+ diffusion_prediction_type = None,+ )+ base.update(kw)+ return types.SimpleNamespace(**base)+++def test_profile_resolves(model):+ assert resolve_diffusion_profile(model.config) is DIFFUSION_GEMMA_PROFILE+++def test_canvas_reaches_the_decoder(model):+ # The notebook passed canvas_ids, which forward() silently swallows; decoder_input_ids must change the logits.+ cfg = model.config+ ids = torch.randint(10, 1000, (1, 8))+ a = torch.randint(10, 1000, (1, cfg.canvas_length))+ b = torch.randint(10, 1000, (1, cfg.canvas_length))+ with torch.no_grad():+ la = model(input_ids = ids, decoder_input_ids = a).logits+ lb = model(input_ids = ids, decoder_input_ids = b).logits+ assert not torch.allclose(la, lb)+++def test_loss_finite_and_metrics(model):+ batch = _batch(model.config.text_config.vocab_size, model.config.canvas_length)+ torch.manual_seed(0)+ loss, _, metrics = DIFFUSION_GEMMA_PROFILE.compute_loss(model, batch, _args())+ assert torch.isfinite(loss)+ assert set(metrics) == {"diffusion_loss", "encoder_ar_loss", "mean_t"}+++ TRL_EXAMPLE is None, reason = "set UNSLOTH_TRL_DIFFUSION_GEMMA_EXAMPLE to TRL's example file"+)[email protected]("prediction_type", ["mean", "mean_loo"])+def test_parity_with_trl_reference(model, prediction_type):+ source = open(TRL_EXAMPLE, encoding = "utf-8").read()+ source = source.replace(+ "from trl.trainer.utils import maybe_gather_lm_head_ctx",+ "import contextlib\nmaybe_gather_lm_head_ctx = lambda *a: contextlib.nullcontext()",+ )+ namespace = {"__name__": "trl_example"}+ exec(compile(source, TRL_EXAMPLE, "exec"), namespace)+ reference = namespace["DiffusionGemmaSFTTrainer"]+ cfg = model.config+ fake = types.SimpleNamespace(+ args = types.SimpleNamespace(max_length = 80),+ canvas_length = cfg.canvas_length,+ vocab_size = cfg.text_config.vocab_size,+ final_logit_softcapping = cfg.text_config.final_logit_softcapping,+ eos_token_id = 1,+ model_prediction_type = prediction_type,+ model = model,+ eps = reference.eps,+ self_conditioning_p = reference.self_conditioning_p,+ )+ batch = _batch(cfg.text_config.vocab_size, cfg.canvas_length)+ params = [p for p in model.parameters() if p.requires_grad]++ def run(fn):+ model.zero_grad(set_to_none = True)+ torch.manual_seed(1234)+ loss = fn()+ loss.backward()+ return loss.detach(), [None if p.grad is None else p.grad.clone() for p in params]++ ref_loss, ref_grads = run(lambda: reference.compute_loss(fake, model, batch))+ our_loss, our_grads = run(+ lambda: DIFFUSION_GEMMA_PROFILE.compute_loss(+ model, batch, _args(diffusion_prediction_type = prediction_type)+ )[0]+ )+ torch.testing.assert_close(our_loss, ref_loss, rtol = 0, atol = 1e-6)+ for a, b in zip(our_grads, ref_grads):+ if a is None or b is None:+ assert a is None and b is None+ else:+ torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-7)+++def test_tied_encoder_decoder_lora_merge_is_refused():+ from peft import LoraConfig, get_peft_model++ from unsloth.models.diffusion import _refuse_tied_lora_merge++ m = transformers.DiffusionGemmaForBlockDiffusion.from_pretrained(TINY, dtype = torch.float32)+ enc = m.model.encoder.language_model.layers[0].self_attn.q_proj+ dec = m.model.decoder.layers[0].self_attn.q_proj+ assert enc is not dec and enc.weight.data_ptr() == dec.weight.data_ptr()+ m = get_peft_model(+ m, LoraConfig(r = 4, target_modules = DIFFUSION_GEMMA_PROFILE.lora_target_modules)+ )+ m = _refuse_tied_lora_merge(m)+ with pytest.raises(RuntimeError, match = "share base weights"):+ m.merge_and_unload()+++def test_untied_lora_merge_still_allowed():+ from peft import LoraConfig, get_peft_model++ from unsloth.models.diffusion import _refuse_tied_lora_merge++ m = transformers.AutoModelForCausalLM.from_pretrained(+ "trl-internal-testing/tiny-Qwen3ForCausalLM"+ )+ m = _refuse_tied_lora_merge(+ get_peft_model(m, LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))+ )+ assert "merge_and_unload" not in vars(m)+ m.merge_and_unload()+++class _DistributedLike(torch.nn.Module):+ """DDP / FSDP expose neither config nor heads; only forward reaches the module."""++ def __init__(self, inner):+ super().__init__()+ self.module = inner++ def forward(self, **kwargs):+ return self.module(**kwargs)+++def test_loss_through_distributed_wrapper(model):+ batch = _batch(model.config.text_config.vocab_size, model.config.canvas_length)+ torch.manual_seed(0)+ plain = DIFFUSION_GEMMA_PROFILE.compute_loss(model, batch, _args())[0]+ torch.manual_seed(0)+ wrapped = DIFFUSION_GEMMA_PROFILE.compute_loss(_DistributedLike(model), batch, _args())[0]+ torch.testing.assert_close(wrapped, plain)+++def test_row_without_supervised_tokens_adds_no_canvas_loss(model):+ batch = _batch(model.config.text_config.vocab_size, model.config.canvas_length, rows = [(6, 0)])+ torch.manual_seed(0)+ _, _, metrics = DIFFUSION_GEMMA_PROFILE.compute_loss(model, batch, _args())+ assert metrics["diffusion_loss"] == 0.0+++def test_reloaded_adapter_still_refuses_merge(tmp_path):+ from peft import LoraConfig, get_peft_model++ from unsloth import FastModel++ m = transformers.DiffusionGemmaForBlockDiffusion.from_pretrained(TINY, dtype = torch.float32)+ m = get_peft_model(+ m, LoraConfig(r = 4, target_modules = DIFFUSION_GEMMA_PROFILE.lora_target_modules)+ )+ m.save_pretrained(tmp_path)+ reloaded, _ = FastModel.from_pretrained(str(tmp_path), dtype = torch.float32, device_map = "cpu")+ with pytest.raises(RuntimeError, match = "share base weights"):+ reloaded.merge_and_unload()+ # The objective reads its token bookkeeping from the unwrapped module.+ from unsloth.models.diffusion_profiles import unwrap_diffusion_model++ assert "_unsloth_diffusion_eos_token_id" in vars(unwrap_diffusion_model(reloaded))+++def test_reload_uses_the_adapters_saved_tokenizer(tmp_path):+ from peft import LoraConfig, get_peft_model++ from unsloth import FastModel++ m = transformers.DiffusionGemmaForBlockDiffusion.from_pretrained(TINY, dtype = torch.float32)+ m = get_peft_model(+ m, LoraConfig(r = 4, target_modules = DIFFUSION_GEMMA_PROFILE.lora_target_modules)+ )+ m.save_pretrained(tmp_path)+ tokenizer = transformers.AutoTokenizer.from_pretrained(TINY)+ tokenizer.chat_template = "{{ 'adapter-template-marker' }}"+ tokenizer.save_pretrained(tmp_path)+ _, reloaded = FastModel.from_pretrained(str(tmp_path), dtype = torch.float32, device_map = "cpu")+ reloaded = getattr(reloaded, "tokenizer", reloaded)+ assert "adapter-template-marker" in (reloaded.chat_template or "")+++def test_diffusion_load_restores_progress_bars():+ from huggingface_hub.utils import are_progress_bars_disabled, enable_progress_bars++ from unsloth import FastModel++ enable_progress_bars()+ FastModel.from_pretrained(TINY, dtype = torch.float32, device_map = "cpu")+ assert not are_progress_bars_disabled()++[email protected](not has_real_cuda(), reason = "bitsandbytes needs a GPU")+def test_caller_quantization_config_is_kept():+ from transformers import BitsAndBytesConfig++ from unsloth import FastModel++ qcfg = BitsAndBytesConfig(load_in_4bit = True, bnb_4bit_quant_type = "fp4")+ m, _ = FastModel.from_pretrained(TINY, quantization_config = qcfg, load_in_4bit = True)+ assert m.config.quantization_config.bnb_4bit_quant_type == "fp4"+++def test_self_conditioning_off_skips_the_conditioning_pass(model):+ batch = _batch(model.config.text_config.vocab_size, model.config.canvas_length)+ calls = []+ hook = model.register_forward_pre_hook(lambda *a: calls.append(1))+ try:+ torch.manual_seed(0)+ DIFFUSION_GEMMA_PROFILE.compute_loss(model, batch, _args(diffusion_self_conditioning_p = 0.0))+ finally:+ hook.remove()+ assert len(calls) == 1+++def test_full_finetuning_skips_lora():+ from unsloth import FastModel++ m, _ = FastModel.from_pretrained(+ TINY, dtype = torch.float32, device_map = "cpu", full_finetuning = True+ )+ assert FastModel.get_peft_model(m, r = 4) is m+ assert not any("lora_" in name for name, _ in m.named_parameters())+++def test_caller_config_is_the_one_loaded():+ from unsloth import FastModel++ config = transformers.AutoConfig.from_pretrained(TINY)+ config.unsloth_test_marker = 7+ m, _ = FastModel.from_pretrained(TINY, config = config, dtype = torch.float32, device_map = "cpu")+ assert getattr(m.config, "unsloth_test_marker", None) == 7diff --git a/tests/diffusion/test_diffusion_masked.py b/tests/diffusion/test_diffusion_masked.pynew file mode 100644index 00000000000..313e5f38495--- /dev/null+++ b/tests/diffusion/test_diffusion_masked.py@@ -0,0 +1,475 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Masked-diffusion recipes reproduce each family's published loss code on the same batch and seed."""++import os+import re+from types import SimpleNamespace++import pytest+import torch+import torch.nn.functional as F++VOCAB = 97+MASK_ID = 90+EOS_ID = 3+PAD_ID = 5+++class _Bidirectional(torch.nn.Module):+ """Every position sees the whole row, so a wrong mask or fill changes every logit."""++ def __init__(self, mask_token_id = MASK_ID):+ super().__init__()+ torch.manual_seed(0)+ self.embed = torch.nn.Embedding(VOCAB, 16)+ self.mix = torch.nn.Linear(16, 16)+ self.head = torch.nn.Linear(16, VOCAB)+ self.config = SimpleNamespace(mask_token_id = mask_token_id, eos_token_id = EOS_ID)+ self.seen = []++ def forward(+ self,+ input_ids,+ attention_mask = None,+ ):+ self.seen.append((input_ids.clone(), attention_mask))+ h = self.embed(input_ids)+ h = h + self.mix(h.mean(dim = 1, keepdim = True))+ return SimpleNamespace(logits = self.head(torch.tanh(h)))+++def _batch():+ # (prompt, response) lengths; right padded with a pad id distinct from EOS.+ shapes = [(4, 7), (6, 3), (3, 5)]+ length = max(p + r for p, r in shapes) + 2+ gen = torch.Generator().manual_seed(1)+ input_ids = torch.full((len(shapes), length), PAD_ID)+ attention_mask = torch.zeros_like(input_ids)+ labels = torch.full_like(input_ids, -100)+ prompt_lengths = []+ for i, (p, r) in enumerate(shapes):+ row = torch.randint(10, 80, (p + r,), generator = gen)+ row[-1] = EOS_ID+ input_ids[i, : p + r] = row+ attention_mask[i, : p + r] = 1+ labels[i, p : p + r] = row[p:]+ prompt_lengths.append(p)+ return {+ "input_ids": input_ids,+ "attention_mask": attention_mask,+ "labels": labels,+ }, torch.tensor(prompt_lengths)+++def _eos_filled(inputs):+ return inputs["input_ids"].masked_fill(inputs["attention_mask"] == 0, EOS_ID)+++def _profile(name):+ from unsloth.models.diffusion_profiles import diffusion_profiles+ return {p.name: p for p in diffusion_profiles()}[name]+++# LLaDA GUIDELINES.md, "SFT" section, verbatim apart from names.+def _llada_reference(+ model,+ input_ids,+ prompt_length,+ mask_id,+ eps = 1e-3,+):+ b, l = input_ids.shape+ t = torch.rand(b, device = input_ids.device)+ p_mask = (1 - eps) * t + eps+ p_mask = p_mask[:, None].repeat(1, l)+ masked_indices = torch.rand((b, l), device = input_ids.device) < p_mask+ noisy_batch = torch.where(masked_indices, mask_id, input_ids)+ token_positions = torch.arange(noisy_batch.shape[1]).expand(+ noisy_batch.size(0), noisy_batch.size(1)+ )+ prompt_mask = token_positions < prompt_length.unsqueeze(1)+ noisy_batch[prompt_mask] = input_ids[prompt_mask]+ prompt_mask = prompt_mask.to(torch.int64)+ answer_lengths = torch.sum((1 - prompt_mask), dim = -1, keepdim = True)+ answer_lengths = answer_lengths.repeat(1, noisy_batch.shape[1])+ masked_indices = noisy_batch == mask_id+ logits = model(input_ids = noisy_batch).logits+ token_loss = (+ F.cross_entropy(logits[masked_indices], input_ids[masked_indices], reduction = "none")+ / p_mask[masked_indices]+ )+ return torch.sum(token_loss / answer_lengths[masked_indices]) / input_ids.shape[0]+++# Dream src/trainer/fsdp_sft_trainer.py _compute_loss_and_backward + src/diffllm/gen_utils.q_sample,+# with the default config (time_reweighting "original", no token reweighting, treat_eos_as_one off).+def _dream_reference(model, input_ids, loss_mask, mask_id):+ t = torch.rand((input_ids.shape[0],), dtype = torch.float)+ u = torch.rand_like(input_ids, dtype = torch.float)+ t_mask = (u < t[:, None]) & loss_mask+ masked_input_ids = input_ids.masked_fill(t_mask, mask_id)+ logits = model(input_ids = masked_input_ids).logits+ shift_logits = torch.cat([logits[:, 0:1], logits[:, :-1]], dim = 1)+ loss = F.cross_entropy(shift_logits.reshape(-1, VOCAB), input_ids.reshape(-1), reduction = "none")+ loss = loss.masked_fill(~t_mask.reshape(-1), 0)+ loss = loss * (1 / t[:, None].float().expand(input_ids.size())).reshape(-1)+ return torch.sum(loss) / torch.sum(t_mask)+++# Nemotron-Labs-Diffusion modeling_nemotron_labs_diffusion.py forward_process + bidirectional loss branch;+# the (sum, count) pair it returns is divided by the trainer, as global_loss_avg does on one rank.+def _nemotron_reference(+ model,+ input_ids,+ loss_mask,+ mask_id,+ eps = 1e-3,+):+ b, l = input_ids.shape+ t = torch.rand(b)+ p_mask = ((1 - eps) * t + eps)[:, None].expand(-1, l)+ masked_indices = torch.rand((b, l)) < p_mask+ masked_indices[loss_mask == 0] = 0+ noisy = torch.where(masked_indices, mask_id, input_ids)+ logits = model(input_ids = noisy).logits+ token_loss = (+ F.cross_entropy(logits[masked_indices], input_ids[masked_indices], reduction = "none")+ / p_mask[masked_indices]+ )+ return token_loss.sum() / masked_indices.sum()+++def _ours(name, model, inputs, seed):+ torch.manual_seed(seed)+ loss, _, metrics = _profile(name).compute_loss(model, inputs, None)+ return loss, metrics++[email protected]("name", ["llada", "llada_moe"])[email protected]("seed", [0, 1, 2])+def test_llada_matches_guidelines_sft_loss(name, seed):+ inputs, prompt_lengths = _batch()+ model = _Bidirectional()+ torch.manual_seed(seed)+ reference = _llada_reference(model, _eos_filled(inputs), prompt_lengths, MASK_ID)+ loss, _ = _ours(name, model, inputs, seed)+ assert torch.allclose(loss, reference, atol = 0, rtol = 1e-6), (+ loss.item(),+ reference.item(),+ (loss - reference).item(),+ )++[email protected]("seed", [0, 1, 2])+def test_dream_matches_official_sft_loss(seed):+ inputs, _ = _batch()+ model = _Bidirectional()+ loss_mask = (inputs["labels"] != -100) | (inputs["attention_mask"] == 0)+ torch.manual_seed(seed)+ reference = _dream_reference(model, _eos_filled(inputs), loss_mask, MASK_ID)+ loss, _ = _ours("dream", model, inputs, seed)+ assert torch.allclose(loss, reference, atol = 0, rtol = 1e-6), (+ loss.item(),+ reference.item(),+ (loss - reference).item(),+ )++[email protected]("seed", [0, 1, 2])+def test_nemotron_matches_remote_forward_loss(seed):+ inputs, _ = _batch()+ model = _Bidirectional()+ loss_mask = (inputs["labels"] != -100) | (inputs["attention_mask"] == 0)+ torch.manual_seed(seed)+ reference = _nemotron_reference(model, _eos_filled(inputs), loss_mask, MASK_ID)+ loss, _ = _ours("nemotron_labs_diffusion", model, inputs, seed)+ assert torch.allclose(loss, reference, atol = 0, rtol = 1e-6), (+ loss.item(),+ reference.item(),+ (loss - reference).item(),+ )++[email protected]("name", ["llada", "llada_moe", "dream", "nemotron_labs_diffusion"])+def test_prompt_stays_clean_and_padding_becomes_attended_eos(name):+ inputs, prompt_lengths = _batch()+ model = _Bidirectional()+ _ours(name, model, inputs, 0)+ noisy, attention_mask = model.seen[-1]+ assert attention_mask is None+ for i, p in enumerate(prompt_lengths.tolist()):+ assert torch.equal(noisy[i, :p], inputs["input_ids"][i, :p])+ padding = inputs["attention_mask"] == 0+ assert set(noisy[padding].tolist()) <= {EOS_ID, MASK_ID}+ assert PAD_ID not in noisy.tolist()+++def test_unsupervised_batch_masks_every_attended_token_and_no_padding():+ inputs, _ = _batch()+ inputs.pop("labels")+ model = _Bidirectional()+ torch.manual_seed(0)+ _profile("llada").compute_loss(model, inputs, None)+ noisy, _ = model.seen[-1]+ assert not (noisy[inputs["attention_mask"] == 0] == MASK_ID).any()+++class _DistributedLike(torch.nn.Module):+ """DDP / FSDP expose neither config nor Unsloth attributes; only forward reaches the module."""++ def __init__(self, inner):+ super().__init__()+ self.module = inner++ def forward(self, **kwargs):+ return self.module(**kwargs)++[email protected]("name", ["llada", "dream", "nemotron_labs_diffusion"])+def test_loss_through_distributed_wrapper(name):+ inputs, _ = _batch()+ model = _Bidirectional()+ plain, _ = _ours(name, model, inputs, 0)+ wrapped, _ = _ours(name, _DistributedLike(model), inputs, 0)+ torch.testing.assert_close(wrapped, plain)+++def test_row_without_response_gets_no_eos_targets():+ inputs, _ = _batch()+ inputs["labels"][1] = -100+ model = _Bidirectional()+ torch.manual_seed(0)+ _profile("llada").compute_loss(model, inputs, None)+ noisy, _ = model.seen[-1]+ assert not (noisy[1] == MASK_ID).any()+++def test_mask_token_falls_back_to_family_default():+ inputs, _ = _batch()+ model = _Bidirectional(mask_token_id = None)+ _profile("llada_moe").prepare_model(model, None)+ assert model._unsloth_mask_token_id == 156895+ model = _Bidirectional(mask_token_id = None)+ _profile("llada").prepare_model(model, None)+ assert model._unsloth_mask_token_id == 126336+++def test_all_masked_rows_absent_gives_zero_loss_with_grad():+ inputs, _ = _batch()+ inputs["labels"].fill_(-100)+ inputs["attention_mask"].fill_(1)+ model = _Bidirectional()+ loss, _ = _ours("dream", model, inputs, 0)+ assert loss.item() == 0.0 and loss.requires_grad+++ "config, expected",+ [+ (dict(model_type = "llada", architectures = ["LLaDAModelLM"]), "llada"),+ (dict(model_type = "llada", architectures = ["LLaDAMoEModel"]), "llada_moe"),+ (dict(model_type = "Dream", architectures = ["DreamModel"]), "dream"),+ (+ dict(+ model_type = "nemotron_labs_diffusion", architectures = ["NemotronLabsDiffusionModel"]+ ),+ "nemotron_labs_diffusion",+ ),+ (dict(model_type = "nemotron", architectures = ["NemotronForCausalLM"]), None),+ (dict(model_type = "qwen3", architectures = ["Qwen3ForCausalLM"]), None),+ ],+)+def test_profile_routing(config, expected):+ from unsloth.models.diffusion_profiles import resolve_diffusion_profile+ profile = resolve_diffusion_profile(SimpleNamespace(**config))+ assert (profile.name if profile else None) == expected+++def test_lora_targets_skip_output_head_router_and_experts():+ llada = _profile("llada").lora_target_modules+ assert re.fullmatch(llada, "model.transformer.blocks.3.ff_out")+ assert re.fullmatch(llada, "model.transformer.blocks.3.attn_out")+ assert not re.fullmatch(llada, "model.transformer.ff_out")+ moe = _profile("llada_moe").lora_target_modules+ assert re.fullmatch(moe, "model.layers.0.self_attn.o_proj")+ assert not re.fullmatch(moe, "model.layers.0.mlp.gate")+ assert not re.fullmatch(moe, "model.layers.0.mlp.experts.7.down_proj")+++def test_rotary_buffers_rebuilt_from_rope_init_fn():+ from unsloth.models.diffusion_profiles import restore_rotary_buffers as _rebuild_rotary_buffers++ def init(config, device, **kwargs):+ return 1.0 / (config.rope_theta ** (torch.arange(0, 8, 2, dtype = torch.float32) / 8)), 1.0++ class Rotary(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = SimpleNamespace(rope_theta = 10000.0)+ self.rope_init_fn = init+ self.rope_kwargs = {}+ self.register_buffer("inv_freq", torch.zeros(4), persistent = False)+ self.original_inv_freq = torch.empty(4, device = "meta")++ model = torch.nn.Module()+ model.config = SimpleNamespace()+ model.rotary = Rotary()+ expected = init(model.rotary.config, "cpu")[0]+ assert _rebuild_rotary_buffers(model) == 1+ assert torch.equal(model.rotary.inv_freq, expected)+ assert torch.equal(model.rotary.original_inv_freq, expected)+ assert _rebuild_rotary_buffers(model) == 0 and torch.equal(model.rotary.inv_freq, expected)+++def test_dream_generate_defaults_to_filled_model_generation_config():+ from unsloth.models.diffusion_masked import _default_diffusion_generate_config++ seen = {}++ class Dream:+ config = SimpleNamespace(+ mask_token_id = 151666, eos_token_id = 151643, pad_token_id = 151643, bos_token_id = 151643+ )+ generation_config = SimpleNamespace(+ mask_token_id = None, eos_token_id = 151643, pad_token_id = None, bos_token_id = None+ )++ def diffusion_generate(+ self,+ inputs = None,+ generation_config = None,+ **kwargs,+ ):+ seen["config"] = generation_config+ return inputs++ model = Dream()+ _default_diffusion_generate_config(model)+ _default_diffusion_generate_config(model)+ model.diffusion_generate("x", steps = 4)+ assert seen["config"].mask_token_id == 151666 and seen["config"].pad_token_id == 151643+ assert model.generation_config.mask_token_id is None+ explicit = SimpleNamespace(mask_token_id = 7)+ model.diffusion_generate("x", generation_config = explicit)+ assert seen["config"] is explicit+++def test_mask_builder_signature_exposed_from_wrapper_closure():+ import inspect++ from unsloth.models import diffusion_masked++ def original(+ config,+ inputs_embeds,+ attention_mask,+ past_key_values,+ position_ids = None,+ ):+ return None++ def make(signature):+ def return_attention_mask(*args, **kwargs):+ return signature++ return return_attention_mask++ wrapper = make(inspect.signature(original))+ fake = SimpleNamespace(create_causal_mask = wrapper, create_sliding_window_causal_mask = None)+ import sys++ real = sys.modules.get("transformers.masking_utils")+ sys.modules["transformers.masking_utils"] = fake+ try:+ import transformers++ saved = getattr(transformers, "masking_utils", None)+ transformers.masking_utils = fake+ diffusion_masked._expose_mask_signatures()+ finally:+ sys.modules["transformers.masking_utils"] = real+ transformers.masking_utils = saved+ assert list(inspect.signature(wrapper).parameters)[:2] == ["config", "inputs_embeds"]+++_TINY_TRAIN = r"""+import math, os, sys, torch+import unsloth+from datasets import Dataset+from unsloth import DiffusionConfig, DiffusionTrainer, FastModel++path, out = sys.argv[1], sys.argv[2]+device = os.environ.get("TINY_DEVICE", "cpu")+model, processor = FastModel.from_pretrained(path, trust_remote_code = True, dtype = torch.float32, device_map = {"": device})+assert getattr(model, "_unsloth_slow_diffusion", False), type(model)+tokenizer = getattr(processor, "tokenizer", processor)+for module in model.modules():+ if hasattr(module, "rope_init_fn") and isinstance(getattr(module, "inv_freq", None), torch.Tensor):+ assert module.inv_freq.abs().sum() > 0, "RoPE inv_freq left uninitialised"+model = FastModel.get_peft_model(model, r = 16, lora_alpha = 32, use_gradient_checkpointing = True)+before = {k: v.detach().clone() for k, v in model.named_parameters() if "lora_B" in k}+rows = [{"prompt": f"Question {i}: what is {i} plus {i}?", "completion": f" The answer is {2 * i}."} for i in range(4)] * 16+args = DiffusionConfig(+ output_dir = out, max_steps = 20, per_device_train_batch_size = 4, learning_rate = 2e-2, logging_steps = 1,+ report_to = [], save_strategy = "no", seed = 3407, use_cpu = device == "cpu", max_length = 64,+ bf16 = False, fp16 = False, dataset_num_proc = 1, gradient_checkpointing = True,+)+trainer = DiffusionTrainer(model = model, args = args, train_dataset = Dataset.from_list(rows), processing_class = tokenizer)+fixed = trainer._prepare_inputs(next(iter(trainer.get_train_dataloader())))+++def fixed_noise_ce():+ # Per-step losses are too noisy over 20 steps (1 / t weight, fresh noise); score one batch under one draw.+ torch.manual_seed(0)+ with torch.no_grad():+ return trainer.diffusion_profile.compute_loss(model, fixed, args)[2]["masked_ce"]+++first = fixed_noise_ce()+trainer.train()+losses = [h["loss"] for h in trainer.state.log_history if "loss" in h]+assert len(losses) == 20 and all(math.isfinite(x) for x in losses), losses+last = fixed_noise_ce()+assert last < first, (first, last)+after = {k: v for k, v in model.named_parameters() if k in before}+assert any(not torch.equal(before[k], after[k].detach()) for k in before)+model.save_pretrained(os.path.join(out, "adapter"))+tokenizer.save_pretrained(os.path.join(out, "adapter"))+reloaded, _ = FastModel.from_pretrained(os.path.join(out, "adapter"), trust_remote_code = True, dtype = torch.float32, device_map = {"": device})+saved = {k.replace(".default", ""): v for k, v in model.named_parameters() if "lora_B" in k}+loaded = {k.replace(".default", ""): v for k, v in reloaded.named_parameters() if "lora_B" in k}+assert saved.keys() == loaded.keys() and saved, (list(saved)[:3], list(loaded)[:3])+assert all(torch.equal(saved[k].detach().cpu(), loaded[k].detach().cpu()) for k in saved)+print("TINY_OK", trainer.diffusion_profile.name, round(first, 4), round(last, 4))+"""+++# Real remote code, tiny random weights: UNSLOTH_DIFFUSION_TINY_DIR/<name> built from the published config and code.+_TINY = os.environ.get("UNSLOTH_DIFFUSION_TINY_DIR")++[email protected](not _TINY, reason = "set UNSLOTH_DIFFUSION_TINY_DIR to tiny remote-code copies")+ "name",+ [+ "LLaDA-8B-Instruct",+ "LLaDA-MoE-7B-A1B-Instruct",+ "Dream-v0-Instruct-7B",+ "Nemotron-Labs-Diffusion-3B",+ ],+)+def test_tiny_remote_checkpoint_trains_and_reloads(name, tmp_path):+ import subprocess+ import sys++ # One process per family: LLaDA's remote config registers "llada" globally, shadowing LLaDA-MoE's.+ out = subprocess.run(+ [sys.executable, "-c", _TINY_TRAIN, os.path.join(_TINY, name), str(tmp_path)],+ capture_output = True,+ text = True,+ timeout = 900,+ )+ assert out.returncode == 0, out.stdout[-3000:] + out.stderr[-3000:]+ assert "TINY_OK" in out.stdoutdiff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.pyindex 2ef7fec7dfa..53ec1d02849 100644--- a/unsloth/models/__init__.py+++ b/unsloth/models/__init__.py@@ -18,6 +18,8 @@ from .qwen3_moe import FastQwen3MoeModelfrom .granite import FastGraniteModelfrom .sentence_transformer import FastSentenceTransformerfrom .decision import *+from .diffusion import *+from .diffusion_trainer import *try:from .falcon_h1 import FastFalconH1Modeldiff --git a/unsloth/models/diffusion.py b/unsloth/models/diffusion.pyindex 259e0e02c8b..e7e51ccd817 100644--- a/unsloth/models/diffusion.py+++ b/unsloth/models/diffusion.py@@ -18,11 +18,18 @@ bit-identical to transformers), keeping only the safe conveniences: 4bit/8bit lo"""import os+import reimport torchfrom transformers import AutoConfig, AutoProcessor, AutoTokenizerfrom unsloth_zoo.hf_utils import add_dtype_kwargsfrom ._utils import is_bfloat16_supported, maybe_prefetch_hf_snapshot+from .diffusion_profiles import (+ diffusion_model_types,+ resolve_diffusion_profile,+ restore_rotary_buffers,+ unwrap_diffusion_model,+)from .llama import loggerfrom .loader_utils import (planner_config_overrides,@@ -58,7 +65,33 @@ def is_diffusion_model_type(model_types):"""model_types: str or iterable -> True if any is a known diffusion model_type."""if isinstance(model_types, str):model_types = (model_types,)- return any(mt in DIFFUSION_MODEL_TYPES for mt in model_types)+ known = DIFFUSION_MODEL_TYPES + diffusion_model_types()+ return any(mt in known for mt in model_types)+++def _refuse_tied_lora_merge(model):+ """DiffusionGemma's encoder and decoder layers share one base tensor but get separate adapters, so a merge+ adds both deltas to both. Refuse it instead of writing a silently wrong checkpoint."""+ owners = {}+ for name, module in model.named_modules():+ weight = getattr(getattr(module, "base_layer", None), "weight", None)+ if weight is None or not hasattr(module, "lora_A") or weight.is_meta:+ continue+ if owners.setdefault(weight.data_ptr(), name) != name:+ break+ else:+ return model++ def refuse(*args, **kwargs):+ raise RuntimeError(+ "Unsloth: this diffusion model's encoder and decoder share base weights but train separate LoRA "+ "adapters, so merging would add both deltas to both. Save the adapter with "+ "model.save_pretrained(...) and load it on top of the base model unmerged."+ )++ for attr in ("merge_and_unload", "merge_adapter"):+ setattr(model, attr, refuse)+ return modeldef _resolve_diffusion_model_class(config):@@ -153,7 +186,7 @@ class FastDiffusionModel:# Planner hints for device_map = "unsloth"; see resolve_unsloth_device_map.device_map_planner_kwargs = None,trust_remote_code = False,- attn_implementation = "eager", # exact match with the reference golden logits+ attn_implementation = None, # family default: eager for DiffusionGemma (reference golden logits)revision = None,return_tokenizer = True,**kwargs,@@ -176,22 +209,63 @@ class FastDiffusionModel:cache_dir = kwargs.get("cache_dir")- config = _load_diffusion_config(- model_name,- token,- trust_remote_code,- revision,- local_files_only,- cache_dir = cache_dir,- )+ adapter_name = None+ adapter_revision = None+ try:+ from peft import PeftConfig+ adapter_config = PeftConfig.from_pretrained(+ model_name,+ token = token,+ revision = revision,+ local_files_only = local_files_only,+ )+ adapter_name, adapter_revision, model_name, revision = (+ model_name,+ revision,+ adapter_config.base_model_name_or_path,+ adapter_config.revision,+ )+ except Exception:+ pass++ config = kwargs.pop("config", None)+ if config is None:+ config = _load_diffusion_config(+ model_name,+ token,+ trust_remote_code,+ revision,+ local_files_only,+ cache_dir = cache_dir,+ )model_type = getattr(config, "model_type", None)- if not is_diffusion_model_type(model_type):+ profile = resolve_diffusion_profile(config)+ if profile is None and not is_diffusion_model_type(model_type):raise RuntimeError(- f"Unsloth: FastDiffusionModel only supports diffusion model_types {DIFFUSION_MODEL_TYPES}, "+ f"Unsloth: FastDiffusionModel only supports diffusion model_types "+ f"{tuple(dict.fromkeys(DIFFUSION_MODEL_TYPES + diffusion_model_types()))}, "f"got '{model_type}'. Use FastModel/FastLanguageModel for autoregressive models.")-- model_cls = _resolve_diffusion_model_class(config)+ if profile is not None and profile.requires_remote_code and not trust_remote_code:+ raise RuntimeError(+ f"Unsloth: {model_name} ships its own modeling code. Pass trust_remote_code = True."+ )+ if attn_implementation is None:+ attn_implementation = (+ profile.defaults.get("attn_implementation") if profile else None+ ) or "eager"++ if profile is not None and model_type not in DIFFUSION_MODEL_TYPES:+ model_cls = profile.model_class(+ config,+ trust_remote_code,+ token = token,+ cache_dir = cache_dir,+ local_files_only = local_files_only,+ code_revision = kwargs.get("code_revision"),+ )+ else:+ model_cls = _resolve_diffusion_model_class(config)# Prefetch the whole repo root so the weight load is a cache hit. No subfolder: the pipeline loads# every component subfolder, so narrowing would leave unet/vae/text_encoder to Xet.@@ -212,8 +286,9 @@ class FastDiffusionModel:# them, and lm_head/embeddings stay full precision. Before the plan, since the skip list becomes# modules_to_not_convert and sizing those at 4 bits while they load in compute dtype OOMs a tight# map.- qcfg = None- if load_in_4bit or load_in_8bit:+ # A caller's own quantization_config (NF4 vs FP4, skip modules, non-bnb quantizers) wins.+ qcfg = kwargs.get("quantization_config")+ if qcfg is None and (load_in_4bit or load_in_8bit):from transformers import BitsAndBytesConfigif load_in_4bit:qcfg = BitsAndBytesConfig(@@ -221,13 +296,11 @@ class FastDiffusionModel:bnb_4bit_use_double_quant = True,bnb_4bit_quant_type = "nf4",bnb_4bit_compute_dtype = dtype,- llm_int8_skip_modules = [- "lm_head",- "embed_tokens",- "experts",- "self_conditioning",- "router",- ],+ llm_int8_skip_modules = (+ profile.quant_skip_modules+ if profile is not None and profile.quant_skip_modules is not None+ else ["lm_head", "embed_tokens", "experts", "self_conditioning", "router"]+ ),)else:qcfg = BitsAndBytesConfig(load_in_8bit = True)@@ -292,39 +365,98 @@ class FastDiffusionModel:load_kwargs["quantization_config"] = qcfgprint(f"==(( Unsloth: FastDiffusionModel (slow / transformers-only path) ))==")- print(f" Model: {model_name} | class: {model_cls.__name__} | model_type: {model_type}")+ print(+ f" Model: {model_name} | class: {model_cls.__name__} | model_type: {model_type}"+ f" | recipe: {profile.name if profile else 'none'}"+ )print(f" dtype: {dtype} | 4bit: {load_in_4bit} | 8bit: {load_in_8bit} | attn: {attn_implementation}")- model = model_cls.from_pretrained(model_name, **load_kwargs).eval()+ model = model_cls.from_pretrained(model_name, config = config, **load_kwargs).eval()+ model._unsloth_full_finetuning = bool(full_finetuning)+ if trust_remote_code:+ # 4.x remote code builds RoPE inv_freq in __init__; transformers 5 leaves it uninitialised.+ from ._remote_code_buffers import restore_remote_code_non_persistent_buffers++ restore_remote_code_non_persistent_buffers(model)+ # That helper misses rotary modules whose class overrides _init_weights (LLaDA, LLaDA2, SDAR, Dream).+ restore_rotary_buffers(model)+ # Remote classes without GenerationMixin (LLaDA-MoE) have none; TRL 1.x SFTTrainer reads it.+ if getattr(model, "generation_config", None) is None:+ from transformers import GenerationConfig+ try:+ model.generation_config = GenerationConfig.from_model_config(model.config)+ except Exception:+ model.generation_config = GenerationConfig()# Mark before any early return so get_peft_model/for_* route to the slow path.model._unsloth_slow_diffusion = True+ if profile is not None:+ model._unsloth_diffusion_profile = profile.name++ if adapter_name is not None:+ from peft import PeftModel++ model = PeftModel.from_pretrained(+ model,+ adapter_name,+ is_trainable = True,+ token = token,+ revision = adapter_revision,+ local_files_only = local_files_only,+ )+ model = _refuse_tied_lora_merge(model)+ model._unsloth_slow_diffusion = Trueif not return_tokenizer:+ if profile is not None:+ profile.prepare_model(unwrap_diffusion_model(model), None)return model, None# Prefer the processor (chat template plus tokenizer), falling back to a bare tokenizer, returned as# "tokenizer" to match the Unsloth (model, tokenizer) contract.+ tokenizer_source, tokenizer_revision = model_name, revision+ if adapter_name is not None:+ from transformers.utils import cached_file++ # A local directory or a Hub repo; the adapter's own tokenizer carries its chat template / tokens.+ try:+ saved = cached_file(+ adapter_name,+ "tokenizer_config.json",+ token = token,+ revision = adapter_revision,+ local_files_only = local_files_only,+ cache_dir = cache_dir,+ _raise_exceptions_for_missing_entries = False,+ _raise_exceptions_for_connection_errors = False,+ )+ except Exception:+ saved = None+ if saved is not None:+ tokenizer_source, tokenizer_revision = adapter_name, adapter_revisiontry:tokenizer = AutoProcessor.from_pretrained(- model_name,+ tokenizer_source,token = token,trust_remote_code = trust_remote_code,- revision = revision,+ revision = tokenizer_revision,local_files_only = local_files_only,cache_dir = cache_dir,)except Exception:tokenizer = AutoTokenizer.from_pretrained(- model_name,+ tokenizer_source,token = token,trust_remote_code = trust_remote_code,- revision = revision,+ revision = tokenizer_revision,local_files_only = local_files_only,cache_dir = cache_dir,)+ if profile is not None:+ # On the base module: the objectives read these attributes after unwrapping PEFT / DDP.+ profile.prepare_model(unwrap_diffusion_model(model), tokenizer)return model, tokenizer@staticmethod@@ -341,10 +473,16 @@ class FastDiffusionModel:**kwargs,):"""Attach a PEFT LoRA to the diffusion backbone (attention + dense MLP). No fused kernels."""+ if getattr(model, "_unsloth_full_finetuning", False):+ print("Unsloth: Full finetuning is enabled, so .get_peft_model has no effect")+ return modelfrom peft import LoraConfig, get_peft_model as peft_get_peft_model+ profile = resolve_diffusion_profile(getattr(model, "config", None))if target_modules is None:- target_modules = DIFFUSION_LORA_TARGETS+ target_modules = (+ profile.lora_target_modules if profile is not None else DIFFUSION_LORA_TARGETS+ )# use_dora, and any other LoraConfig kwarg outside this allowlist, is silently dropped: Unsloth does# not reach this path today, so it is untested on diffusion models.@@ -358,9 +496,18 @@ class FastDiffusionModel:**{k: v for k, v in kwargs.items() if k in ("modules_to_save", "init_lora_weights")},)# Exclude the vision tower's custom (non-Linear) modules that share suffix names.- exclude = kwargs.get("exclude_modules", DIFFUSION_LORA_EXCLUDE)+ default_exclude = None+ if profile is not None and profile.lora_exclude_modules is not None:+ default_exclude = profile.lora_exclude_modules+ elif any(re.match(DIFFUSION_LORA_EXCLUDE, name) for name, _ in model.named_modules()):+ default_exclude = DIFFUSION_LORA_EXCLUDE+ exclude = kwargs.get("exclude_modules", default_exclude)try:- lora_config = LoraConfig(exclude_modules = exclude, **lora_kwargs)+ lora_config = (+ LoraConfig(exclude_modules = exclude, **lora_kwargs)+ if exclude is not None+ else LoraConfig(**lora_kwargs)+ )except TypeError:# Older PEFT without exclude_modules: scope the target to the text decoder by regex.lora_kwargs["target_modules"] = (@@ -373,6 +520,7 @@ class FastDiffusionModel:model.enable_input_require_grads()model = peft_get_peft_model(model, lora_config)+ model = _refuse_tied_lora_merge(model)model._unsloth_slow_diffusion = Truemodel._unsloth_gradient_checkpointing = use_gradient_checkpointingtry:diff --git a/unsloth/models/diffusion_block.py b/unsloth/models/diffusion_block.pynew file mode 100644index 00000000000..29b5df60360--- /dev/null+++ b/unsloth/models/diffusion_block.py@@ -0,0 +1,489 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Block-diffusion families (LLaDA2.x, SDAR).++Training follows BD3-LM (https://arxiv.org/abs/2503.09573): each row is fed as [x_t ; x_0] with position ids+restarting at the clean copy. A noisy token sees its own noisy block and the clean blocks before it; clean tokens+see clean blocks causally. The loss reads the noisy half at the same position (no shift).+"""++import contextlib+import functools+import importlib.machinery+import inspect+import sys+import types++import torch+import torch.nn.functional as F++from .diffusion_profiles import (+ DiffusionProfile,+ register_diffusion_profile,+ response_mask,+ unwrap_diffusion_model,+)++__all__ = [+ "BlockDiffusionProfile",+ "block_diffusion_attention_mask",+ "LLADA2_PROFILE",+ "SDAR_PROFILE",+]+++def block_diffusion_attention_mask(+ length,+ block_size,+ device = None,+):+ """Bool [2L, 2L], True = attend. Rows/cols 0..L-1 are x_t, L..2L-1 are x_0."""+ idx = torch.arange(2 * length, device = device)+ q, kv = idx[:, None], idx[None, :]+ q_clean, kv_clean = q >= length, kv >= length+ q_block = torch.where(q_clean, q - length, q) // block_size+ kv_block = torch.where(kv_clean, kv - length, kv) // block_size+ same_noisy_block = (q_block == kv_block) & (q_clean == kv_clean)+ noisy_to_past_clean = (q_block > kv_block) & kv_clean & ~q_clean+ clean_causal = (q_block >= kv_block) & kv_clean & q_clean+ return same_noisy_block | noisy_to_past_clean | clean_causal+++_unwrap = unwrap_diffusion_model+++def _is_distributed_wrapper(model):+ return type(model).__name__ in (+ "DistributedDataParallel",+ "FullyShardedDataParallel",+ "DeepSpeedEngine",+ )+++class BlockDiffusionProfile(DiffusionProfile):+ # defaults keys: diffusion_block_size, noise_low, noise_high (mask probability range), time_weighting+ # ("none" | "inverse_t"), normalize ("masked" | "supervised"), eos_fill, key_padding, ensure_masked,+ # mask_token_id, mask_token_names, mask_format ("additive" | "bool").++ def prepare_model(self, model, tokenizer):+ mask_id = self._mask_token_from_tokenizer(tokenizer)+ if mask_id is not None:+ model._unsloth_diffusion_mask_token_id = mask_id+ eos = None+ if tokenizer is not None:+ tok = getattr(tokenizer, "tokenizer", tokenizer)+ eos = tok.eos_token_id if tok.eos_token_id is not None else tok.pad_token_id+ if eos is not None:+ model._unsloth_diffusion_eos_token_id = eos+ return model++ def _mask_token_from_tokenizer(self, tokenizer):+ if tokenizer is None:+ return None+ tok = getattr(tokenizer, "tokenizer", tokenizer)+ if getattr(tok, "mask_token_id", None) is not None:+ return tok.mask_token_id+ unk = getattr(tok, "unk_token_id", None)+ for name in self.defaults.get("mask_token_names", ()):+ token_id = tok.convert_tokens_to_ids(name)+ if isinstance(token_id, int) and token_id != unk:+ return token_id+ return None++ def mask_token_id(self, model, args):+ base = _unwrap(model)+ for value in (+ getattr(args, "diffusion_mask_token_id", None),+ getattr(base, "_unsloth_diffusion_mask_token_id", None),+ getattr(getattr(base, "config", None), "mask_token_id", None),+ self.defaults.get("mask_token_id"),+ ):+ if value is not None:+ return int(value)+ raise ValueError(+ f"Unsloth: no [MASK] token id for {self.name}; set DiffusionConfig.diffusion_mask_token_id."+ )++ def eos_token_id(self, model):+ base = _unwrap(model)+ config = getattr(base, "config", None)+ for value in (+ getattr(base, "_unsloth_diffusion_eos_token_id", None),+ getattr(config, "eos_token_id", None),+ getattr(config, "pad_token_id", None),+ ):+ if isinstance(value, (list, tuple)):+ value = value[0] if value else None+ if value is not None:+ return int(value)+ raise ValueError(f"Unsloth: no EOS token id for {self.name}.")++ def block_size(self, model, args):+ value = getattr(args, "diffusion_block_size", None)+ if value is None:+ value = getattr(getattr(_unwrap(model), "config", None), "block_size", None)+ if value is None:+ value = self.defaults["diffusion_block_size"]+ return int(value)++ def build_batch(self, model, inputs, args):+ """-> clean ids [B, L], maskable [B, L], valid [B, L] (attended positions)."""+ input_ids = inputs["input_ids"]+ attention_mask = inputs.get("attention_mask")+ valid = (+ torch.ones_like(input_ids, dtype = torch.bool)+ if attention_mask is None+ else attention_mask.bool()+ )+ maskable = response_mask(inputs)+ if not self.defaults.get("eos_fill", False):+ return input_ids, maskable, valid+ if (valid[:, 1:] & ~valid[:, :-1]).any():+ raise ValueError("Unsloth: block-diffusion training expects right padding.")+ # Turns end with a terminator, not EOS, so stopping is learnt only from a supervised EOS tail (as the+ # reference's max-length EOS padding); pad to the next block past the longest row.+ block = self.block_size(model, args)+ batch, length = input_ids.shape+ new_length = -(-(length + 1) // block) * block+ pad = new_length - length+ eos = self.eos_token_id(model)+ input_ids = F.pad(input_ids, (0, pad), value = eos)+ maskable = F.pad(maskable, (0, pad), value = False)+ valid = F.pad(valid, (0, pad), value = False)+ real_length = valid.sum(dim = 1, keepdim = True)+ positions = torch.arange(new_length, device = input_ids.device)+ # A row clipped at max_length has no real end, so it gets no EOS tail (as for DiffusionGemma).+ max_length = getattr(args, "max_length", None)+ truncated = torch.zeros_like(real_length, dtype = torch.bool)+ if max_length is not None:+ last = torch.where(maskable, positions[None, :], -1).amax(dim = 1, keepdim = True)+ truncated = (real_length >= max_length) & (last == real_length - 1)+ fill = (positions[None, :] >= real_length) & maskable.any(dim = 1, keepdim = True)+ input_ids = torch.where(fill, eos, input_ids)+ # Clipped rows still see an EOS tail rather than pad ids, but it is never a target.+ return input_ids, maskable | (fill & ~truncated), valid | fill++ def noise_range(self, args):+ eps = getattr(args, "diffusion_eps", None)+ if eps is not None:+ return float(eps), 1.0 - float(eps)+ return float(self.defaults["noise_low"]), float(self.defaults["noise_high"])++ def sample_noise(self, clean, maskable, mask_token_id, args):+ """-> noisy ids, masked bool [B, L], p [B] mask probability per row."""+ batch, length = clean.shape+ low, high = self.noise_range(args)+ p = low + (high - low) * torch.rand(batch, device = clean.device)+ masked = (torch.rand(batch, length, device = clean.device) < p[:, None]) & maskable+ if self.defaults.get("ensure_masked", False):+ empty = maskable.any(dim = 1) & ~masked.any(dim = 1)+ if empty.any():+ scores = torch.rand(batch, length, device = clean.device).masked_fill(~maskable, -1.0)+ pick = F.one_hot(scores.argmax(dim = 1), length).bool()+ masked = masked | (pick & empty[:, None])+ return torch.where(masked, mask_token_id, clean), masked, p++ def attention_mask(self, length, block_size, valid, dtype, device):+ mask = block_diffusion_attention_mask(length, block_size, device)[None, None]+ batch = valid.shape[0]+ if self.defaults.get("key_padding", False) and not bool(valid.all()):+ keys = torch.cat([valid, valid], dim = 1)[:, None, None, :]+ # Padding is its own segment in the reference; a padding query keeps its diagonal so no row is empty.+ mask = (mask & keys) | torch.eye(2 * length, dtype = torch.bool, device = device)[+ None, None+ ]+ mask = mask.expand(batch, 1, 2 * length, 2 * length)+ if self.defaults.get("mask_format", "additive") == "bool":+ return mask+ additive = torch.zeros(mask.shape, dtype = dtype, device = device)+ return additive.masked_fill(~mask, float("-inf"))++ def noisy_hidden_and_head(self, model, noisy, clean, valid, block_size):+ """Hidden states of the noisy half [B, L, H] and the output head."""+ batch, length = clean.shape+ base = _unwrap(model)+ head = base.get_output_embeddings()+ concat = torch.cat([noisy, clean], dim = 1)+ positions = torch.arange(length, device = clean.device)+ position_ids = torch.cat([positions, positions])[None].expand(batch, -1)+ dtype = head.weight.dtype if head.weight.dtype.is_floating_point else torch.float32+ mask = self.attention_mask(length, block_size, valid, dtype, clean.device)+ if _is_distributed_wrapper(model):+ # DDP / FSDP / DeepSpeed must see their own forward; this path materialises logits for both halves.+ out = model(+ input_ids = concat, attention_mask = mask, position_ids = position_ids, use_cache = False+ )+ return out.logits[:, :length], None+ out = base.get_decoder()(+ input_ids = concat,+ attention_mask = mask,+ position_ids = position_ids,+ use_cache = False,+ return_dict = True,+ )+ return out[0][:, :length], head++ def loss_from_noise(self, model, clean, noisy, masked, maskable, p, valid, args):+ block_size = self.block_size(model, args)+ hidden, head = self.noisy_hidden_and_head(model, noisy, clean, valid, block_size)+ if head is None:+ logits = hidden[masked].float()+ else:+ # Project only the masked positions: the vocab-sized logits are the memory peak otherwise.+ logits = head(hidden[masked]).float()+ nll = F.cross_entropy(logits, clean[masked], reduction = "none")+ weighting = self.option(args, "diffusion_time_weighting", "none")+ if weighting not in ("inverse_t", "none"):+ raise ValueError(+ f"Unsloth: {self.name} does not support diffusion_time_weighting={weighting!r}."+ )+ if weighting == "inverse_t":+ nll = nll / p[:, None].expand_as(masked)[masked].float()+ if self.defaults.get("normalize", "masked") == "supervised":+ denominator = maskable.sum()+ else:+ denominator = masked.sum()+ loss = nll.sum() / denominator.clamp_min(1)+ if not masked.any():+ loss = loss + hidden.sum() * 0.0+ return loss++ def compute_loss(+ self,+ model,+ inputs,+ args,+ num_items_in_batch = None,+ ):+ clean, maskable, valid = self.build_batch(model, inputs, args)+ mask_token_id = self.mask_token_id(model, args)+ noisy, masked, p = self.sample_noise(clean, maskable, mask_token_id, args)+ loss = self.loss_from_noise(model, clean, noisy, masked, maskable, p, valid, args)+ metrics = {+ "mask_ratio": (masked.sum() / maskable.sum().clamp_min(1)).item(),+ "masked_tokens": masked.sum().item(),+ }+ return loss, None, metrics+++class _LLaDA2Profile(BlockDiffusionProfile):+ pass+++# Reference: inclusionAI/dFactory tasks/train_llada2_bd.py and configs/sft/llada2_mini_bd_sft.yaml.+LLADA2_PROFILE = register_diffusion_profile(+ _LLaDA2Profile(+ name = "llada2",+ model_types = ("llada2_moe",),+ architectures = ("LLaDA2MoeModelLM",),+ noise = "mask",+ requires_remote_code = True,+ # Attention, the dense first-layer MLP and the shared expert; routed experts and the router stay frozen.+ lora_target_modules = (+ r".*\.layers\.\d+\.(attention\.(query_key_value|dense)"+ r"|mlp\.(shared_experts\.)?(gate_proj|up_proj|down_proj))"+ ),+ defaults = dict(+ diffusion_block_size = 32,+ noise_low = 0.3,+ noise_high = 0.8,+ diffusion_time_weighting = "none",+ normalize = "masked",+ eos_fill = True,+ key_padding = False,+ ensure_masked = False,+ mask_token_id = 156895,+ mask_token_names = ("<|mask|>",),+ mask_format = "additive",+ attn_implementation = "sdpa",+ ),+ )+)+++def _rms_norm_fn(+ x,+ weight,+ bias = None,+ eps = 1e-6,+ **kwargs,+):+ dtype = x.dtype+ x = x.float()+ x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim = True) + eps)+ out = weight * x.to(dtype) if weight is not None else x.to(dtype)+ return out + bias if bias is not None else out+++def _flash_attn_func(+ q,+ k,+ v,+ dropout_p = 0.0,+ softmax_scale = None,+ causal = False,+ **kwargs,+):+ # [B, S, H, D] layout like flash-attn; SDPA handles GQA via enable_gqa.+ out = F.scaled_dot_product_attention(+ q.transpose(1, 2),+ k.transpose(1, 2),+ v.transpose(1, 2),+ is_causal = causal,+ scale = softmax_scale,+ enable_gqa = k.shape[2] != q.shape[2],+ )+ return out.transpose(1, 2)+++def _module(name, **attributes):+ module = types.ModuleType(name)+ module.__spec__ = importlib.machinery.ModuleSpec(name, None)+ module.__path__ = []+ module.__dict__.update(attributes)+ return module+++def _sdar_remote_code_imports():+ """SDAR's modeling file imports flash-attn's RMSNorm and 4.x `LossKwargs` at module level. Provide+ torch stand-ins only while it imports when flash-attn is absent; the module keeps its references."""+ import importlib.util+ import transformers.utils as transformers_utils++ if not hasattr(transformers_utils, "LossKwargs"):+ from typing import Optional, TypedDict+ class LossKwargs(TypedDict, total = False):+ num_items_in_batch: Optional[torch.Tensor]++ transformers_utils.LossKwargs = LossKwargs+ try:+ has_flash = importlib.util.find_spec("flash_attn") is not None+ except (ImportError, ValueError):+ has_flash = False+ if has_flash:+ yield+ return+ names = (+ "flash_attn",+ "flash_attn.ops",+ "flash_attn.ops.triton",+ "flash_attn.ops.triton.layer_norm",+ )+ stubs = {+ "flash_attn": _module("flash_attn", flash_attn_func = _flash_attn_func),+ "flash_attn.ops": _module("flash_attn.ops"),+ "flash_attn.ops.triton": _module("flash_attn.ops.triton"),+ "flash_attn.ops.triton.layer_norm": _module(+ "flash_attn.ops.triton.layer_norm", rms_norm_fn = _rms_norm_fn+ ),+ }+ from transformers import dynamic_module_utils++ original_check_imports = dynamic_module_utils.check_imports++ def check_imports(filename):+ try:+ return original_check_imports(filename)+ except ImportError as error:+ if "flash_attn" not in str(error):+ raise+ return dynamic_module_utils.get_relative_imports(filename)++ saved = {name: sys.modules.get(name) for name in names}+ sys.modules.update(stubs)+ dynamic_module_utils.check_imports = check_imports+ try:+ yield+ finally:+ dynamic_module_utils.check_imports = original_check_imports+ for name, module in saved.items():+ if module is None:+ sys.modules.pop(name, None)+ else:+ sys.modules[name] = module+++def _accept_cache_objects(model):+ """SDAR's attention reads `past_key_value[layer]` (4.x tuple API); transformers 5 caches have no+ __getitem__. Hand it a read view while `update` / `len` still reach the real cache."""+ from .remote_code_shims import _LegacyCacheView, _needs_legacy_view++ wrapped = 0+ for module in model.modules():+ cls = type(module)+ forward = cls.__dict__.get("forward")+ if forward is None or getattr(forward, "_unsloth_cache_view", False):+ continue+ try:+ source = inspect.getsource(forward)+ except (OSError, TypeError):+ continue+ if (+ "past_key_value[" not in source+ or "past_key_value" not in inspect.signature(forward).parameters+ ):+ continue++ def make(original):+ @functools.wraps(original)+ def forward(self, *args, **kwargs):+ cache = kwargs.get("past_key_value")+ if _needs_legacy_view(cache):+ kwargs["past_key_value"] = _LegacyCacheView(cache)+ return original(self, *args, **kwargs)++ forward._unsloth_cache_view = True+ return forward++ cls.forward = make(forward)+ wrapped += 1+ return wrapped+++class _SDARProfile(BlockDiffusionProfile):+ def prepare_model(self, model, tokenizer):+ _accept_cache_objects(model)+ # The released config fuses the head into its training loss, so a train-mode forward returns no logits.+ config = getattr(model, "config", None)+ if getattr(config, "fuse_cross_entropy", False):+ config.fuse_cross_entropy = False+ return super().prepare_model(model, tokenizer)++ def model_class(self, config, trust_remote_code, **hub_kwargs):+ from transformers import AutoModelForCausalLM+ class _Loader:+ @staticmethod+ def from_pretrained(*args, **kwargs):+ with _sdar_remote_code_imports():+ return AutoModelForCausalLM.from_pretrained(*args, **kwargs)++ return _Loader+++# Reference: JetAstra/SDAR training/model/SDAR-8B-Chat/modeling_sdar.py (forward_add_noise_packed,+# block_attn_mask, FusedLinearDiffusionCrossEntropyLoss); block length 4 from the model card.+SDAR_PROFILE = register_diffusion_profile(+ _SDARProfile(+ name = "sdar",+ model_types = ("sdar",),+ architectures = ("SDARForCausalLM",),+ noise = "mask",+ requires_remote_code = True,+ defaults = dict(+ diffusion_block_size = 4,+ noise_low = 1e-3,+ noise_high = 1.0,+ diffusion_time_weighting = "inverse_t",+ normalize = "supervised",+ eos_fill = False,+ key_padding = True,+ ensure_masked = True,+ mask_token_id = 151669,+ mask_token_names = ("<MASK>", "<|MASK|>"),+ mask_format = "bool",+ attn_implementation = "sdpa",+ ),+ )+)diff --git a/unsloth/models/diffusion_gemma_objective.py b/unsloth/models/diffusion_gemma_objective.pynew file mode 100644index 00000000000..bff1c8fb1a5--- /dev/null+++ b/unsloth/models/diffusion_gemma_objective.py@@ -0,0 +1,213 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""DiffusionGemma block-diffusion SFT objective, matching the reference fine-tune of the released checkpoint+(the recipe in TRL's examples/sft_diffusion_gemma/sft_diffusion_gemma.py)."""++import contextlib++import torch+import torch.nn.functional as F++from .diffusion_profiles import DiffusionProfile, register_diffusion_profile, unwrap_diffusion_model++__all__ = ["DiffusionGemmaProfile"]++try:+ from trl.trainer.utils import maybe_gather_lm_head_ctx as _gather_lm_head+except ImportError: # trl < 1.x: no ZeRO-3 aware helper, plain access is fine off ZeRO-3++ def _gather_lm_head(*params):+ return contextlib.nullcontext()+++def _find_attr(model, name):+ seen = set()+ stack = [model]+ while stack:+ m = stack.pop()+ if m is None or id(m) in seen:+ continue+ seen.add(id(m))+ value = m.__dict__.get(name) if hasattr(m, "__dict__") else None+ if value is not None:+ return value+ stack += [+ getattr(m, "base_model", None),+ getattr(m, "model", None),+ getattr(m, "module", None),+ ]+ return None+++def _eos_token_id(model):+ eos = _find_attr(model, "_unsloth_diffusion_eos_token_id")+ if eos is None:+ config = model.config+ eos = getattr(getattr(config, "text_config", config), "eos_token_id", None)+ if eos is None:+ eos = getattr(getattr(model, "generation_config", None), "eos_token_id", None)+ if isinstance(eos, (list, tuple)):+ eos = eos[0]+ if eos is None:+ raise ValueError(+ "Unsloth: DiffusionGemma training needs an eos_token_id to fill the canvas."+ )+ return int(eos)+++class DiffusionGemmaProfile(DiffusionProfile):+ def prepare_model(self, model, tokenizer):+ tok = getattr(tokenizer, "tokenizer", tokenizer)+ eos = getattr(tok, "eos_token_id", None)+ if eos is not None:+ model._unsloth_diffusion_eos_token_id = int(eos)+ return model++ def compute_loss(+ self,+ model,+ inputs,+ args,+ num_items_in_batch = None,+ ):+ eps = self.option(args, "diffusion_eps")+ sc_p = self.option(args, "diffusion_self_conditioning_p")+ ar_weight = self.option(args, "diffusion_encoder_ar_weight")+ prediction_type = self.option(args, "diffusion_prediction_type")+ if prediction_type not in ("mean", "mean_loo"):+ raise ValueError(+ f"Unsloth: diffusion_prediction_type must be 'mean' or 'mean_loo', got {prediction_type!r}."+ )++ base = unwrap_diffusion_model(model)+ config = base.config+ text_config = config.text_config+ block_size = config.canvas_length+ vocab_size = text_config.vocab_size+ softcap = text_config.final_logit_softcapping+ eos_token_id = _eos_token_id(base)+ max_length = getattr(args, "max_length", None)++ input_ids = inputs["input_ids"]+ attention_mask = inputs.get("attention_mask")+ if attention_mask is None:+ attention_mask = torch.ones_like(input_ids)+ labels = inputs.get("labels")+ if labels is None:+ labels = input_ids.masked_fill(~attention_mask.bool(), -100)+ device = input_ids.device+ batch_size, seq_len = input_ids.shape++ # The canvas covers the last supervised span only: the denoising loss needs one contiguous response.+ supervised = labels != -100+ positions = torch.arange(seq_len, device = device)+ span_starts = supervised & ~F.pad(supervised, (1, 0))[:, :-1]+ span_end = torch.where(supervised, positions, -1).amax(dim = 1)+ prefix_len = torch.where(span_starts, positions, -1).amax(dim = 1).clamp(min = 0)+ response_len = (span_end - prefix_len + 1) * (span_end >= 0)++ # A clipped response has no real end, so its EOS fill must not teach the model to stop there.+ if max_length is None:+ truncated = torch.zeros(batch_size, dtype = torch.bool, device = device)+ else:+ real_len = attention_mask.sum(dim = 1)+ truncated = (real_len >= max_length) & (span_end == real_len - 1)++ # One response block per example; the decoder sees the prompt plus clean blocks before it, nothing later.+ num_blocks = (response_len - 1).clamp(min = 0) // block_size + 1+ block_idx = (torch.rand(batch_size, device = device) * num_blocks).long()+ encoder_len = prefix_len + block_idx * block_size++ offsets = torch.arange(block_size, device = device)+ abs_idx = (encoder_len[:, None] + offsets).clamp(max = seq_len - 1)+ in_response = offsets < (response_len - block_idx * block_size)[:, None]+ canvas_target = torch.where(in_response, input_ids.gather(1, abs_idx), eos_token_id)++ t = eps + (1 - 2 * eps) * torch.rand(batch_size, 1, device = device)+ corrupt = torch.rand(batch_size, block_size, device = device) < t+ random_tokens = torch.randint(vocab_size, (batch_size, block_size), device = device)+ canvas_ids = torch.where(corrupt, random_tokens, canvas_target)++ cache_mask = positions < encoder_len[:, None]+ canvas_mask = torch.ones(batch_size, block_size, dtype = torch.bool, device = device)+ model_kwargs = {+ "input_ids": input_ids,+ "attention_mask": attention_mask,+ "decoder_input_ids": canvas_ids,+ "decoder_attention_mask": torch.cat([cache_mask, canvas_mask], dim = 1),+ "decoder_position_ids": encoder_len[:, None] + offsets,+ }+ # Two-pass self-conditioning: a no-grad first pass supplies the logits, kept per example with prob p.+ if sc_p > 0:+ with torch.no_grad():+ model_kwargs["self_conditioning_logits"] = model(**model_kwargs).logits+ model_kwargs["self_conditioning_mask"] = torch.rand(batch_size, device = device) < sc_p+ outputs = model(**model_kwargs)++ # Flat CE over the whole canvas, corrupted and clean alike: the uniform kernel's ELBO has no 1/t weight.+ diffusion_target = canvas_target.masked_fill(truncated[:, None] & ~in_response, -100)+ # A row with no supervised token (completion truncated away) would otherwise train an all-EOS canvas.+ diffusion_target = diffusion_target.masked_fill((span_end < 0)[:, None], -100)+ diffusion_logits = outputs.logits+ if prediction_type == "mean_loo":+ # The softmax parameterises the leave-one-out posterior (arXiv 2605.22765): convert in fp32, ~log(K) nats.+ alpha = (1 - t).float()+ bump = torch.log1p(diffusion_logits.shape[-1] * alpha / (1 - alpha))+ diffusion_logits = diffusion_logits.float().scatter_add(+ 2, canvas_ids.unsqueeze(-1), bump[:, :, None].expand(-1, block_size, 1)+ )+ if (diffusion_target != -100).any():+ diffusion_loss = F.cross_entropy(+ diffusion_logits.flatten(0, 1), diffusion_target.flatten(), ignore_index = -100+ )+ else:+ diffusion_loss = outputs.logits.sum() * 0.0++ lm_head = base.get_output_embeddings()+ hidden_states = outputs.encoder_last_hidden_state.to(lm_head.weight.dtype)+ with _gather_lm_head(lm_head.weight, lm_head.bias):+ encoder_logits = hidden_states @ lm_head.weight.t()+ if lm_head.bias is not None:+ encoder_logits = encoder_logits + lm_head.bias+ encoder_logits = encoder_logits.float()+ encoder_logits = torch.tanh(encoder_logits / softcap) * softcap+ ar_mask = attention_mask[:, :-1].bool() & attention_mask[:, 1:].bool()+ ar_targets = torch.where(ar_mask, input_ids[:, 1:], -100)+ ar_loss = F.cross_entropy(+ encoder_logits[:, :-1].flatten(0, 1), ar_targets.flatten(), ignore_index = -100+ )++ loss = diffusion_loss + ar_weight * ar_loss+ metrics = {+ "diffusion_loss": diffusion_loss.detach().item(),+ "encoder_ar_loss": ar_loss.detach().item(),+ "mean_t": t.mean().item(),+ }+ return loss, outputs, metrics+++DIFFUSION_GEMMA_PROFILE = register_diffusion_profile(+ DiffusionGemmaProfile(+ name = "diffusion_gemma",+ model_types = ("diffusion_gemma", "diffusion_gemma4"),+ architectures = (+ "DiffusionGemmaForBlockDiffusion",+ "DiffusionGemma4ModelForBlockDiffusion",+ "DiffusionGemma4ForBlockDiffusion",+ ),+ noise = "uniform",+ # Encoder + decoder attention / dense MLP only; suffix targets would also wrap the self-conditioning block.+ lora_target_modules = (+ r".*model\.(encoder\.language_model|decoder)\.layers\.\d+\."+ r"(self_attn\.[qkvo]_proj|mlp\.(gate|up|down)_proj)"+ ),+ defaults = {+ "diffusion_eps": 1e-4,+ "diffusion_self_conditioning_p": 0.5,+ "diffusion_encoder_ar_weight": 1.0,+ "diffusion_prediction_type": "mean",+ "attn_implementation": "eager",+ },+ )+)diff --git a/unsloth/models/diffusion_masked.py b/unsloth/models/diffusion_masked.pynew file mode 100644index 00000000000..f1928b6a2f7--- /dev/null+++ b/unsloth/models/diffusion_masked.py@@ -0,0 +1,341 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Absorbing-mask diffusion LMs over one bidirectional sequence: LLaDA, LLaDA-MoE, Dream, Nemotron-Labs-Diffusion.++Each response token is replaced by [MASK] with probability p, the clean prompt stays visible, and the+loss is cross-entropy at the masked positions weighted by 1 / p. The families differ only in the knobs+below, each taken from that family's published training code.+"""++import functools+from dataclasses import dataclass++import torch+import torch.nn.functional as F++from .diffusion_profiles import (+ DiffusionProfile,+ register_diffusion_profile,+ response_mask,+ unwrap_diffusion_model,+)++__all__ = ["MaskedDiffusionProfile"]+++def _ensure_post_init(cls):+ """4.x remote code that never calls post_init() loads on transformers 5 only once it does."""+ if getattr(cls, "_unsloth_post_init_guard", False):+ return cls+ original_init = cls.__init__++ @functools.wraps(original_init)+ def __init__(self, *args, **kwargs):+ original_init(self, *args, **kwargs)+ if not hasattr(self, "all_tied_weights_keys") and hasattr(self, "post_init"):+ self.post_init()++ cls.__init__ = __init__+ cls._unsloth_post_init_guard = True+ return cls+++def _accept_new_validate_kwargs(cls):+ """Dream's GenerationConfig.validate(self, is_init) predates the keywords transformers 5 passes it."""+ import inspect+ import sys++ from transformers import GenerationConfig++ module = sys.modules.get(cls.__module__)+ for value in list(vars(module).values()) if module is not None else ():+ if (+ not (isinstance(value, type) and issubclass(value, GenerationConfig))+ or value is GenerationConfig+ ):+ continue+ validate = value.__dict__.get("validate")+ if validate is None or getattr(validate, "_unsloth_tolerant", False):+ continue+ parameters = inspect.signature(validate).parameters+ if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters.values()):+ continue+ accepted = set(parameters)++ @functools.wraps(validate)+ def tolerant(+ self,+ *args,+ _validate = validate,+ _accepted = accepted,+ **kwargs,+ ):+ return _validate(self, *args, **{k: v for k, v in kwargs.items() if k in _accepted})++ tolerant._unsloth_tolerant = True+ value.validate = tolerant+++def _default_diffusion_generate_config(model):+ """Dream's diffusion_generate() without a generation_config rebuilds one through+ GenerationConfig.from_model_config, which transformers 5 breaks on Dream-only fields (eps)."""+ generate = getattr(model, "diffusion_generate", None)+ if generate is None or getattr(generate, "_unsloth_default_config", False):+ return++ @functools.wraps(generate)+ def diffusion_generate(+ inputs = None,+ generation_config = None,+ **kwargs,+ ):+ if generation_config is None:+ import copy+ generation_config = copy.deepcopy(model.generation_config)+ # from_model_config used to fill these from the model config; Dream ships mask_token_id null.+ for name in ("mask_token_id", "eos_token_id", "pad_token_id", "bos_token_id"):+ if getattr(generation_config, name, None) is None:+ setattr(generation_config, name, getattr(model.config, name, None))+ return generate(inputs, generation_config = generation_config, **kwargs)++ diffusion_generate._unsloth_default_config = True+ model.diffusion_generate = diffusion_generate+++def _expose_mask_signatures():+ """unsloth_zoo wraps the transformers mask builders as (*args, **kwargs); remote code that filters its+ arguments by inspect.signature (Nemotron-Labs-Diffusion's causal prefill) then passes nothing."""+ import inspect++ try:+ import transformers.masking_utils as masking_utils+ except ImportError:+ return+ for name in ("create_causal_mask", "create_sliding_window_causal_mask"):+ function = getattr(masking_utils, name, None)+ if (+ function is None+ or "__signature__" in vars(function)+ or hasattr(function, "__wrapped__")+ ):+ continue+ for cell_name, cell in zip(function.__code__.co_freevars, function.__closure__ or ()):+ if cell_name == "signature" and isinstance(cell.cell_contents, inspect.Signature):+ function.__signature__ = cell.cell_contents+ break+++def _bridge_native_checkpointing(model):+ """LLaDA checkpoints through its own set_activation_checkpointing, not the transformers API."""+ inner = getattr(model, "model", None)+ if getattr(model, "supports_gradient_checkpointing", False) or not hasattr(+ inner, "set_activation_checkpointing"+ ):+ return++ def gradient_checkpointing_enable(gradient_checkpointing_kwargs = None, **kwargs):+ inner.set_activation_checkpointing("whole_layer")++ def gradient_checkpointing_disable():+ inner.set_activation_checkpointing(None)++ model.gradient_checkpointing_enable = gradient_checkpointing_enable+ model.gradient_checkpointing_disable = gradient_checkpointing_disable+ if hasattr(model, "enable_input_require_grads"):+ model.enable_input_require_grads = lambda: None+++@dataclass+class MaskedDiffusionProfile(DiffusionProfile):+ # Dream is adapted from an AR model: position i's logits predict token i + 1 (Dream src/trainer).+ shift_logits: bool = False+ # "answer_length": LLaDA GUIDELINES.md SFT, each example / its answer length, mean over the batch.+ # "masked_tokens": Dream / Nemotron, summed then divided by the number of masked tokens.+ loss_normalization: str = "answer_length"+ # LLaDA and Dream train on EOS-filled, attended padding so the model learns where answers stop.+ pad_with_eos: bool = True+ auto_class: str = "AutoModelForCausalLM"+ default_mask_token_id: int = None++ def __post_init__(self):+ self.noise = "mask"+ self.defaults = {+ "diffusion_eps": 1e-3,+ "diffusion_time_weighting": "inverse_t",+ **self.defaults,+ }++ def model_class(self, config, trust_remote_code, **hub_kwargs):+ from transformers.dynamic_module_utils import get_class_from_dynamic_module++ auto_map = getattr(config, "auto_map", None) or {}+ class_ref = (+ auto_map.get(self.auto_class)+ or auto_map.get("AutoModelForCausalLM")+ or auto_map.get("AutoModel")+ )+ if class_ref is None or not trust_remote_code:+ return super().model_class(config, trust_remote_code, **hub_kwargs)+ cls = get_class_from_dynamic_module(+ class_ref,+ config._name_or_path,+ revision = getattr(config, "_commit_hash", None),+ **hub_kwargs,+ )+ _accept_new_validate_kwargs(cls)+ return _ensure_post_init(cls)++ def prepare_model(self, model, tokenizer):+ config = model.config+ # transformers 5 dropped the PretrainedConfig.use_cache default that LLaDA's forward reads.+ if getattr(config, "use_cache", None) is None:+ config.use_cache = False+ mask_id = getattr(config, "mask_token_id", None)+ if mask_id is None and tokenizer is not None:+ mask_id = getattr(getattr(tokenizer, "tokenizer", tokenizer), "mask_token_id", None)+ if mask_id is None:+ mask_id = self.default_mask_token_id+ eos_id = getattr(config, "eos_token_id", None)+ if isinstance(eos_id, (list, tuple)):+ eos_id = eos_id[0] if eos_id else None+ if eos_id is None and tokenizer is not None:+ eos_id = getattr(getattr(tokenizer, "tokenizer", tokenizer), "eos_token_id", None)+ model._unsloth_mask_token_id = mask_id+ model._unsloth_eos_token_id = eos_id+ _bridge_native_checkpointing(model)+ _default_diffusion_generate_config(model)+ _expose_mask_signatures()+ return model++ def _token_ids(self, model, args):+ model = unwrap_diffusion_model(model)+ mask_id = self.option(args, "diffusion_mask_token_id")+ if mask_id is None:+ mask_id = getattr(model, "_unsloth_mask_token_id", None)+ if mask_id is None:+ mask_id = getattr(model.config, "mask_token_id", None) or self.default_mask_token_id+ if mask_id is None:+ raise ValueError(+ f"Unsloth: {self.name} needs a [MASK] token id. Set DiffusionConfig(diffusion_mask_token_id = ...)."+ )+ eos_id = getattr(model, "_unsloth_eos_token_id", None)+ if eos_id is None:+ eos_id = getattr(model.config, "eos_token_id", None)+ if isinstance(eos_id, (list, tuple)):+ eos_id = eos_id[0]+ return mask_id, eos_id++ def compute_loss(+ self,+ model,+ inputs,+ args,+ num_items_in_batch = None,+ ):+ input_ids = inputs["input_ids"]+ attention_mask = inputs.get("attention_mask")+ batch, length = input_ids.shape+ device = input_ids.device+ mask_id, eos_id = self._token_ids(model, args)+ eps = float(self.option(args, "diffusion_eps"))+ weighting = self.option(args, "diffusion_time_weighting")++ maskable = response_mask(inputs)+ forward_mask = attention_mask+ if self.pad_with_eos and attention_mask is not None and eos_id is not None:+ padding = ~attention_mask.bool()+ input_ids = input_ids.masked_fill(padding, eos_id)+ if inputs.get("labels") is not None:+ # Only rows with a response get an EOS tail; a fully truncated row contributes nothing.+ maskable = maskable | (padding & maskable.any(dim = 1, keepdim = True))+ forward_mask = None++ # Same draw order as the references (t, then the per-token draw) so a seed reproduces their batch.+ t = torch.rand(batch, device = device)+ p_mask = ((1 - eps) * t + eps)[:, None].expand(batch, length)+ masked = (torch.rand((batch, length), device = device) < p_mask) & maskable+ noisy = torch.where(masked, mask_id, input_ids)++ model_inputs = {"input_ids": noisy}+ if forward_mask is not None and not bool(forward_mask.all()):+ model_inputs["attention_mask"] = forward_mask+ logits = model(**model_inputs).logits+ if self.shift_logits:+ logits = torch.cat([logits[:, :1], logits[:, :-1]], dim = 1)++ if not bool(masked.any()):+ zero = logits.sum() * 0.0+ return zero, None, {"masked_fraction": 0.0}++ token_loss = F.cross_entropy(logits[masked].float(), input_ids[masked], reduction = "none")+ if weighting == "inverse_t":+ token_loss = token_loss / p_mask[masked]+ elif weighting not in ("none", None):+ raise ValueError(+ f"Unsloth: {self.name} does not support diffusion_time_weighting={weighting!r}."+ )++ if self.loss_normalization == "answer_length":+ answer_length = maskable.sum(dim = 1, keepdim = True).clamp_min(1).expand(batch, length)+ loss = (token_loss / answer_length[masked]).sum() / batch+ else:+ loss = token_loss.sum() / masked.sum()++ metrics = {+ "masked_fraction": (masked.sum() / maskable.sum().clamp_min(1)).item(),+ "masked_ce": F.cross_entropy(logits[masked].float(), input_ids[masked]).item(),+ }+ return loss, None, metrics+++_ATTENTION_AND_MLP = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]++# LLaDA-MoE shares model_type "llada" with the dense model, so it is matched first by architecture.+register_diffusion_profile(+ MaskedDiffusionProfile(+ name = "llada_moe",+ architectures = ("LLaDAMoEModel", "LLaDAMoEModelLM"),+ requires_remote_code = True,+ # Every layer is MoE: adapt attention only, never the router or the experts.+ lora_target_modules = r".*\.layers\.\d+\.self_attn\.(q_proj|k_proj|v_proj|o_proj)",+ default_mask_token_id = 156895, # model card generate(mask_id = 156895); absent from config.json+ )+)+register_diffusion_profile(+ MaskedDiffusionProfile(+ name = "llada",+ model_types = ("llada",),+ architectures = ("LLaDAModelLM",),+ requires_remote_code = True,+ # OLMo-style names; anchored to blocks so the output head (transformer.ff_out) stays frozen.+ lora_target_modules = r".*\.blocks\.\d+\.(q_proj|k_proj|v_proj|attn_out|ff_proj|up_proj|ff_out)",+ quant_skip_modules = ["transformer.ff_out", "transformer.wte", "lm_head"],+ default_mask_token_id = 126336,+ )+)+register_diffusion_profile(+ MaskedDiffusionProfile(+ name = "dream",+ model_types = ("Dream", "dream"),+ architectures = ("DreamModel",),+ requires_remote_code = True,+ lora_target_modules = list(_ATTENTION_AND_MLP),+ shift_logits = True,+ loss_normalization = "masked_tokens",+ auto_class = "AutoModel",+ # Dream q_sample draws t ~ U(0, 1) with no floor and weights by 1 / t (time_reweighting "original").+ defaults = {"diffusion_eps": 0.0},+ )+)+register_diffusion_profile(+ MaskedDiffusionProfile(+ name = "nemotron_labs_diffusion",+ model_types = ("nemotron_labs_diffusion",),+ architectures = ("NemotronLabsDiffusionModel",),+ requires_remote_code = True,+ lora_target_modules = list(_ATTENTION_AND_MLP),+ loss_normalization = "masked_tokens",+ auto_class = "AutoModel",+ )+)diff --git a/unsloth/models/diffusion_profiles.py b/unsloth/models/diffusion_profiles.pynew file mode 100644index 00000000000..0667da7b513--- /dev/null+++ b/unsloth/models/diffusion_profiles.py@@ -0,0 +1,209 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Per-family training recipes for text-diffusion language models.++A profile names the checkpoints it handles (``model_types`` / ``architectures``), how they load, which+LoRA targets are safe, and the training objective. ``DiffusionTrainer`` calls ``profile.compute_loss``+on ordinary SFT batches (``input_ids``, ``attention_mask``, ``labels`` with -100 outside the response).+"""++from dataclasses import dataclass, field+from typing import Optional++import torch++__all__ = [+ "DiffusionProfile",+ "register_diffusion_profile",+ "resolve_diffusion_profile",+ "diffusion_profiles",+ "diffusion_model_types",+ "response_mask",+ "sample_noise_level",+ "restore_rotary_buffers",+ "unwrap_diffusion_model",+]+++@dataclass+class DiffusionProfile:+ name: str+ model_types: tuple = ()+ architectures: tuple = ()+ # "mask" = absorbing [MASK] noise (LLaDA, Dream, Nemotron); "uniform" = random-token noise (DiffusionGemma).+ noise: str = "mask"+ requires_remote_code: bool = False+ lora_target_modules: object = field(+ default_factory = lambda: [+ "q_proj",+ "k_proj",+ "v_proj",+ "o_proj",+ "gate_proj",+ "up_proj",+ "down_proj",+ ]+ )+ lora_exclude_modules: Optional[str] = None+ # bitsandbytes keeps these in full precision; None = the shared default list.+ quant_skip_modules: Optional[list] = None+ # Profile-specific defaults a DiffusionConfig field left at None falls back to.+ defaults: dict = field(default_factory = dict)++ def matches(self, config) -> bool:+ model_type = getattr(config, "model_type", None)+ if model_type in self.model_types:+ return True+ archs = getattr(config, "architectures", None) or ()+ return any(arch in self.architectures for arch in archs)++ def model_class(self, config, trust_remote_code, **hub_kwargs):+ """The class ``from_pretrained`` is called on. Remote-code families use the auto class."""+ from transformers import AutoModel, AutoModelForCausalLM++ auto_map = getattr(config, "auto_map", None) or {}+ if "AutoModelForCausalLM" in auto_map or not auto_map:+ return AutoModelForCausalLM+ return AutoModel++ def prepare_model(self, model, tokenizer):+ """Post-load fixes (remote-code compat, mask-token bookkeeping). Must be idempotent."""+ return model++ def option(+ self,+ args,+ name,+ default = None,+ ):+ value = getattr(args, name, None) if args is not None else None+ if value is None:+ value = self.defaults.get(name, default)+ return value++ def compute_loss(+ self,+ model,+ inputs,+ args,+ num_items_in_batch = None,+ ):+ """Returns (loss, outputs_or_None, metrics: dict[str, float])."""+ raise NotImplementedError(+ f"Unsloth: diffusion profile {self.name} has no training objective."+ )+++_PROFILES = []+++def register_diffusion_profile(profile):+ for i, existing in enumerate(_PROFILES):+ if existing.name == profile.name:+ _PROFILES[i] = profile+ return profile+ _PROFILES.append(profile)+ return profile+++def diffusion_profiles():+ _load_builtin_profiles()+ return tuple(_PROFILES)+++def diffusion_model_types():+ return tuple(mt for p in diffusion_profiles() for mt in p.model_types)+++def resolve_diffusion_profile(config):+ if config is None:+ return None+ for profile in diffusion_profiles():+ if profile.matches(config):+ return profile+ return None+++_BUILTINS_LOADED = False+++def _load_builtin_profiles():+ global _BUILTINS_LOADED+ if _BUILTINS_LOADED:+ return+ _BUILTINS_LOADED = True+ # Each module registers its profiles on import.+ from . import diffusion_block, diffusion_gemma_objective, diffusion_masked # noqa: F401+++def unwrap_diffusion_model(model):+ """The module holding config / heads / Unsloth attributes under DDP, FSDP and PEFT wrappers. Forwards+ still go through the wrapper so gradients synchronise."""+ while hasattr(model, "module") and isinstance(model.module, torch.nn.Module):+ model = model.module+ if hasattr(model, "get_base_model"):+ model = model.get_base_model()+ return model+++def response_mask(inputs):+ """Bool [B, L]: supervised positions. No labels -> every attended token (pretraining-style)."""+ labels = inputs.get("labels")+ attention_mask = inputs.get("attention_mask")+ if labels is not None:+ mask = labels != -100+ if attention_mask is not None:+ mask = mask & attention_mask.bool()+ return mask+ if attention_mask is not None:+ return attention_mask.bool()+ return torch.ones_like(inputs["input_ids"], dtype = torch.bool)+++def sample_noise_level(+ batch_size,+ eps,+ device,+ low = None,+ high = None,+):+ """Per-example t ~ U(low, high), default U(eps, 1 - eps) as in the published recipes."""+ low = eps if low is None else low+ high = 1.0 - eps if high is None else high+ return low + (high - low) * torch.rand(batch_size, device = device)+++def restore_rotary_buffers(model):+ """4.x remote code keeps RoPE frequencies in a non-persistent buffer filled in __init__; transformers 5 builds+ on the meta device and only re-fills it through `_init_weights`, which these classes override, so `inv_freq`+ comes back zero (GPU) or uninitialised (CPU) and positions are silently ignored. Recompute it."""+ repaired = 0+ for module in model.modules():+ init_fn = getattr(module, "rope_init_fn", None)+ inv_freq = getattr(module, "inv_freq", None)+ config = getattr(module, "config", None) or getattr(model, "config", None)+ if (+ init_fn is None+ or config is None+ or not isinstance(inv_freq, torch.Tensor)+ or inv_freq.is_meta+ ):+ continue+ try:+ expected, scaling = init_fn(+ config, inv_freq.device, **getattr(module, "rope_kwargs", {})+ )+ except Exception:+ continue+ expected = expected.to(device = inv_freq.device, dtype = inv_freq.dtype)+ if expected.shape != inv_freq.shape or torch.equal(inv_freq, expected):+ continue+ with torch.no_grad():+ inv_freq.copy_(expected)+ if isinstance(getattr(module, "original_inv_freq", None), torch.Tensor):+ module.original_inv_freq = inv_freq.clone()+ if hasattr(module, "attention_scaling"):+ module.attention_scaling = scaling+ repaired += 1+ return repaireddiff --git a/unsloth/models/diffusion_trainer.py b/unsloth/models/diffusion_trainer.pynew file mode 100644index 00000000000..e611babe470--- /dev/null+++ b/unsloth/models/diffusion_trainer.py@@ -0,0 +1,136 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""SFT for text-diffusion language models: SFTTrainer data handling, the family's own denoising loss."""++from collections import defaultdict+from dataclasses import dataclass, field+from typing import Optional++import torch+from trl import SFTConfig, SFTTrainer++from .diffusion_profiles import resolve_diffusion_profile++__all__ = ["DiffusionConfig", "DiffusionTrainer"]+++@dataclass+class DiffusionConfig(SFTConfig):+ """``SFTConfig`` plus the diffusion objective knobs. ``None`` = the model family's reference value."""++ diffusion_eps: Optional[float] = field(+ default = None, metadata = {"help": "Noise level t is drawn from U(eps, 1 - eps)."}+ )+ diffusion_time_weighting: Optional[str] = field(+ default = None,+ metadata = {+ "help": "'inverse_t' (masked-diffusion ELBO), 'none', or a family-specific scheme."+ },+ )+ diffusion_mask_token_id: Optional[int] = field(+ default = None, metadata = {"help": "Override the [MASK] token for absorbing-noise families."}+ )+ diffusion_block_size: Optional[int] = field(+ default = None, metadata = {"help": "Block length for block-diffusion families."}+ )+ diffusion_self_conditioning_p: Optional[float] = field(+ default = None, metadata = {"help": "Probability of the two-pass self-conditioned forward."}+ )+ diffusion_encoder_ar_weight: Optional[float] = field(+ default = None,+ metadata = {+ "help": "Weight of the encoder autoregressive co-loss (encoder/decoder families)."+ },+ )+ diffusion_prediction_type: Optional[str] = field(+ default = None,+ metadata = {"help": "Family-specific prediction parameterisation (e.g. 'mean', 'mean_loo')."},+ )++ def __post_init__(self):+ # Packed rows share one bidirectional attention window, so documents would see each other.+ self.packing = False+ self.padding_free = False+ # TRL 1.x enables MoE router logits from this coef; no diffusion objective adds the aux term.+ if hasattr(self, "router_aux_loss_coef"):+ self.router_aux_loss_coef = 0.0+ super().__post_init__()+++def _unwrap_config(model):+ for candidate in (model, getattr(model, "base_model", None), getattr(model, "module", None)):+ config = getattr(candidate, "config", None)+ if config is not None:+ return config+ return None+++class DiffusionTrainer(SFTTrainer):+ def __init__(+ self,+ model = None,+ args = None,+ *pos,+ diffusion_profile = None,+ **kwargs,+ ):+ if args is None:+ args = DiffusionConfig(output_dir = "outputs")+ elif not isinstance(args, DiffusionConfig):+ args = DiffusionConfig(+ **{+ k: v+ for k, v in args.to_dict().items()+ if k in DiffusionConfig.__dataclass_fields__+ }+ )+ args.packing = False+ args.padding_free = False+ profile = diffusion_profile or resolve_diffusion_profile(_unwrap_config(model))+ if profile is None:+ raise ValueError(+ "Unsloth: DiffusionTrainer needs a text-diffusion model. "+ f"Got model_type={getattr(_unwrap_config(model), 'model_type', None)!r}."+ )+ self.diffusion_profile = profile+ self._diffusion_metrics = defaultdict(list)+ super().__init__(model, args, *pos, **kwargs)+ # The loss is ours, so the Trainer must scale it for gradient accumulation itself.+ self.model_accepts_loss_kwargs = False++ def compute_loss(+ self,+ model,+ inputs,+ return_outputs = False,+ num_items_in_batch = None,+ ):+ loss, outputs, metrics = self.diffusion_profile.compute_loss(+ model, inputs, self.args, num_items_in_batch = num_items_in_batch+ )+ mode = "train" if model.training else "eval"+ for key, value in (metrics or {}).items():+ self._diffusion_metrics[f"{mode}/{key}"].append(float(value))+ return (loss, outputs) if return_outputs else loss++ def prediction_step(+ self,+ model,+ inputs,+ prediction_loss_only,+ ignore_keys = None,+ ):+ inputs = self._prepare_inputs(inputs)+ with torch.no_grad(), self.compute_loss_context_manager():+ loss = self.compute_loss(model, inputs)+ return (loss.detach(), None, None)++ def log(self, logs, *args, **kwargs):+ mode = "eval" if any(k.startswith("eval_") for k in logs) else "train"+ prefix = "eval_" if mode == "eval" else ""+ for key in [k for k in self._diffusion_metrics if k.startswith(mode + "/")]:+ values = self._diffusion_metrics.pop(key)+ if values:+ logs[prefix + key.split("/", 1)[1]] = sum(values) / len(values)+ return super().log(logs, *args, **kwargs)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex aed68d94888..f3fd14cdc77 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -1790,6 +1790,7 @@ from ..kernels import ()from .vision import FastBaseModel, _is_text_seq2seq_configfrom .diffusion import FastDiffusionModel, is_diffusion_model_type+from .diffusion_profiles import resolve_diffusion_profilefrom transformers import (AutoModelForCausalLM,)@@ -2131,11 +2132,15 @@ class FastModel(FastBaseModel):device_map_planner_kwargs = device_map_planner_kwargs,trust_remote_code = trust_remote_code,revision = base_revision,+ **({"config": user_config} if user_config is not None else {}),**kwargs,)# Returns before the FORCE_FLOAT32 scan and no diffusion type is on that list, so False. Stamped, not left unset, or the trainer reads whatever an earlier load wrote.model = _mark_forced_float32(model, False)model = _mark_full_finetuning(model, full_finetuning)+ # This early return skips the re-enable at the end of from_pretrained.+ if not was_disabled:+ enable_progress_bars()return _mark_requested_float32(model, user_float32), tokenizertry:@@ -2253,7 +2258,22 @@ class FastModel(FastBaseModel):fast_inference = False# Text-diffusion models (DiffusionGemma) take a transformers-only slow path: a custom block-diffusion generate over a novel backbone, so Unsloth's autoregressive kernel/compile patching is skipped and the unmodified HF model is loaded, keeping 4bit/8bit and PEFT LoRA.- if is_diffusion_model_type(model_types):+ # Remote-code checkpoints can report their backbone's type (Nemotron-Labs-Diffusion says "nemotron"), so also match the raw config.+ diffusion_config = model_config+ if diffusion_config is None and peft_config is not None:+ try:+ diffusion_config = AutoConfig.from_pretrained(+ peft_config.base_model_name_or_path,+ token = token,+ trust_remote_code = trust_remote_code,+ local_files_only = local_files_only,+ )+ except Exception:+ diffusion_config = None+ if (+ is_diffusion_model_type(model_types)+ or resolve_diffusion_profile(diffusion_config) is not None+ ):return _dispatch_diffusion()lowered_model_name = model_name.lower()