Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix SDAR empty-mask resampling to match the reference (#13771)

cherry77-cloud1389ec4cbed1
diff --git a/tests/diffusion/test_diffusion_block.py b/tests/diffusion/test_diffusion_block.py
index 91c7c461e0f..5854c5111a5 100644
--- a/tests/diffusion/test_diffusion_block.py
+++ b/tests/diffusion/test_diffusion_block.py
@@ -253,7 +253,14 @@ def test_noise_ranges_and_prompt_untouched():
assert p.min() >= low and p.max() <= high
assert not masked[:, :5].any() and torch.equal(noisy == MASK_ID, masked)
ratio = masked[:, 5:].float().mean().item()
- assert abs(ratio - (low + high) / 2) < 0.02
+ expected = (low + high) / 2
+ if profile is sdar:
+ # Rejection resamples both p and mask; only ten empty draws force one token.
+ n = int(maskable[0].sum())
+ empty_probability = (1 - low) ** n / (n + 1)
+ expected *= sum(empty_probability**attempt for attempt in range(10))
+ expected += empty_probability**10 / n
+ assert abs(ratio - expected) < 0.02
_, masked, _ = sdar.sample_noise(clean[:, :6], maskable[:, :6], MASK_ID, SimpleNamespace())
assert masked.any(dim = 1).all() # SDAR masks at least one supervised token per row
_, _, p = llada2.sample_noise(clean, maskable, MASK_ID, SimpleNamespace(diffusion_eps = 0.1))
diff --git a/unsloth/models/diffusion_block.py b/unsloth/models/diffusion_block.py
index b7759c0246f..dfd29e7d07b 100644
--- a/unsloth/models/diffusion_block.py
+++ b/unsloth/models/diffusion_block.py
@@ -183,10 +183,21 @@ class BlockDiffusionProfile(DiffusionProfile):
"""-> noisy ids, masked bool [B, L], p [B] mask probability per row."""
batch, length = clean.shape
low, high = self.noise_range(args)
- p = low + (high - low) * torch.rand(batch, device = clean.device)
- masked = (torch.rand(batch, length, device = clean.device) < p[:, None]) & maskable
- if self.defaults.get("ensure_masked", False):
- # Rescue draw taken every step (fixed shape, no host sync); it only changes rows left empty.
+ if not self.defaults.get("ensure_masked", False):
+ p = low + (high - low) * torch.rand(batch, device = clean.device)
+ masked = (torch.rand(batch, length, device = clean.device) < p[:, None]) & maskable
+ else:
+ # SDAR redraws (p, mask) jointly up to ten times, then forces a token with the last p. All ten
+ # draws at once (fixed shape, no host sync); each row keeps its first non-empty one.
+ tries = 10
+ ps = low + (high - low) * torch.rand(tries, batch, device = clean.device)
+ draws = (
+ torch.rand(tries, batch, length, device = clean.device) < ps[..., None]
+ ) & maskable
+ hit = draws.any(dim = 2)
+ pick_try = torch.where(hit.any(dim = 0), hit.int().argmax(dim = 0), tries - 1)
+ rows = torch.arange(batch, device = clean.device)
+ p, masked = ps[pick_try, rows], draws[pick_try, rows]
empty = maskable.any(dim = 1) & ~masked.any(dim = 1)
scores = torch.rand(batch, length, device = clean.device).masked_fill(~maskable, -1.0)
pick = F.one_hot(scores.argmax(dim = 1), length).bool()