Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep xFormers attention causal when the decoder is called without a mask (#12199)
Daniel Han825c1f2bd29f
diff --git a/tests/utils/test_attention_masks.py b/tests/utils/test_attention_masks.pyindex 0392f78218d..9a5991c5e13 100644--- a/tests/utils/test_attention_masks.py+++ b/tests/utils/test_attention_masks.py@@ -719,3 +719,38 @@ def test_the_outgoing_window_mask_is_freed_before_its_replacement(monkeypatch):cached_during_build), "the previous mask was still alive while its replacement was allocated"attention_dispatch._WINDOW_MASK_CACHE.clear()+++ not has_real_cuda() or not attention_dispatch.HAS_XFORMERS, reason = "needs xformers on CUDA"+)+def test_real_xformers_without_mask_is_causal():+ config = attention_dispatch.AttentionConfig(+ backend = attention_dispatch.XFORMERS, n_kv_heads = 2, n_groups = 2+ )+ context = attention_dispatch.AttentionContext(+ bsz = 2,+ q_len = 8,+ kv_seq_len = 8,+ n_heads = 4,+ head_dim = 64,+ requires_grad = True,+ seq_info = None,+ attention_mask = None,+ causal_mask = None,+ )+ dtype = torch.bfloat16 if attention_dispatch.SUPPORTS_BFLOAT16 else torch.float16+ g = torch.Generator(device = "cuda").manual_seed(0)+ Q = torch.randn(2, 4, 8, 64, device = "cuda", dtype = dtype, generator = g)+ K = torch.randn(2, 2, 8, 64, device = "cuda", dtype = dtype, generator = g)+ V = torch.randn(2, 2, 8, 64, device = "cuda", dtype = dtype, generator = g)+ got = attention_dispatch.run_attention(config = config, context = context, Q = Q, K = K, V = V)++ keep = torch.ones(8, 8, dtype = torch.bool, device = "cuda").tril()+ want = torch.nn.functional.scaled_dot_product_attention(+ Q.float(),+ K.float().repeat_interleave(2, 1),+ V.float().repeat_interleave(2, 1),+ attn_mask = keep,+ ).transpose(1, 2)+ torch.testing.assert_close(got.float().reshape(want.shape), want, atol = 2e-2, rtol = 2e-2)diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 30622f09c4e..82b4e7cb4ed 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -419,10 +419,19 @@ def run_attention(bsz, q_len, n_heads, head_dim)elif backend == XFORMERS:+ base_mask = context.causal_mask+ # Only CausalLM_fast_forward supplies the mask; a direct decoder call (Liger, TRL's get_decoder paths) would attend bidirectionally.+ if (+ base_mask is None+ and xformers is not None+ and context.seq_info is None+ and q_len == kv_seq_len+ ):+ base_mask = xformers.attn_bias.LowerTriangularMask()attn_bias = build_xformers_block_causal_mask(context.seq_info,sliding_window = sliding_window,- base_mask = context.causal_mask,+ base_mask = base_mask,)attn_bias = move_xformers_attention_bias(attn_bias, Q.device)