Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Honor padding masks in Llama and Qwen3 cached attention (#13256)
Arthur60a006d1fff4
diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.ymlindex 760d96dea3d..d4bcb31a253 100644--- a/.github/workflows/consolidated-tests-ci.yml+++ b/.github/workflows/consolidated-tests-ci.yml@@ -435,6 +435,7 @@ jobs:run: |python -m pytest -v --tb=short \tests/utils/test_prepare_inputs_leftpad.py \+ tests/utils/test_single_row_cached_attention.py \tests/utils/test_rope_scaling_drift.py- name: dataset_num_proc worker bounds (HARD GATE)diff --git a/tests/utils/test_single_row_cached_attention.py b/tests/utils/test_single_row_cached_attention.pynew file mode 100644index 00000000000..7ef50fb951d--- /dev/null+++ b/tests/utils/test_single_row_cached_attention.py@@ -0,0 +1,120 @@+# Unsloth Zoo - Utilities for Unsloth+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++import ast+from math import sqrt+from pathlib import Path+from types import SimpleNamespace+from typing import Optional, Tuple++import pytest+import torch+from torch.nn import functional as F+++def _linear(projection, inputs, out):+ return out.copy_(F.linear(inputs, projection.weight))+++def _rms_norm(layer, inputs):+ return F.rms_norm(inputs, (inputs.shape[-1],), layer.weight, layer.variance_epsilon)+++class _IdentityRope:+ def extend_rope_embedding(self, *args):+ pass++ def get_cached(self, seq_len, device):+ return torch.ones(seq_len, 2), torch.zeros(seq_len, 2)++[email protected](params = ["llama", "qwen3"])+def decoder(request):+ name = f"{request.param.title()}Attention_fast_forward_inference"+ source = Path(__file__).resolve().parents[2] / "unsloth" / "models" / f"{request.param}.py"+ tree = ast.parse(source.read_text(encoding = "utf-8"))+ function = next(+ node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name+ )+ namespace = {+ "torch": torch,+ "Optional": Optional,+ "Tuple": Tuple,+ "KV_CACHE_INCREMENT": 256,+ "math_sqrt": sqrt,+ "fast_linear_forward": _linear,+ "fast_rms_layernorm_inference": _rms_norm,+ "torch_matmul": torch.matmul,+ "torch_nn_functional_softmax": F.softmax,+ "scaled_dot_product_attention": F.scaled_dot_product_attention,+ "SDPA_HAS_GQA": True,+ }+ exec(compile(ast.Module(body = [function], type_ignores = []), str(source), "exec"), namespace)+ return namespace[name]+++ "mask_kind",+ [+ "padding",+ "boolean",+ "additive",+ "none",+ ],+)[email protected]("groups", [1, 2])[email protected]("batch_size", [1, 2])[email protected]("sliding_window", [None, 2])+def test_single_row_cached_attention_masks(decoder, mask_kind, groups, batch_size, sliding_window):+ attention = SimpleNamespace(+ config = SimpleNamespace(+ num_attention_heads = groups,+ num_key_value_heads = 1,+ hidden_size = 2,+ sliding_window = sliding_window,+ ),+ num_key_value_groups = groups,+ head_dim = 2,+ rotary_emb = _IdentityRope(),+ q_proj = SimpleNamespace(weight = torch.zeros(2 * groups, 2)),+ k_proj = SimpleNamespace(weight = torch.zeros(2, 2)),+ v_proj = SimpleNamespace(weight = torch.eye(2)),+ o_proj = SimpleNamespace(weight = torch.eye(2).repeat(1, groups) / groups),+ q_norm = SimpleNamespace(weight = torch.ones(2), variance_epsilon = 1e-6),+ k_norm = SimpleNamespace(weight = torch.ones(2), variance_epsilon = 1e-6),+ )+ keys = torch.zeros(batch_size, 1, 2, 2)+ values = torch.tensor([[[[100.0, 100.0], [2.0, 2.0]]]]).expand(batch_size, -1, -1, -1)+ keep_mask = torch.tensor([[[[False, True, True]]]]).expand(batch_size, -1, -1, -1)+ masks = {+ "padding": keep_mask[:, 0, 0, :].long(),+ "boolean": keep_mask,+ "additive": torch.tensor([[[[float("-inf"), 0.0, 0.0]]]]).expand(batch_size, -1, -1, -1),+ "none": None,+ }++ output, _ = decoder(+ attention,+ torch.zeros(batch_size, 1, 2),+ (keys, values),+ torch.ones(batch_size, 1, dtype = torch.long),+ do_prefill = True,+ attention_mask = masks[mask_kind],+ )++ # The window of 2 drops the padded slot, so every mask agrees there.+ expected = 34.0 if mask_kind == "none" and sliding_window is None else 1.0+ torch.testing.assert_close(output, torch.full((batch_size, 1, 2), expected))diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex c56751907e6..5fce9674afb 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -676,6 +676,16 @@ def LlamaAttention_fast_forward_inference(else:Knn, Vnn = Kn, Vn+ if attention_mask is not None and attention_mask.dim() == 2:+ attention_mask = attention_mask[:, None, None, :].to(torch.bool)+ elif (+ attention_mask is not None+ and attention_mask.dim() == 4+ and attention_mask.dtype != torch.bool+ ):+ # Decode is more stable with boolean keep masks than additive bf16 masks.+ attention_mask = attention_mask.eq(0)+# Grouped query attention._, _, cached_len, _ = Knn.shapeif bsz == 1 or ((not SDPA_HAS_GQA) and n_groups != 1):@@ -695,20 +705,11 @@ def LlamaAttention_fast_forward_inference(Qn *= self.scalar# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805.A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])+ if attention_mask is not None:+ A.masked_fill_(~attention_mask, float("-inf"))A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)A = torch_matmul(A, Vnn, out = Qn)- # attention_mask fixup for SDPA when the user passes a 2D padding mask.else:- if attention_mask is not None and attention_mask.dim() == 2:- attention_mask = attention_mask[:, None, None, :].to(torch.bool)- elif (- attention_mask is not None- and attention_mask.dim() == 4- and attention_mask.dtype != torch.bool- ):- # Decode is more stable with boolean keep masks than additive bf16 masks.- attention_mask = attention_mask.eq(0)-if SDPA_HAS_GQA:A = scaled_dot_product_attention(Qn,diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex e064826e6d0..676bc11b228 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -329,6 +329,8 @@ def Qwen3Attention_fast_forward_inference(# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805# (comment 2153349963).A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])+ if attention_mask is not None:+ A.masked_fill_(~attention_mask, float("-inf"))A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)A = torch_matmul(A, Vnn, out = Qn)else: