Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Honor padding masks in Llama and Qwen3 cached attention (#13256)

Arthur60a006d1fff4
diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml
index 760d96dea3d..d4bcb31a253 100644
--- a/.github/workflows/consolidated-tests-ci.yml
+++ b/.github/workflows/consolidated-tests-ci.yml
@@ -435,6 +435,7 @@ jobs:
run: |
python -m pytest -v --tb=short \
tests/utils/test_prepare_inputs_leftpad.py \
+ tests/utils/test_single_row_cached_attention.py \
tests/utils/test_rope_scaling_drift.py
- name: dataset_num_proc worker bounds (HARD GATE)
diff --git a/tests/utils/test_single_row_cached_attention.py b/tests/utils/test_single_row_cached_attention.py
new file mode 100644
index 00000000000..7ef50fb951d
--- /dev/null
+++ b/tests/utils/test_single_row_cached_attention.py
@@ -0,0 +1,120 @@
+# Unsloth Zoo - Utilities for Unsloth
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import ast
+from math import sqrt
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Optional, Tuple
+
+import pytest
+import torch
+from torch.nn import functional as F
+
+
+def _linear(projection, inputs, out):
+ return out.copy_(F.linear(inputs, projection.weight))
+
+
+def _rms_norm(layer, inputs):
+ return F.rms_norm(inputs, (inputs.shape[-1],), layer.weight, layer.variance_epsilon)
+
+
+class _IdentityRope:
+ def extend_rope_embedding(self, *args):
+ pass
+
+ def get_cached(self, seq_len, device):
+ return torch.ones(seq_len, 2), torch.zeros(seq_len, 2)
+
+
[email protected](params = ["llama", "qwen3"])
+def decoder(request):
+ name = f"{request.param.title()}Attention_fast_forward_inference"
+ source = Path(__file__).resolve().parents[2] / "unsloth" / "models" / f"{request.param}.py"
+ tree = ast.parse(source.read_text(encoding = "utf-8"))
+ function = next(
+ node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name
+ )
+ namespace = {
+ "torch": torch,
+ "Optional": Optional,
+ "Tuple": Tuple,
+ "KV_CACHE_INCREMENT": 256,
+ "math_sqrt": sqrt,
+ "fast_linear_forward": _linear,
+ "fast_rms_layernorm_inference": _rms_norm,
+ "torch_matmul": torch.matmul,
+ "torch_nn_functional_softmax": F.softmax,
+ "scaled_dot_product_attention": F.scaled_dot_product_attention,
+ "SDPA_HAS_GQA": True,
+ }
+ exec(compile(ast.Module(body = [function], type_ignores = []), str(source), "exec"), namespace)
+ return namespace[name]
+
+
+ "mask_kind",
+ [
+ "padding",
+ "boolean",
+ "additive",
+ "none",
+ ],
+)
[email protected]("groups", [1, 2])
[email protected]("batch_size", [1, 2])
[email protected]("sliding_window", [None, 2])
+def test_single_row_cached_attention_masks(decoder, mask_kind, groups, batch_size, sliding_window):
+ attention = SimpleNamespace(
+ config = SimpleNamespace(
+ num_attention_heads = groups,
+ num_key_value_heads = 1,
+ hidden_size = 2,
+ sliding_window = sliding_window,
+ ),
+ num_key_value_groups = groups,
+ head_dim = 2,
+ rotary_emb = _IdentityRope(),
+ q_proj = SimpleNamespace(weight = torch.zeros(2 * groups, 2)),
+ k_proj = SimpleNamespace(weight = torch.zeros(2, 2)),
+ v_proj = SimpleNamespace(weight = torch.eye(2)),
+ o_proj = SimpleNamespace(weight = torch.eye(2).repeat(1, groups) / groups),
+ q_norm = SimpleNamespace(weight = torch.ones(2), variance_epsilon = 1e-6),
+ k_norm = SimpleNamespace(weight = torch.ones(2), variance_epsilon = 1e-6),
+ )
+ keys = torch.zeros(batch_size, 1, 2, 2)
+ values = torch.tensor([[[[100.0, 100.0], [2.0, 2.0]]]]).expand(batch_size, -1, -1, -1)
+ keep_mask = torch.tensor([[[[False, True, True]]]]).expand(batch_size, -1, -1, -1)
+ masks = {
+ "padding": keep_mask[:, 0, 0, :].long(),
+ "boolean": keep_mask,
+ "additive": torch.tensor([[[[float("-inf"), 0.0, 0.0]]]]).expand(batch_size, -1, -1, -1),
+ "none": None,
+ }
+
+ output, _ = decoder(
+ attention,
+ torch.zeros(batch_size, 1, 2),
+ (keys, values),
+ torch.ones(batch_size, 1, dtype = torch.long),
+ do_prefill = True,
+ attention_mask = masks[mask_kind],
+ )
+
+ # The window of 2 drops the padded slot, so every mask agrees there.
+ expected = 34.0 if mask_kind == "none" and sliding_window is None else 1.0
+ torch.testing.assert_close(output, torch.full((batch_size, 1, 2), expected))
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index c56751907e6..5fce9674afb 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -676,6 +676,16 @@ def LlamaAttention_fast_forward_inference(
else:
Knn, Vnn = Kn, Vn
+ if attention_mask is not None and attention_mask.dim() == 2:
+ attention_mask = attention_mask[:, None, None, :].to(torch.bool)
+ elif (
+ attention_mask is not None
+ and attention_mask.dim() == 4
+ and attention_mask.dtype != torch.bool
+ ):
+ # Decode is more stable with boolean keep masks than additive bf16 masks.
+ attention_mask = attention_mask.eq(0)
+
# Grouped query attention.
_, _, cached_len, _ = Knn.shape
if bsz == 1 or ((not SDPA_HAS_GQA) and n_groups != 1):
@@ -695,20 +705,11 @@ def LlamaAttention_fast_forward_inference(
Qn *= self.scalar
# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805.
A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
+ if attention_mask is not None:
+ A.masked_fill_(~attention_mask, float("-inf"))
A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
A = torch_matmul(A, Vnn, out = Qn)
- # attention_mask fixup for SDPA when the user passes a 2D padding mask.
else:
- if attention_mask is not None and attention_mask.dim() == 2:
- attention_mask = attention_mask[:, None, None, :].to(torch.bool)
- elif (
- attention_mask is not None
- and attention_mask.dim() == 4
- and attention_mask.dtype != torch.bool
- ):
- # Decode is more stable with boolean keep masks than additive bf16 masks.
- attention_mask = attention_mask.eq(0)
-
if SDPA_HAS_GQA:
A = scaled_dot_product_attention(
Qn,
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index e064826e6d0..676bc11b228 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -329,6 +329,8 @@ def Qwen3Attention_fast_forward_inference(
# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805
# (comment 2153349963).
A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
+ if attention_mask is not None:
+ A.masked_fill_(~attention_mask, float("-inf"))
A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
A = torch_matmul(A, Vnn, out = Qn)
else: