Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep packed documents apart under flash attention when layers get mRoPE position ids (#15771)

Daniel Han3b01845d090d
diff --git a/tests/test_flash_attention_mrope_packed.py b/tests/test_flash_attention_mrope_packed.py
index 04aff428a65..bd9708b368a 100644
--- a/tests/test_flash_attention_mrope_packed.py
+++ b/tests/test_flash_attention_mrope_packed.py
@@ -82,3 +82,124 @@ def test_flash_attention_forward_keeps_one_row_unpacked(patched, monkeypatch):
assert calls == []
assert torch.isfinite(out).all()
torch.testing.assert_close(out, ref, atol = 2e-2, rtol = 2e-2)
+
+
+fa_integration = pytest.importorskip("transformers.integrations.flash_attention")
+
+from unsloth.import_fixes import ( # noqa: E402
+ _packed_row_varlen_kwargs,
+ fix_transformers_flash_attention_mrope_packed_rows,
+)
+
+
+def packed_rows(monkeypatch):
+ calls = []
+ monkeypatch.setattr(
+ fa_integration,
+ "_flash_attention_forward",
+ lambda q, k, v, mask, *a, **kw: calls.append(kw) or q,
+ )
+ fix_transformers_flash_attention_mrope_packed_rows()
+ return fa_integration._flash_attention_forward, calls
+
+
+def _query(length):
+ return torch.zeros(1, length, 2, 8)
+
+
+def test_packed_row_with_mrope_ids_gets_its_document_boundaries(packed_rows):
+ forward, calls = packed_rows
+ lengths = torch.tensor([3, 5], dtype = torch.int32)
+ q = _query(8)
+ forward(q, q, q, None, position_ids = _mrope(8), packed_seq_lengths = lengths)
+ assert calls[-1]["cu_seq_lens_q"].tolist() == [0, 3, 8]
+ assert calls[-1]["cu_seq_lens_k"].tolist() == [0, 3, 8]
+ assert calls[-1]["max_length_q"] == calls[-1]["max_length_k"] == 5
+
+
+def test_every_other_call_is_passed_through_unchanged(packed_rows):
+ forward, calls = packed_rows
+ lengths = torch.tensor([3, 5], dtype = torch.int32)
+ q = _query(8)
+ flat = torch.tensor([[0, 1, 2, 0, 1, 2, 3, 4]])
+ cases = [
+ (q, None, {"position_ids": _mrope(8)}), # not an Unsloth packed row
+ (
+ q,
+ None,
+ {"position_ids": flat, "packed_seq_lengths": lengths},
+ ), # 2D ids already mark starts
+ (q, torch.ones(1, 8), {"position_ids": _mrope(8), "packed_seq_lengths": lengths}),
+ (
+ torch.zeros(2, 8, 2, 8),
+ None,
+ {"position_ids": _mrope(8, 2), "packed_seq_lengths": lengths},
+ ),
+ (
+ _query(9),
+ None,
+ {"position_ids": _mrope(9), "packed_seq_lengths": lengths},
+ ), # rows not covered
+ (
+ q,
+ None,
+ {"position_ids": _mrope(8), "packed_seq_lengths": lengths, "cu_seq_lens_q": "caller"},
+ ),
+ ]
+ for query, mask, kwargs in cases:
+ forward(query, query, query, mask, **dict(kwargs))
+ assert calls[-1] == kwargs
+
+
+def test_packed_rows_patch_is_idempotent(packed_rows):
+ forward, _ = packed_rows
+ fix_transformers_flash_attention_mrope_packed_rows()
+ assert fa_integration._flash_attention_forward is forward
+
+
+def test_helper_ignores_rows_without_packed_lengths():
+ assert _packed_row_varlen_kwargs(_query(4), None, {"position_ids": _mrope(4)}) is None
+
+
[email protected](not has_real_cuda(), reason = "flash-attn needs CUDA")
+def test_packed_documents_do_not_see_each_other_under_flash_attention():
+ pytest.importorskip("flash_attn")
+ from transformers.integrations.flash_attention import flash_attention_forward
+
+ fix_transformers_flash_attention_mrope_packed_sequence()
+ fix_transformers_flash_attention_mrope_packed_rows()
+
+ class _Attention(torch.nn.Module):
+ is_causal = True
+ layer_idx = 0
+
+ def __init__(self):
+ super().__init__()
+ self.config = type("C", (), {"_attn_implementation": "flash_attention_2"})()
+
+ torch.manual_seed(0)
+ first, second, heads, dim = 48, 32, 4, 64
+ total = first + second
+ q, k, v = (
+ torch.randn(1, heads, total, dim, device = "cuda", dtype = torch.bfloat16) for _ in range(3)
+ )
+ positions = torch.cat([torch.arange(first), torch.arange(second)]).cuda()
+ out, _ = flash_attention_forward(
+ _Attention(),
+ q,
+ k,
+ v,
+ None,
+ position_ids = positions.view(1, 1, total).expand(3, 1, total),
+ packed_seq_lengths = torch.tensor([first, second], dtype = torch.int32),
+ )
+
+ def alone(s, e):
+ return torch.nn.functional.scaled_dot_product_attention(
+ q[:, :, s:e], k[:, :, s:e], v[:, :, s:e], is_causal = True
+ ).transpose(1, 2)
+
+ torch.testing.assert_close(
+ out, torch.cat([alone(0, first), alone(first, total)], dim = 1), atol = 2e-2, rtol = 2e-2
+ )
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 845767ea4cc..5e67910983d 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -335,6 +335,7 @@ from .import_fixes import (
fix_transformers_bnb_prequantized_save,
fix_transformers_fully_masked_rows,
fix_transformers_flash_attention_mrope_packed_sequence,
+ fix_transformers_flash_attention_mrope_packed_rows,
fix_transformers_untrusted_config_fields,
fix_transformers_chat_template_path_traversal,
fix_transformers_tokenizer_config_vocab_file_override,
@@ -400,6 +401,7 @@ fix_transformers5_bare_annotation_configs()
# Probe-gated: no-op unless SDPA gets a query row that attends to nothing; runs before model imports so plain generate is covered.
fix_transformers_fully_masked_rows()
fix_transformers_flash_attention_mrope_packed_sequence()
+fix_transformers_flash_attention_mrope_packed_rows()
fix_transformers_chunked_mask_block_sequence_ids()
fix_transformers_flex_mask_graph_breaks()
# CVE-2026-4372 / 5241 / 9856 and the tokenizer_config.json vocab path traversal, no-ops once
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index c3b718a027a..2f1c8294c80 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -2125,6 +2125,75 @@ def fix_transformers_flash_attention_mrope_packed_sequence():
)
+_PACKED_ROWS_PATCH_FLAG = "_unsloth_patched_mrope_packed_rows"
+_PACKED_ROWS_CHECKED = [None, None, False]
+
+
+def _packed_row_varlen_kwargs(query, attention_mask, kwargs):
+ """cu_seqlens from an Unsloth packed row whose layers got mRoPE ids, else None."""
+ if kwargs.get("packed_seq_lengths") is None:
+ return None
+ import torch
+
+ position_ids = kwargs.get("position_ids")
+ if (
+ attention_mask is not None
+ or kwargs.get("cu_seq_lens_q") is not None
+ or not isinstance(position_ids, torch.Tensor)
+ or position_ids.dim() <= 2
+ or query.dim() != 4
+ or query.shape[0] != 1
+ ):
+ return None
+ from .utils.packing import get_packed_info_from_kwargs
+
+ info = get_packed_info_from_kwargs(kwargs, query.device)
+ if info is None:
+ return None
+ _, cu_seqlens, max_seqlen = info
+ # One host read per batch, not per layer: rows that do not cover the query keep the original path.
+ if not torch.compiler.is_compiling():
+ checked = _PACKED_ROWS_CHECKED
+ if checked[0] is not cu_seqlens or checked[1] != query.shape[1]:
+ checked[:] = [cu_seqlens, query.shape[1], int(cu_seqlens[-1]) == query.shape[1]]
+ if not checked[2]:
+ return None
+ return {
+ "cu_seq_lens_q": cu_seqlens,
+ "cu_seq_lens_k": cu_seqlens,
+ "max_length_q": max_seqlen,
+ "max_length_k": max_seqlen,
+ }
+
+
+def fix_transformers_flash_attention_mrope_packed_rows():
+ """Qwen3.5 / Qwen3.5-MoE on transformers 5.3 give decoder layers 3-row mRoPE ids, which cannot mark
+ document starts: take a packed row's boundaries from `packed_seq_lengths` instead."""
+ try:
+ from transformers.integrations import flash_attention as fa_integration
+ except Exception:
+ return
+ current = getattr(fa_integration, "_flash_attention_forward", None)
+ if current is None or getattr(current, _PACKED_ROWS_PATCH_FLAG, False):
+ return
+
+ @functools.wraps(current)
+ def _flash_attention_forward(
+ query_states, key_states, value_states, attention_mask, *args, **kwargs
+ ):
+ varlen = _packed_row_varlen_kwargs(query_states, attention_mask, kwargs)
+ if varlen is not None:
+ kwargs.update(varlen)
+ return current(query_states, key_states, value_states, attention_mask, *args, **kwargs)
+
+ setattr(_flash_attention_forward, _PACKED_ROWS_PATCH_FLAG, True)
+ fa_integration._flash_attention_forward = _flash_attention_forward
+ logger.info(
+ "Unsloth: Patching transformers flash attention so packed rows with mRoPE layer "
+ "position ids keep their documents apart"
+ )
+
+
_CHUNKED_MASK_PATCH_FLAG = "_unsloth_patched_chunked_block_sequence_ids"
_BLOCK_SEQUENCE_IDS = "block_sequence_ids"