Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep packed documents apart under flash attention when layers get mRoPE position ids (#15771)
Daniel Han3b01845d090d
diff --git a/tests/test_flash_attention_mrope_packed.py b/tests/test_flash_attention_mrope_packed.pyindex 04aff428a65..bd9708b368a 100644--- a/tests/test_flash_attention_mrope_packed.py+++ b/tests/test_flash_attention_mrope_packed.py@@ -82,3 +82,124 @@ def test_flash_attention_forward_keeps_one_row_unpacked(patched, monkeypatch):assert calls == []assert torch.isfinite(out).all()torch.testing.assert_close(out, ref, atol = 2e-2, rtol = 2e-2)+++fa_integration = pytest.importorskip("transformers.integrations.flash_attention")++from unsloth.import_fixes import ( # noqa: E402+ _packed_row_varlen_kwargs,+ fix_transformers_flash_attention_mrope_packed_rows,+)+++def packed_rows(monkeypatch):+ calls = []+ monkeypatch.setattr(+ fa_integration,+ "_flash_attention_forward",+ lambda q, k, v, mask, *a, **kw: calls.append(kw) or q,+ )+ fix_transformers_flash_attention_mrope_packed_rows()+ return fa_integration._flash_attention_forward, calls+++def _query(length):+ return torch.zeros(1, length, 2, 8)+++def test_packed_row_with_mrope_ids_gets_its_document_boundaries(packed_rows):+ forward, calls = packed_rows+ lengths = torch.tensor([3, 5], dtype = torch.int32)+ q = _query(8)+ forward(q, q, q, None, position_ids = _mrope(8), packed_seq_lengths = lengths)+ assert calls[-1]["cu_seq_lens_q"].tolist() == [0, 3, 8]+ assert calls[-1]["cu_seq_lens_k"].tolist() == [0, 3, 8]+ assert calls[-1]["max_length_q"] == calls[-1]["max_length_k"] == 5+++def test_every_other_call_is_passed_through_unchanged(packed_rows):+ forward, calls = packed_rows+ lengths = torch.tensor([3, 5], dtype = torch.int32)+ q = _query(8)+ flat = torch.tensor([[0, 1, 2, 0, 1, 2, 3, 4]])+ cases = [+ (q, None, {"position_ids": _mrope(8)}), # not an Unsloth packed row+ (+ q,+ None,+ {"position_ids": flat, "packed_seq_lengths": lengths},+ ), # 2D ids already mark starts+ (q, torch.ones(1, 8), {"position_ids": _mrope(8), "packed_seq_lengths": lengths}),+ (+ torch.zeros(2, 8, 2, 8),+ None,+ {"position_ids": _mrope(8, 2), "packed_seq_lengths": lengths},+ ),+ (+ _query(9),+ None,+ {"position_ids": _mrope(9), "packed_seq_lengths": lengths},+ ), # rows not covered+ (+ q,+ None,+ {"position_ids": _mrope(8), "packed_seq_lengths": lengths, "cu_seq_lens_q": "caller"},+ ),+ ]+ for query, mask, kwargs in cases:+ forward(query, query, query, mask, **dict(kwargs))+ assert calls[-1] == kwargs+++def test_packed_rows_patch_is_idempotent(packed_rows):+ forward, _ = packed_rows+ fix_transformers_flash_attention_mrope_packed_rows()+ assert fa_integration._flash_attention_forward is forward+++def test_helper_ignores_rows_without_packed_lengths():+ assert _packed_row_varlen_kwargs(_query(4), None, {"position_ids": _mrope(4)}) is None++[email protected](not has_real_cuda(), reason = "flash-attn needs CUDA")+def test_packed_documents_do_not_see_each_other_under_flash_attention():+ pytest.importorskip("flash_attn")+ from transformers.integrations.flash_attention import flash_attention_forward++ fix_transformers_flash_attention_mrope_packed_sequence()+ fix_transformers_flash_attention_mrope_packed_rows()++ class _Attention(torch.nn.Module):+ is_causal = True+ layer_idx = 0++ def __init__(self):+ super().__init__()+ self.config = type("C", (), {"_attn_implementation": "flash_attention_2"})()++ torch.manual_seed(0)+ first, second, heads, dim = 48, 32, 4, 64+ total = first + second+ q, k, v = (+ torch.randn(1, heads, total, dim, device = "cuda", dtype = torch.bfloat16) for _ in range(3)+ )+ positions = torch.cat([torch.arange(first), torch.arange(second)]).cuda()+ out, _ = flash_attention_forward(+ _Attention(),+ q,+ k,+ v,+ None,+ position_ids = positions.view(1, 1, total).expand(3, 1, total),+ packed_seq_lengths = torch.tensor([first, second], dtype = torch.int32),+ )++ def alone(s, e):+ return torch.nn.functional.scaled_dot_product_attention(+ q[:, :, s:e], k[:, :, s:e], v[:, :, s:e], is_causal = True+ ).transpose(1, 2)++ torch.testing.assert_close(+ out, torch.cat([alone(0, first), alone(first, total)], dim = 1), atol = 2e-2, rtol = 2e-2+ )diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 845767ea4cc..5e67910983d 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -335,6 +335,7 @@ from .import_fixes import (fix_transformers_bnb_prequantized_save,fix_transformers_fully_masked_rows,fix_transformers_flash_attention_mrope_packed_sequence,+ fix_transformers_flash_attention_mrope_packed_rows,fix_transformers_untrusted_config_fields,fix_transformers_chat_template_path_traversal,fix_transformers_tokenizer_config_vocab_file_override,@@ -400,6 +401,7 @@ fix_transformers5_bare_annotation_configs()# Probe-gated: no-op unless SDPA gets a query row that attends to nothing; runs before model imports so plain generate is covered.fix_transformers_fully_masked_rows()fix_transformers_flash_attention_mrope_packed_sequence()+fix_transformers_flash_attention_mrope_packed_rows()fix_transformers_chunked_mask_block_sequence_ids()fix_transformers_flex_mask_graph_breaks()# CVE-2026-4372 / 5241 / 9856 and the tokenizer_config.json vocab path traversal, no-ops oncediff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex c3b718a027a..2f1c8294c80 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -2125,6 +2125,75 @@ def fix_transformers_flash_attention_mrope_packed_sequence():)+_PACKED_ROWS_PATCH_FLAG = "_unsloth_patched_mrope_packed_rows"+_PACKED_ROWS_CHECKED = [None, None, False]+++def _packed_row_varlen_kwargs(query, attention_mask, kwargs):+ """cu_seqlens from an Unsloth packed row whose layers got mRoPE ids, else None."""+ if kwargs.get("packed_seq_lengths") is None:+ return None+ import torch++ position_ids = kwargs.get("position_ids")+ if (+ attention_mask is not None+ or kwargs.get("cu_seq_lens_q") is not None+ or not isinstance(position_ids, torch.Tensor)+ or position_ids.dim() <= 2+ or query.dim() != 4+ or query.shape[0] != 1+ ):+ return None+ from .utils.packing import get_packed_info_from_kwargs++ info = get_packed_info_from_kwargs(kwargs, query.device)+ if info is None:+ return None+ _, cu_seqlens, max_seqlen = info+ # One host read per batch, not per layer: rows that do not cover the query keep the original path.+ if not torch.compiler.is_compiling():+ checked = _PACKED_ROWS_CHECKED+ if checked[0] is not cu_seqlens or checked[1] != query.shape[1]:+ checked[:] = [cu_seqlens, query.shape[1], int(cu_seqlens[-1]) == query.shape[1]]+ if not checked[2]:+ return None+ return {+ "cu_seq_lens_q": cu_seqlens,+ "cu_seq_lens_k": cu_seqlens,+ "max_length_q": max_seqlen,+ "max_length_k": max_seqlen,+ }+++def fix_transformers_flash_attention_mrope_packed_rows():+ """Qwen3.5 / Qwen3.5-MoE on transformers 5.3 give decoder layers 3-row mRoPE ids, which cannot mark+ document starts: take a packed row's boundaries from `packed_seq_lengths` instead."""+ try:+ from transformers.integrations import flash_attention as fa_integration+ except Exception:+ return+ current = getattr(fa_integration, "_flash_attention_forward", None)+ if current is None or getattr(current, _PACKED_ROWS_PATCH_FLAG, False):+ return++ @functools.wraps(current)+ def _flash_attention_forward(+ query_states, key_states, value_states, attention_mask, *args, **kwargs+ ):+ varlen = _packed_row_varlen_kwargs(query_states, attention_mask, kwargs)+ if varlen is not None:+ kwargs.update(varlen)+ return current(query_states, key_states, value_states, attention_mask, *args, **kwargs)++ setattr(_flash_attention_forward, _PACKED_ROWS_PATCH_FLAG, True)+ fa_integration._flash_attention_forward = _flash_attention_forward+ logger.info(+ "Unsloth: Patching transformers flash attention so packed rows with mRoPE layer "+ "position ids keep their documents apart"+ )++_CHUNKED_MASK_PATCH_FLAG = "_unsloth_patched_chunked_block_sequence_ids"_BLOCK_SEQUENCE_IDS = "block_sequence_ids"