Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep packed documents apart on Unsloth's own models (flash path, eval, cached eval) and match transformers' Mistral window (#13313)

Daniel Hanbef1c3dcf6f7
diff --git a/tests/utils/test_packed_flash_documents.py b/tests/utils/test_packed_flash_documents.py
new file mode 100644
index 00000000000..c16c307970f
--- /dev/null
+++ b/tests/utils/test_packed_flash_documents.py
@@ -0,0 +1,199 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""Packed rows on Unsloth's own models keep their documents apart on the flash-attn path, in training and
+in eval (where use_cache stays on), and Mistral's window matches transformers' (k > q - sliding_window)."""
+
+import pytest
+import torch
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+
+pytestmark = pytest.mark.skipif(
+ not has_real_cuda(), reason = "Unsloth's model forwards need a CUDA device"
+)
+
+WINDOW = 4
+LENGTHS = [7, 9, 5]
+
+
+def _reference(q, k, v, causal, window_size, softmax_scale):
+ T, H, D = q.shape
+ k = k.repeat_interleave(H // k.shape[1], dim = 1)
+ v = v.repeat_interleave(H // v.shape[1], dim = 1)
+ scale = D**-0.5 if softmax_scale is None else softmax_scale
+ scores = torch.einsum("qhd,khd->hqk", q.float(), k.float()) * scale
+ qi = torch.arange(T, device = q.device)[:, None]
+ ki = torch.arange(T, device = q.device)[None, :]
+ keep = torch.ones(T, T, dtype = torch.bool, device = q.device)
+ if causal:
+ keep &= ki <= qi
+ if window_size[0] >= 0:
+ keep &= qi - ki <= window_size[0]
+ scores = scores.masked_fill(~keep, float("-inf"))
+ return torch.einsum("hqk,khd->qhd", scores.softmax(-1), v.float()).to(q.dtype)
+
+
+def _flash_attn_func(
+ q,
+ k,
+ v,
+ dropout_p = 0.0,
+ softmax_scale = None,
+ causal = False,
+ window_size = (-1, -1),
+ **kwargs,
+):
+ return torch.stack(
+ [
+ _reference(q[b], k[b], v[b], causal, window_size, softmax_scale)
+ for b in range(q.shape[0])
+ ]
+ )
+
+
+def _flash_attn_varlen_func(
+ q,
+ k,
+ v,
+ cu_q,
+ cu_k,
+ max_q,
+ max_k,
+ dropout_p = 0.0,
+ softmax_scale = None,
+ causal = False,
+ window_size = (-1, -1),
+ **kwargs,
+):
+ out = torch.empty_like(q)
+ cu = cu_q.tolist()
+ for start, end in zip(cu[:-1], cu[1:]):
+ out[start:end] = _reference(
+ q[start:end], k[start:end], v[start:end], causal, window_size, softmax_scale
+ )
+ return out
+
+
[email protected](scope = "module")
+def model():
+ from unsloth import FastLanguageModel
+
+ try:
+ model, _ = FastLanguageModel.from_pretrained(
+ model_name = "hf-internal-testing/tiny-random-MistralForCausalLM",
+ max_seq_length = 64,
+ load_in_4bit = False,
+ dtype = torch.bfloat16,
+ )
+ except OSError as exc: # pragma: no cover - offline CI
+ pytest.skip(f"Requires access to the tiny Mistral checkpoint: {exc}")
+ model.config.sliding_window = WINDOW
+ for layer in model.model.layers:
+ layer.self_attn.config.sliding_window = WINDOW
+ model.train()
+ return model
+
+
+def _logits(model, ids, **kwargs):
+ with torch.no_grad():
+ return model(input_ids = ids, **kwargs).logits.float()
+
+
+def _use_fake_flash(monkeypatch):
+ from unsloth.utils import attention_dispatch as ad
+
+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", True)
+ monkeypatch.setattr(ad, "flash_attn_func", _flash_attn_func)
+ monkeypatch.setattr(ad, "flash_attn_varlen_func", _flash_attn_varlen_func)
+
+
+def fake_flash(monkeypatch):
+ _use_fake_flash(monkeypatch)
+
+
+def test_packed_documents_do_not_attend_across_each_other(model, fake_flash):
+ torch.manual_seed(0)
+ docs = [torch.randint(5, 100, (n,), device = "cuda") for n in LENGTHS]
+ packed = torch.cat(docs)[None]
+ position_ids = torch.cat([torch.arange(n, device = "cuda") for n in LENGTHS])[None]
+ out = _logits(
+ model,
+ packed,
+ position_ids = position_ids,
+ packed_seq_lengths = torch.tensor(LENGTHS, dtype = torch.int32),
+ )
+ alone = torch.cat([_logits(model, d[None]) for d in docs], dim = 1)
+ torch.testing.assert_close(out, alone, atol = 2e-2, rtol = 2e-2)
+
+
+def test_flash_window_matches_sdpa(model, monkeypatch):
+ # One document longer than the window: dense flash and SDPA must drop the same keys.
+ from unsloth.utils import attention_dispatch as ad
+
+ torch.manual_seed(1)
+ ids = torch.randint(5, 100, (1, 12), device = "cuda")
+ sdpa = _logits(model, ids)
+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", True)
+ monkeypatch.setattr(ad, "flash_attn_func", _flash_attn_func)
+ monkeypatch.setattr(ad, "flash_attn_varlen_func", _flash_attn_varlen_func)
+ torch.testing.assert_close(_logits(model, ids), sdpa, atol = 2e-2, rtol = 2e-2)
+
+
[email protected]("window", [WINDOW, None])
[email protected]("flash", [False, True])
+def test_packed_eval_keeps_documents_apart(model, monkeypatch, window, flash):
+ # Eval builds a causal mask over the whole row unless the row is packed (padding-free eval batches are).
+ torch.manual_seed(2)
+ docs = [torch.randint(5, 100, (n,), device = "cuda") for n in LENGTHS]
+ packed = torch.cat(docs)[None]
+ position_ids = torch.cat([torch.arange(n, device = "cuda") for n in LENGTHS])[None]
+ if flash:
+ _use_fake_flash(monkeypatch)
+ model.config.sliding_window = window
+ model.eval()
+ try:
+ out = _logits(
+ model,
+ packed,
+ position_ids = position_ids,
+ use_cache = True,
+ packed_seq_lengths = torch.tensor(LENGTHS, dtype = torch.int32),
+ )
+ alone = torch.cat([_logits(model, d[None]) for d in docs], dim = 1)
+ finally:
+ model.config.sliding_window = WINDOW
+ model.train()
+ torch.testing.assert_close(out, alone, atol = 2e-2, rtol = 2e-2)
+
+
+ "name",
+ [
+ "hf-internal-testing/tiny-random-LlamaForCausalLM",
+ "trl-internal-testing/tiny-Qwen3ForCausalLM",
+ ],
+)
+def test_packed_eval_with_cache_keeps_documents_apart_on_flash(name, monkeypatch):
+ # Eval returns a cache, which must not turn a packed row into one dense flash call.
+ from unsloth import FastLanguageModel
+
+ try:
+ other, _ = FastLanguageModel.from_pretrained(
+ model_name = name, max_seq_length = 64, load_in_4bit = False, dtype = torch.bfloat16
+ )
+ except OSError as exc: # pragma: no cover - offline CI
+ pytest.skip(f"Requires access to {name}: {exc}")
+ _use_fake_flash(monkeypatch)
+ other.eval()
+ torch.manual_seed(3)
+ vocab = other.config.vocab_size
+ docs = [torch.randint(5, min(vocab, 100), (n,), device = "cuda") for n in LENGTHS]
+ out = _logits(
+ other,
+ torch.cat(docs)[None],
+ position_ids = torch.cat([torch.arange(n, device = "cuda") for n in LENGTHS])[None],
+ use_cache = True,
+ packed_seq_lengths = torch.tensor(LENGTHS, dtype = torch.int32),
+ )
+ alone = torch.cat([_logits(other, d[None], use_cache = True) for d in docs], dim = 1)
+ torch.testing.assert_close(out, alone, atol = 2e-2, rtol = 2e-2)
diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py
index c81e30550d9..d770f23c6ba 100644
--- a/unsloth/models/cohere.py
+++ b/unsloth/models/cohere.py
@@ -136,9 +136,11 @@ def CohereAttention_fast_forward(
if past_key_value is not None:
K = torch.cat([past_key_value[0], K], dim = 2)
V = torch.cat([past_key_value[1], V], dim = 2)
+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).
+ had_cache = past_key_value is not None
past_key_value = (K, V) if use_cache else None
- use_varlen = seq_info is not None and past_key_value is None
+ use_varlen = seq_info is not None and not had_cache
backend = select_attention_backend(use_varlen)
attention_config = AttentionConfig(
backend = backend,
diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py
index 2175a3fed64..2c9302195a1 100644
--- a/unsloth/models/falcon_h1.py
+++ b/unsloth/models/falcon_h1.py
@@ -131,14 +131,13 @@ def FalconH1Attention_fast_forward(
if past_key_value is not None:
K = torch.cat([past_key_value[0], K], dim = 2)
V = torch.cat([past_key_value[1], V], dim = 2)
+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).
+ had_cache = past_key_value is not None
past_key_value = (K, V) if use_cache else None
window = (-1, -1)
use_varlen = (
- attention_mask is None
- and seq_info is not None
- and past_key_value is None
- and window == (-1, -1)
+ attention_mask is None and seq_info is not None and not had_cache and window == (-1, -1)
)
backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index 562328f6231..57a7bbb72d2 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -184,6 +184,8 @@ def Gemma2Attention_fast_forward(
if past_key_value is not None:
K = torch.cat([past_key_value[0], K], dim = 2)
V = torch.cat([past_key_value[1], V], dim = 2)
+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).
+ had_cache = past_key_value is not None
past_key_value = (K, V) if use_cache else None
# Only enable if the attention_mask is True.
@@ -206,7 +208,7 @@ def Gemma2Attention_fast_forward(
if not hasattr(self, "_flash_attention_softmax_scale"):
self._flash_attention_softmax_scale = 1.0 / (self.config.query_pre_attn_scalar**0.5)
- use_varlen = seq_info is not None and past_key_value is None
+ use_varlen = seq_info is not None and not had_cache
attention_config = AttentionConfig(
backend = select_attention_backend(use_varlen),
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index 293378a5e73..51fc0c135bc 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -132,9 +132,11 @@ def GraniteAttention_fast_forward(
if past_key_value is not None:
K = torch.cat([past_key_value[0], K], dim = 2)
V = torch.cat([past_key_value[1], V], dim = 2)
+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).
+ had_cache = past_key_value is not None
past_key_value = (K, V) if use_cache else None
- use_varlen = attention_mask is None and seq_info is not None and past_key_value is None
+ use_varlen = attention_mask is None and seq_info is not None and not had_cache
backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 6abad517777..f6efd38db96 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -885,9 +885,11 @@ def LlamaAttention_fast_forward(
if past_key_value is not None:
K = torch.cat([past_key_value[0], K], dim = 2)
V = torch.cat([past_key_value[1], V], dim = 2)
+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).
+ had_cache = past_key_value is not None
past_key_value = (K, V) if use_cache else None
- use_varlen = seq_info is not None and past_key_value is None
+ use_varlen = seq_info is not None and not had_cache
backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)
# Should dropout be hardcoded to 0.0?
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index e20471c851f..242879e10b0 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -110,6 +110,8 @@ def MistralAttention_fast_forward(
if past_key_value is not None:
K = torch.cat([past_key_value[0], K], dim = 2)
V = torch.cat([past_key_value[1], V], dim = 2)
+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).
+ had_cache = past_key_value is not None
past_key_value = (K, V) if use_cache else None
sw_cfg = getattr(self.config, "sliding_window", None)
@@ -120,9 +122,11 @@ def MistralAttention_fast_forward(
sw = kv_seq_len
else:
sw = sw_cfg
- window_size = (-1, -1) if (kv_seq_len <= sw) else (sw, sw)
+ # transformers' window: k > q - sliding_window, so flash's left window is sliding_window - 1.
+ window_size = (-1, -1) if (kv_seq_len <= sw) else (sw - 1, sw - 1)
- use_varlen = seq_info is not None and past_key_value is None and window_size == (-1, -1)
+ # Packed rows always take varlen: dense flash would let a document attend into the one before it.
+ use_varlen = seq_info is not None and not had_cache
backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)
attention_config = AttentionConfig(
backend = backend,
@@ -133,6 +137,7 @@ def MistralAttention_fast_forward(
"dropout_p": 0.0,
"causal": True,
"softmax_scale": getattr(self, "softmax_scale", None),
+ "window_size": window_size,
},
)
# PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /
@@ -208,9 +213,10 @@ def MistralForCausalLM_fast_forward(
# If attention_mask exists, it is handled in the attention forward.
- elif self.training:
+ elif self.training or kwargs.get("packed_seq_lengths", None) is not None:
# LlamaModel_fast_forward's DPO embed-masking block needs the 2D attention_mask; it nulls the mask
# before attention anyway, so leaving it 2D avoids a 4D conversion that crashes DPO.
+ # Packed rows need no mask: a row-wide causal mask would let documents attend across.
pass
else:
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index 676bc11b228..a1ffbb49700 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -125,9 +125,11 @@ def Qwen3Attention_fast_forward(
if past_key_value is not None:
K = torch.cat([past_key_value[0], K], dim = 2)
V = torch.cat([past_key_value[1], V], dim = 2)
+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).
+ had_cache = past_key_value is not None
past_key_value = (K, V) if use_cache else None
- use_varlen = seq_info is not None and past_key_value is None
+ use_varlen = seq_info is not None and not had_cache
backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)
attention_config = AttentionConfig(
backend = backend,