Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep packed documents apart on Unsloth's own models (flash path, eval, cached eval) and match transformers' Mistral window (#13313)
Daniel Hanbef1c3dcf6f7
diff --git a/tests/utils/test_packed_flash_documents.py b/tests/utils/test_packed_flash_documents.pynew file mode 100644index 00000000000..c16c307970f--- /dev/null+++ b/tests/utils/test_packed_flash_documents.py@@ -0,0 +1,199 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""Packed rows on Unsloth's own models keep their documents apart on the flash-attn path, in training and+in eval (where use_cache stays on), and Mistral's window matches transformers' (k > q - sliding_window)."""++import pytest+import torch+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py++pytestmark = pytest.mark.skipif(+ not has_real_cuda(), reason = "Unsloth's model forwards need a CUDA device"+)++WINDOW = 4+LENGTHS = [7, 9, 5]+++def _reference(q, k, v, causal, window_size, softmax_scale):+ T, H, D = q.shape+ k = k.repeat_interleave(H // k.shape[1], dim = 1)+ v = v.repeat_interleave(H // v.shape[1], dim = 1)+ scale = D**-0.5 if softmax_scale is None else softmax_scale+ scores = torch.einsum("qhd,khd->hqk", q.float(), k.float()) * scale+ qi = torch.arange(T, device = q.device)[:, None]+ ki = torch.arange(T, device = q.device)[None, :]+ keep = torch.ones(T, T, dtype = torch.bool, device = q.device)+ if causal:+ keep &= ki <= qi+ if window_size[0] >= 0:+ keep &= qi - ki <= window_size[0]+ scores = scores.masked_fill(~keep, float("-inf"))+ return torch.einsum("hqk,khd->qhd", scores.softmax(-1), v.float()).to(q.dtype)+++def _flash_attn_func(+ q,+ k,+ v,+ dropout_p = 0.0,+ softmax_scale = None,+ causal = False,+ window_size = (-1, -1),+ **kwargs,+):+ return torch.stack(+ [+ _reference(q[b], k[b], v[b], causal, window_size, softmax_scale)+ for b in range(q.shape[0])+ ]+ )+++def _flash_attn_varlen_func(+ q,+ k,+ v,+ cu_q,+ cu_k,+ max_q,+ max_k,+ dropout_p = 0.0,+ softmax_scale = None,+ causal = False,+ window_size = (-1, -1),+ **kwargs,+):+ out = torch.empty_like(q)+ cu = cu_q.tolist()+ for start, end in zip(cu[:-1], cu[1:]):+ out[start:end] = _reference(+ q[start:end], k[start:end], v[start:end], causal, window_size, softmax_scale+ )+ return out++[email protected](scope = "module")+def model():+ from unsloth import FastLanguageModel++ try:+ model, _ = FastLanguageModel.from_pretrained(+ model_name = "hf-internal-testing/tiny-random-MistralForCausalLM",+ max_seq_length = 64,+ load_in_4bit = False,+ dtype = torch.bfloat16,+ )+ except OSError as exc: # pragma: no cover - offline CI+ pytest.skip(f"Requires access to the tiny Mistral checkpoint: {exc}")+ model.config.sliding_window = WINDOW+ for layer in model.model.layers:+ layer.self_attn.config.sliding_window = WINDOW+ model.train()+ return model+++def _logits(model, ids, **kwargs):+ with torch.no_grad():+ return model(input_ids = ids, **kwargs).logits.float()+++def _use_fake_flash(monkeypatch):+ from unsloth.utils import attention_dispatch as ad++ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", True)+ monkeypatch.setattr(ad, "flash_attn_func", _flash_attn_func)+ monkeypatch.setattr(ad, "flash_attn_varlen_func", _flash_attn_varlen_func)+++def fake_flash(monkeypatch):+ _use_fake_flash(monkeypatch)+++def test_packed_documents_do_not_attend_across_each_other(model, fake_flash):+ torch.manual_seed(0)+ docs = [torch.randint(5, 100, (n,), device = "cuda") for n in LENGTHS]+ packed = torch.cat(docs)[None]+ position_ids = torch.cat([torch.arange(n, device = "cuda") for n in LENGTHS])[None]+ out = _logits(+ model,+ packed,+ position_ids = position_ids,+ packed_seq_lengths = torch.tensor(LENGTHS, dtype = torch.int32),+ )+ alone = torch.cat([_logits(model, d[None]) for d in docs], dim = 1)+ torch.testing.assert_close(out, alone, atol = 2e-2, rtol = 2e-2)+++def test_flash_window_matches_sdpa(model, monkeypatch):+ # One document longer than the window: dense flash and SDPA must drop the same keys.+ from unsloth.utils import attention_dispatch as ad++ torch.manual_seed(1)+ ids = torch.randint(5, 100, (1, 12), device = "cuda")+ sdpa = _logits(model, ids)+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", True)+ monkeypatch.setattr(ad, "flash_attn_func", _flash_attn_func)+ monkeypatch.setattr(ad, "flash_attn_varlen_func", _flash_attn_varlen_func)+ torch.testing.assert_close(_logits(model, ids), sdpa, atol = 2e-2, rtol = 2e-2)++[email protected]("window", [WINDOW, None])[email protected]("flash", [False, True])+def test_packed_eval_keeps_documents_apart(model, monkeypatch, window, flash):+ # Eval builds a causal mask over the whole row unless the row is packed (padding-free eval batches are).+ torch.manual_seed(2)+ docs = [torch.randint(5, 100, (n,), device = "cuda") for n in LENGTHS]+ packed = torch.cat(docs)[None]+ position_ids = torch.cat([torch.arange(n, device = "cuda") for n in LENGTHS])[None]+ if flash:+ _use_fake_flash(monkeypatch)+ model.config.sliding_window = window+ model.eval()+ try:+ out = _logits(+ model,+ packed,+ position_ids = position_ids,+ use_cache = True,+ packed_seq_lengths = torch.tensor(LENGTHS, dtype = torch.int32),+ )+ alone = torch.cat([_logits(model, d[None]) for d in docs], dim = 1)+ finally:+ model.config.sliding_window = WINDOW+ model.train()+ torch.testing.assert_close(out, alone, atol = 2e-2, rtol = 2e-2)+++ "name",+ [+ "hf-internal-testing/tiny-random-LlamaForCausalLM",+ "trl-internal-testing/tiny-Qwen3ForCausalLM",+ ],+)+def test_packed_eval_with_cache_keeps_documents_apart_on_flash(name, monkeypatch):+ # Eval returns a cache, which must not turn a packed row into one dense flash call.+ from unsloth import FastLanguageModel++ try:+ other, _ = FastLanguageModel.from_pretrained(+ model_name = name, max_seq_length = 64, load_in_4bit = False, dtype = torch.bfloat16+ )+ except OSError as exc: # pragma: no cover - offline CI+ pytest.skip(f"Requires access to {name}: {exc}")+ _use_fake_flash(monkeypatch)+ other.eval()+ torch.manual_seed(3)+ vocab = other.config.vocab_size+ docs = [torch.randint(5, min(vocab, 100), (n,), device = "cuda") for n in LENGTHS]+ out = _logits(+ other,+ torch.cat(docs)[None],+ position_ids = torch.cat([torch.arange(n, device = "cuda") for n in LENGTHS])[None],+ use_cache = True,+ packed_seq_lengths = torch.tensor(LENGTHS, dtype = torch.int32),+ )+ alone = torch.cat([_logits(other, d[None], use_cache = True) for d in docs], dim = 1)+ torch.testing.assert_close(out, alone, atol = 2e-2, rtol = 2e-2)diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.pyindex c81e30550d9..d770f23c6ba 100644--- a/unsloth/models/cohere.py+++ b/unsloth/models/cohere.py@@ -136,9 +136,11 @@ def CohereAttention_fast_forward(if past_key_value is not None:K = torch.cat([past_key_value[0], K], dim = 2)V = torch.cat([past_key_value[1], V], dim = 2)+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).+ had_cache = past_key_value is not Nonepast_key_value = (K, V) if use_cache else None- use_varlen = seq_info is not None and past_key_value is None+ use_varlen = seq_info is not None and not had_cachebackend = select_attention_backend(use_varlen)attention_config = AttentionConfig(backend = backend,diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.pyindex 2175a3fed64..2c9302195a1 100644--- a/unsloth/models/falcon_h1.py+++ b/unsloth/models/falcon_h1.py@@ -131,14 +131,13 @@ def FalconH1Attention_fast_forward(if past_key_value is not None:K = torch.cat([past_key_value[0], K], dim = 2)V = torch.cat([past_key_value[1], V], dim = 2)+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).+ had_cache = past_key_value is not Nonepast_key_value = (K, V) if use_cache else Nonewindow = (-1, -1)use_varlen = (- attention_mask is None- and seq_info is not None- and past_key_value is None- and window == (-1, -1)+ attention_mask is None and seq_info is not None and not had_cache and window == (-1, -1))backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex 562328f6231..57a7bbb72d2 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -184,6 +184,8 @@ def Gemma2Attention_fast_forward(if past_key_value is not None:K = torch.cat([past_key_value[0], K], dim = 2)V = torch.cat([past_key_value[1], V], dim = 2)+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).+ had_cache = past_key_value is not Nonepast_key_value = (K, V) if use_cache else None# Only enable if the attention_mask is True.@@ -206,7 +208,7 @@ def Gemma2Attention_fast_forward(if not hasattr(self, "_flash_attention_softmax_scale"):self._flash_attention_softmax_scale = 1.0 / (self.config.query_pre_attn_scalar**0.5)- use_varlen = seq_info is not None and past_key_value is None+ use_varlen = seq_info is not None and not had_cacheattention_config = AttentionConfig(backend = select_attention_backend(use_varlen),diff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex 293378a5e73..51fc0c135bc 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -132,9 +132,11 @@ def GraniteAttention_fast_forward(if past_key_value is not None:K = torch.cat([past_key_value[0], K], dim = 2)V = torch.cat([past_key_value[1], V], dim = 2)+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).+ had_cache = past_key_value is not Nonepast_key_value = (K, V) if use_cache else None- use_varlen = attention_mask is None and seq_info is not None and past_key_value is None+ use_varlen = attention_mask is None and seq_info is not None and not had_cachebackend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 6abad517777..f6efd38db96 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -885,9 +885,11 @@ def LlamaAttention_fast_forward(if past_key_value is not None:K = torch.cat([past_key_value[0], K], dim = 2)V = torch.cat([past_key_value[1], V], dim = 2)+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).+ had_cache = past_key_value is not Nonepast_key_value = (K, V) if use_cache else None- use_varlen = seq_info is not None and past_key_value is None+ use_varlen = seq_info is not None and not had_cachebackend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)# Should dropout be hardcoded to 0.0?diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex e20471c851f..242879e10b0 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -110,6 +110,8 @@ def MistralAttention_fast_forward(if past_key_value is not None:K = torch.cat([past_key_value[0], K], dim = 2)V = torch.cat([past_key_value[1], V], dim = 2)+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).+ had_cache = past_key_value is not Nonepast_key_value = (K, V) if use_cache else Nonesw_cfg = getattr(self.config, "sliding_window", None)@@ -120,9 +122,11 @@ def MistralAttention_fast_forward(sw = kv_seq_lenelse:sw = sw_cfg- window_size = (-1, -1) if (kv_seq_len <= sw) else (sw, sw)+ # transformers' window: k > q - sliding_window, so flash's left window is sliding_window - 1.+ window_size = (-1, -1) if (kv_seq_len <= sw) else (sw - 1, sw - 1)- use_varlen = seq_info is not None and past_key_value is None and window_size == (-1, -1)+ # Packed rows always take varlen: dense flash would let a document attend into the one before it.+ use_varlen = seq_info is not None and not had_cachebackend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)attention_config = AttentionConfig(backend = backend,@@ -133,6 +137,7 @@ def MistralAttention_fast_forward("dropout_p": 0.0,"causal": True,"softmax_scale": getattr(self, "softmax_scale", None),+ "window_size": window_size,},)# PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /@@ -208,9 +213,10 @@ def MistralForCausalLM_fast_forward(# If attention_mask exists, it is handled in the attention forward.- elif self.training:+ elif self.training or kwargs.get("packed_seq_lengths", None) is not None:# LlamaModel_fast_forward's DPO embed-masking block needs the 2D attention_mask; it nulls the mask# before attention anyway, so leaving it 2D avoids a 4D conversion that crashes DPO.+ # Packed rows need no mask: a row-wide causal mask would let documents attend across.passelse:diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex 676bc11b228..a1ffbb49700 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -125,9 +125,11 @@ def Qwen3Attention_fast_forward(if past_key_value is not None:K = torch.cat([past_key_value[0], K], dim = 2)V = torch.cat([past_key_value[1], V], dim = 2)+ # Incoming cache; past_key_value is reassigned below (eval keeps use_cache on).+ had_cache = past_key_value is not Nonepast_key_value = (K, V) if use_cache else None- use_varlen = seq_info is not None and past_key_value is None+ use_varlen = seq_info is not None and not had_cachebackend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)attention_config = AttentionConfig(backend = backend,