Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Gemma 4 packed SFT: efficient kernel for the 512-wide global layers, processor on text-only data (#13384)
Daniel Han37f908833a40
diff --git a/tests/test_flex_gemma4_sliding_scope.py b/tests/test_flex_gemma4_sliding_scope.pyindex 3dca3852cbe..b9193e6dd54 100644--- a/tests/test_flex_gemma4_sliding_scope.py+++ b/tests/test_flex_gemma4_sliding_scope.py@@ -51,6 +51,20 @@ def test_gemma4_text_only_is_not_routed_either():assert u._prefers_flex_for_head_dim(text_only) is False[email protected]("top", ["gemma4_unified", None])+def test_gemma4_unified_is_not_routed_either(top):+ text = _Cfg(+ model_type = "gemma4_unified_text",+ head_dim = 256,+ global_head_dim = 512,+ sliding_window = 1024,+ num_attention_heads = 16,+ )+ cfg = text if top is None else _Cfg(model_type = top, text_config = text)+ assert u._text_attention_head_dim(cfg) == 512+ assert u._prefers_flex_for_head_dim(cfg) is False++def test_the_head_dim_is_still_large_so_this_really_is_the_exclusion_talking():assert u._text_attention_head_dim(_gemma4()) == 512diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.pyindex 5c0377bcd39..2ddb796fe47 100644--- a/tests/test_hf_packed_attention.py+++ b/tests/test_hf_packed_attention.py@@ -179,7 +179,7 @@ _TORCH_VARLEN_HOST = pytest.mark.skipif("dropout","batched","cpu",- "head_dim_512",+ "head_dim_640","mla_value_width","autocast_dtype","paged_cache",@@ -228,8 +228,9 @@ def test_packed_calls_that_varlen_cannot_express_fall_back(monkeypatch, why):B = 2elif why == "cpu":device, mask = "cpu", _block_causal(lengths)- elif why == "head_dim_512":- D = v_width = 512+ elif why == "head_dim_640":+ # Past both flash (256) and the efficient route's tested width (512).+ D = v_width = 640elif why == "mla_value_width":D, v_width = 192, 128elif why == "paged_cache":@@ -670,3 +671,443 @@ def test_tiny_mistral_packed_row_matches_the_masked_path(monkeypatch, window):torch.testing.assert_close(fast_logits, ref_logits, atol = 3e-2, rtol = 3e-2)for a, b in zip(fast_grads, ref_grads):torch.testing.assert_close(a, b, atol = 3e-2, rtol = 5e-2)+++# Gemma-4 global layers: head_dim 512, past every flash kernel, take SDPA's efficient kernel over cu_seqlens.+def _efficient_varlen_op() -> bool:+ try:+ forward = torch.ops.aten._efficient_attention_forward.default._schema+ backward = torch.ops.aten._efficient_attention_backward.default._schema+ except Exception:+ return False+ return {"cu_seqlens_q", "scale", "window_size"} <= {a.name for a in forward.arguments} and (+ "window_size" in {a.name for a in backward.arguments}+ )+++_EFFICIENT_HOST = pytest.mark.skipif(+ torch.version.hip is not None or not _efficient_varlen_op(),+ reason = "the efficient varlen route is off on ROCm and on torch without its varlen schema",+)+++def _efficient_fresh(monkeypatch):+ monkeypatch.setattr(hpa, "_EFFICIENT_VARLEN", [])+ monkeypatch.setattr(hpa, "_EFFICIENT_DEVICES", {})+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ monkeypatch.delenv(hpa._EFFICIENT_ENV, raising = False)+++def _wide_inputs(+ lengths,+ H,+ n_kv,+ D,+ scale = 0.1,+):+ T = sum(lengths)++ def make(heads):+ x = torch.randn(1, heads, T, D, device = "cuda", dtype = torch.bfloat16) * scale+ return x.requires_grad_()++ return make(H), make(n_kv), make(n_kv)+++@_CUDA+@_EFFICIENT_HOST[email protected]("window", [None, 5])[email protected]("n_kv", [8, 2, 1])+def test_wide_heads_match_sdpa_over_the_dense_mask(_efficient_fresh, n_kv, window):+ """Output and dQ / dK / dV equal transformers' sdpa over the real mask, Gemma-4 scaling 1.0."""+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ hpa._ORIG_SDPA[0] = sdpa_attention_forward+ torch.manual_seed(0)+ lengths = [37, 100, 2, 5, 230]+ H, D = 8, 512+ module = types.SimpleNamespace(+ is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = window+ )+ q, k, v = _wide_inputs(lengths, H, n_kv, D)+ mask = (+ _block_causal(lengths, "cuda")+ if window is None+ else _hf_window_mask(lengths, window, "cuda")+ )+ psl = torch.tensor(lengths, dtype = torch.int32)+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ fast, _ = hpa._sdpa_packed_varlen(module, q, k, v, mask, scaling = 1.0, packed_seq_lengths = psl)+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"] + 1+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"] + 1+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 1.0)+ assert fast.shape == ref.shape == (1, sum(lengths), H, D)+ torch.testing.assert_close(fast.float(), ref.float(), atol = 1e-2, rtol = 1e-2)+ g = torch.randn_like(ref)+ for a, b in zip(+ torch.autograd.grad(fast, (q, k, v), g), torch.autograd.grad(ref, (q, k, v), g)+ ):+ torch.testing.assert_close(a.float(), b.float(), atol = 2e-2, rtol = 2e-2)+++@_CUDA+@_EFFICIENT_HOST+def test_wide_heads_keep_documents_apart(_efficient_fresh):+ """Each document of the packed row equals that document attended alone."""+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ hpa._ORIG_SDPA[0] = sdpa_attention_forward+ torch.manual_seed(1)+ lengths = [60, 3, 130]+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 4, training = True)+ q, k, v = _wide_inputs(lengths, 8, 2, 512)+ fast, _ = hpa._sdpa_packed_varlen(+ module,+ q,+ k,+ v,+ _block_causal(lengths, "cuda"),+ scaling = 1.0,+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),+ )+ start = 0+ for n in lengths:+ sl = slice(start, start + n)+ alone, _ = sdpa_attention_forward(+ module, q[:, :, sl], k[:, :, sl], v[:, :, sl], None, scaling = 1.0, is_causal = True+ )+ torch.testing.assert_close(fast[:, sl].float(), alone.float(), atol = 1e-2, rtol = 1e-2)+ start += n+++@_CUDA+@_EFFICIENT_HOST+ "why",+ [+ "none",+ "cross_document",+ "wrong_window",+ "softcap",+ "sinks",+ "dropout",+ "fp32",+ "head_dim_640",+ "kill_switch",+ "deterministic",+ "math_only",+ "efficient_disabled",+ "no_op",+ ],+)+def test_wide_heads_fall_back_to_the_wrapped_sdpa(_efficient_fresh, monkeypatch, why):+ """Each reason alone sends a head_dim 512 call to the wrapped sdpa; "none" is the control."""+ import contextlib+ from torch.nn.attention import SDPBackend, sdpa_kernel++ seen, orig = _calls()+ hpa._ORIG_SDPA[0] = orig+ lengths, D, dtype = [5, 7], 512, torch.bfloat16+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 2)+ mask, dropout, context = _block_causal(lengths, "cuda"), 0.0, contextlib.nullcontext()+ if why == "cross_document":+ mask = mask.clone()+ mask[0, 0, 6, 1] = True+ elif why == "wrong_window":+ module.sliding_window = 3 # the mask has no band+ elif why == "softcap":+ module.config = types.SimpleNamespace(attn_logit_softcapping = 50.0)+ elif why == "sinks":+ module.sinks = torch.zeros(4)+ elif why == "dropout":+ dropout = 0.1+ elif why == "fp32":+ dtype = torch.float32+ elif why == "head_dim_640":+ D = 640+ elif why == "kill_switch":+ monkeypatch.setenv(hpa._EFFICIENT_ENV, "0")+ elif why == "math_only":+ context = sdpa_kernel([SDPBackend.MATH])+ elif why == "efficient_disabled":+ context = sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.MATH])+ elif why == "no_op":+ monkeypatch.setattr(hpa, "_EFFICIENT_VARLEN", [False])+ q = torch.randn(1, 4, 12, D, device = "cuda", dtype = dtype)+ k = torch.randn(1, 2, 12, D, device = "cuda", dtype = dtype)+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ deterministic = torch.are_deterministic_algorithms_enabled()+ if why == "deterministic":+ torch.use_deterministic_algorithms(True)+ try:+ with context:+ out = hpa._sdpa_packed_varlen(+ module,+ q,+ k,+ k,+ mask,+ dropout = dropout,+ scaling = 1.0,+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),+ )+ finally:+ torch.use_deterministic_algorithms(deterministic)+ if why == "none":+ assert out[0].shape == (1, 12, 4, D) and seen == []+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"] + 1+ else:+ assert out == ("orig", None)+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"]+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1+++@_CUDA+@_EFFICIENT_HOST+def test_a_refusing_efficient_kernel_hands_the_call_back(_efficient_fresh, monkeypatch):+ seen, orig = _calls()+ hpa._ORIG_SDPA[0] = orig++ def refuse(*args, **kwargs):+ raise RuntimeError("No available kernel.")++ monkeypatch.setattr(hpa._EfficientVarlenAttention, "apply", refuse)+ lengths = [3, 3]+ q = torch.randn(1, 4, 6, 512, device = "cuda", dtype = torch.bfloat16)+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ out = hpa._sdpa_packed_varlen(+ types.SimpleNamespace(is_causal = True),+ q,+ q,+ q,+ _block_causal(lengths, "cuda"),+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),+ )+ assert out == ("orig", None) and len(seen) == 1+ assert hpa._EFFICIENT_VARLEN == [False]+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"]+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1+++@_CUDA+@_EFFICIENT_HOST+def test_the_efficient_kill_switch_is_read_on_every_call(_efficient_fresh, monkeypatch):+ q = torch.randn(1, 4, 12, 512, device = "cuda", dtype = torch.bfloat16)+ assert hpa._efficient_varlen_takes(q, None)+ monkeypatch.setenv(hpa._EFFICIENT_ENV, "0")+ assert not hpa._efficient_varlen_takes(q, None)+ monkeypatch.delenv(hpa._EFFICIENT_ENV)+ assert hpa._efficient_varlen_takes(q, None)+++@_CUDA+@_EFFICIENT_HOST+def test_a_non_contiguous_head_dim_is_copied_not_refused(_efficient_fresh):+ cu = torch.tensor([0, 5, 12], device = "cuda", dtype = torch.int32)+ base = torch.randn(1, 4, 12, 1024, device = "cuda", dtype = torch.bfloat16)+ q = base[..., ::2] # head dim 512 with stride 2+ assert q.stride(-1) == 2+ out = hpa._efficient_varlen(q, q, q, cu, 7, 512**-0.5, None)+ ref = hpa._efficient_varlen(+ q.contiguous(), q.contiguous(), q.contiguous(), cu, 7, 512**-0.5, None+ )+ torch.testing.assert_close(out, ref)+++@_CUDA+@_EFFICIENT_HOST+def test_a_cuda_fault_in_the_efficient_kernel_is_not_a_refusal(_efficient_fresh, monkeypatch):+ if hpa._ACCELERATOR_ERROR is None:+ pytest.skip("this torch has no AcceleratorError")+ seen, orig = _calls()+ hpa._ORIG_SDPA[0] = orig++ def fault(*args, **kwargs):+ raise hpa._ACCELERATOR_ERROR("CUDA error: an illegal memory access was encountered")++ monkeypatch.setattr(hpa._EfficientVarlenAttention, "apply", fault)+ lengths = [3, 3]+ q = torch.randn(1, 4, 6, 512, device = "cuda", dtype = torch.bfloat16)+ with pytest.raises(hpa._ACCELERATOR_ERROR):+ hpa._sdpa_packed_varlen(+ types.SimpleNamespace(is_causal = True),+ q,+ q,+ q,+ _block_causal(lengths, "cuda"),+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),+ )+ assert seen == [] and hpa._EFFICIENT_VARLEN != [False]+++@_CUDA+@_EFFICIENT_HOST[email protected]("backend", ["sdpa", "flash_varlen"])+def test_a_shape_the_flash_impl_refuses_takes_the_efficient_kernel(+ _efficient_fresh, monkeypatch, backend+):+ """FA4 on sm100 lacks hd256 + window: without flash-attn / xformers it takes the efficient kernel."""+ from transformers.integrations.sdpa_attention import sdpa_attention_forward+ from unsloth.utils import attention_dispatch as ad+ from unsloth.utils import flash_impl++ if hpa._torch_varlen(torch.device("cuda", 0)) is None:+ pytest.skip("no torch varlen_attn to refuse")+ monkeypatch.setattr(flash_impl, "flash_impl_allows", lambda head_dim, window, device: False)+ monkeypatch.setattr(ad, "select_attention_backend", lambda use_varlen = False: backend)+ dispatched = []+ monkeypatch.setattr(+ ad, "run_attention", lambda **kwargs: dispatched.append(1) or kwargs["Q"].transpose(1, 2)+ )+ hpa._ORIG_SDPA[0] = sdpa_attention_forward+ torch.manual_seed(2)+ lengths, window = [9, 20, 4], 6+ module = types.SimpleNamespace(+ is_causal = True, num_key_value_groups = 4, training = True, sliding_window = window+ )+ q, k, v = _wide_inputs(lengths, 8, 2, 256)+ mask = _hf_window_mask(lengths, window, "cuda")+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ fast, _ = hpa._sdpa_packed_varlen(+ module,+ q,+ k,+ v,+ mask,+ scaling = 1.0,+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),+ )+ if backend == "flash_varlen":+ assert dispatched == [1]+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"]+ return+ assert dispatched == []+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"] + 1+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 1.0)+ torch.testing.assert_close(fast.float(), ref.float(), atol = 1e-2, rtol = 1e-2)+ g = torch.randn_like(ref)+ for a, b in zip(+ torch.autograd.grad(fast, (q, k, v), g), torch.autograd.grad(ref, (q, k, v), g)+ ):+ torch.testing.assert_close(a.float(), b.float(), atol = 2e-2, rtol = 2e-2)+++@_CUDA+@_EFFICIENT_HOST+def test_wide_heads_under_reentrant_checkpoint(_efficient_fresh):+ from torch.utils.checkpoint import checkpoint+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ hpa._ORIG_SDPA[0] = sdpa_attention_forward+ lengths = [40, 24]+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 4, training = True)+ q, k, v = _wide_inputs(lengths, 8, 2, 512)+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)+ fn = lambda q, k, v: hpa._sdpa_packed_varlen(+ module, q, k, v, mask, scaling = 1.0, packed_seq_lengths = psl+ )[0]+ out = checkpoint(fn, q, k, v, use_reentrant = True)+ g = torch.randn_like(out)+ out.backward(g)+ grads = [x.grad for x in (q, k, v)]+ ref = sdpa_attention_forward(module, q, k, v, mask, scaling = 1.0)[0]+ torch.testing.assert_close(out.float(), ref.float(), atol = 1e-2, rtol = 1e-2)+ for a, b in zip(grads, torch.autograd.grad(ref, (q, k, v), g)):+ torch.testing.assert_close(a.float(), b.float(), atol = 2e-2, rtol = 2e-2)+++@_CUDA+@_EFFICIENT_HOST+def test_wide_heads_leave_a_compiled_caller_alone(_efficient_fresh):+ """Under fullgraph compile the wrapper returns the wrapped sdpa: no graph break, no counters."""+ hpa._ORIG_SDPA[0] = _plain_sdpa+ lengths = [3, 5]+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)+ q = torch.randn(1, 4, 8, 512, device = "cuda", dtype = torch.bfloat16)+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ fn = torch.compile(+ lambda q: hpa._sdpa_packed_varlen(+ module, q, q, q, mask, scaling = 1.0, packed_seq_lengths = psl+ )[0],+ backend = "eager",+ fullgraph = True,+ )+ torch.testing.assert_close(fn(q), _plain_sdpa(module, q, q, q, mask, scaling = 1.0)[0])+ assert hpa.HF_PACKED_ATTENTION_STATS == before+++@_CUDA+@_EFFICIENT_HOST[email protected]("k_eq_v", [False, True])+def test_tiny_gemma4_packed_row_matches_each_document(_efficient_fresh, monkeypatch, k_eq_v):+ """Gemma4 global_head_dim 512: efficient kernel; logits / grads match sdpa and each doc alone."""+ transformers = pytest.importorskip("transformers")+ if not hasattr(transformers, "Gemma4TextConfig"):+ pytest.skip("this transformers has no Gemma 4")+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS++ hpa._ORIG_SDPA[0] = None+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ # transformers' own entry: an earlier test can leave the installed wrapper in the mapping.+ monkeypatch.setitem(ALL_ATTENTION_FUNCTIONS, "sdpa", sdpa_attention_forward)+ assert hpa.enable_hf_packed_attention()+ torch.manual_seed(0)+ config = transformers.Gemma4TextConfig(+ vocab_size = 128,+ vocab_size_per_layer_input = 128,+ hidden_size = 64,+ hidden_size_per_layer_input = 8,+ intermediate_size = 128,+ num_hidden_layers = 2,+ layer_types = ["sliding_attention", "full_attention"],+ num_attention_heads = 4,+ num_key_value_heads = 2,+ num_global_key_value_heads = 1 if k_eq_v else None,+ attention_k_eq_v = k_eq_v,+ head_dim = 256,+ global_head_dim = 512,+ sliding_window = 4,+ attn_implementation = "sdpa",+ )+ model = transformers.Gemma4ForCausalLM(config).cuda().to(torch.bfloat16)+ global_attn = model.model.layers[1].self_attn+ assert global_attn.head_dim == 512 and model.model.layers[0].self_attn.head_dim == 256+ lengths = [3, 11, 7]+ ids = torch.randint(0, 128, (1, sum(lengths)), device = "cuda")+ position_ids = torch.cat([torch.arange(n) for n in lengths])[None].cuda()+ psl = torch.tensor(lengths, dtype = torch.int32)++ def run():+ model.zero_grad()+ out = model(+ input_ids = ids, position_ids = position_ids, packed_seq_lengths = psl, use_cache = False+ )+ out.logits.float().square().mean().backward()+ return out.logits.detach().float(), [+ p.grad.float() for p in model.parameters() if p.grad is not None+ ]++ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ fast_logits, fast_grads = run()+ stats = hpa.HF_PACKED_ATTENTION_STATS+ assert stats["efficient"] == before["efficient"] + 1+ # Without torch varlen (torch 2.10 and older) the windowed sliding layer may have no varlen route.+ assert stats["fast"] + stats["fallback"] == before["fast"] + before["fallback"] + 2+ if hpa._torch_varlen(ids.device) is not None:+ assert stats["fast"] == before["fast"] + 2+ monkeypatch.setenv("UNSLOTH_HF_PACKED_VARLEN", "0")+ ref_logits, ref_grads = run()+ torch.testing.assert_close(fast_logits, ref_logits, atol = 3e-2, rtol = 3e-2)+ for a, b in zip(fast_grads, ref_grads):+ torch.testing.assert_close(a, b, atol = 3e-2, rtol = 5e-2)+ with torch.no_grad():+ start = 0+ for n in lengths:+ alone = model(input_ids = ids[:, start : start + n], use_cache = False).logits.float()+ torch.testing.assert_close(+ fast_logits[:, start : start + n], alone, atol = 3e-2, rtol = 3e-2+ )+ start += ndiff --git a/tests/test_processor_text_packing_gate.py b/tests/test_processor_text_packing_gate.pynew file mode 100644index 00000000000..2a3bea66468--- /dev/null+++ b/tests/test_processor_text_packing_gate.py@@ -0,0 +1,289 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A processor `processing_class` on a text-only dataset may pack; media or mRoPE keeps the block."""++from types import SimpleNamespace++import pytest++pytest.importorskip("torch")+datasets = pytest.importorskip("datasets")+from torch import nn # noqa: E402+from transformers import ProcessorMixin # noqa: E402++try:+ import unsloth.trainer as trainer_module # noqa: E402+ from unsloth.trainer import _is_text_only_dataset, _uses_multimodal_rope # noqa: E402+except ImportError:+ import unsloth+ if getattr(unsloth, "DEVICE_TYPE", None) != "mlx":+ raise+ pytest.skip("unsloth.trainer is the MLX shim here", allow_module_level = True)++[email protected](autouse = True)+def _clean_env(monkeypatch):+ monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)+ monkeypatch.delenv("UNSLOTH_PROCESSOR_TEXT_PACKING", raising = False)+ monkeypatch.setattr(trainer_module.trl, "__version__", "1.13.0")+ for name in ("enable_padding_free_metadata", "enable_sample_packing"):+ monkeypatch.setattr(trainer_module, name, lambda model, trainer: None)+++class _Proc(ProcessorMixin):+ """A processor without running ProcessorMixin.__init__ (no real sub-processors needed)."""++ def __init__(self, tokenizer = object()):+ self.tokenizer = tokenizer+++class _Cfg:+ def __init__(self, **extra):+ self.model_type = "gemma4"+ self.vision_config = {}+ self.__dict__.update(extra)++ def to_dict(self):+ return dict(self.__dict__)+++class _Model(nn.Module):+ def __init__(self, config = None):+ super().__init__()+ self.config = config if config is not None else _Cfg()++ def forward(+ self,+ input_ids = None,+ **kwargs,+ ):+ return None+++def _patched(raise_on = None):+ calls = []++ class _StubSFTTrainer:+ def __init__(+ self,+ model = None,+ args = None,+ **kwargs,+ ):+ calls.append(+ (bool(getattr(args, "packing", False)), getattr(args, "padding_free", None))+ )+ if raise_on is not None and (args.packing or args.padding_free):+ raise ValueError(raise_on)+ self.model = model+ self.args = args++ module = SimpleNamespace(SFTTrainer = _StubSFTTrainer)+ trainer_module._patch_sft_trainer_auto_packing(module)+ return module, calls+++def _config(packing = False):+ return SimpleNamespace(packing = packing, padding_free = None, max_length = 512)+++TEXT = datasets.Dataset.from_dict({"text": ["a b c", "d e"]})++[email protected]("packing", [False, True])+def test_text_only_processor_is_unblocked(packing):+ module, _ = _patched()+ t = module.SFTTrainer(+ model = _Model(), args = _config(packing), train_dataset = TEXT, processing_class = _Proc()+ )+ assert t.args.padding_free is True+ assert bool(t.args.packing) is packing+++def test_kill_switch_restores_the_block(monkeypatch):+ monkeypatch.setenv("UNSLOTH_PROCESSOR_TEXT_PACKING", "0")+ module, _ = _patched()+ t = module.SFTTrainer(+ model = _Model(), args = _config(True), train_dataset = TEXT, processing_class = _Proc()+ )+ assert t.args.padding_free is False and t.args.packing is False+++def _blocked(**kwargs):+ module, _ = _patched()+ kw = dict(model = _Model(), args = _config(True), train_dataset = TEXT, processing_class = _Proc())+ kw.update(kwargs)+ t = module.SFTTrainer(**kw)+ return t.args.padding_free is False and t.args.packing is False+++ "column",+ ["image", "images", "audio", "input_features", "pixel_values", "video", "mm_token_type_ids"],+)+def test_media_columns_stay_blocked(column):+ ds = datasets.Dataset.from_dict({"text": ["a"], column: [[0]]})+ assert _blocked(train_dataset = ds)+++def test_an_image_feature_nested_in_messages_stays_blocked():+ feats = datasets.Features(+ {+ "messages": [+ {+ "role": datasets.Value("string"),+ "content": [{"type": datasets.Value("string"), "image": datasets.Image()}],+ }+ ]+ }+ )+ ds = datasets.Dataset.from_dict(+ {"messages": [[{"role": "user", "content": [{"type": "text", "image": None}]}]]},+ features = feats,+ )+ assert not _is_text_only_dataset(ds)+ assert _blocked(train_dataset = ds)+++ "column", ["image_path", "audio_url", "input_values", "image_patches", "Video_File"]+)+def test_media_named_string_columns_stay_blocked(column):+ ds = datasets.Dataset.from_dict({"text": ["a"], column: ["x"]})+ assert not _is_text_only_dataset(ds)+ assert _blocked(train_dataset = ds)+++def test_an_image_path_string_inside_messages_stays_blocked():+ ds = datasets.Dataset.from_dict(+ {"messages": [[{"role": "user", "content": [{"type": "image", "image": "cat.png"}]}]]}+ )+ assert not _is_text_only_dataset(ds)+ assert _blocked(train_dataset = ds)+++def _chat(*parts):+ return datasets.Dataset.from_dict({"messages": [[{"role": "user", "content": list(parts)}]]})+++ "part",+ [+ {"type": "audio", "url": "speech.wav"},+ {"type": "image", "url": "https://example.com/cat.png"},+ {"type": "video", "path": "clip.mp4"},+ {"type": "image", "base64": "aGk="},+ ],+)+def test_a_media_part_named_only_by_its_type_stays_blocked(part):+ ds = _chat({"type": "text", "text": "hi"}, part)+ assert not _is_text_only_dataset(ds)+ assert _blocked(train_dataset = ds)+++def test_text_parts_and_a_top_level_type_column_stay_text():+ assert _is_text_only_dataset(_chat({"type": "text", "text": "hi"}))+ assert _is_text_only_dataset(+ datasets.Dataset.from_dict({"text": ["a"], "type": ["news"], "source": ["web"]})+ )++[email protected]("model_type", ["mllama", "paligemma", "llava", "mistral3", "qwen2_vl"])+def test_processors_outside_the_verified_families_stay_blocked(model_type):+ assert _blocked(model = _Model(_Cfg(model_type = model_type)))+++ "model_type",+ ["gemma3", "gemma3_text", "gemma4", "gemma4_text", "gemma4_unified", "gemma4_unified_text"],+)+def test_verified_families_are_unblocked(model_type):+ assert not _blocked(model = _Model(_Cfg(model_type = model_type)))++[email protected]("value", ["false", "OFF", "no"])+def test_kill_switch_spellings(monkeypatch, value):+ monkeypatch.setenv("UNSLOTH_PROCESSOR_TEXT_PACKING", value)+ assert _blocked()+++def test_unknown_schema_stays_blocked():+ assert _blocked(train_dataset = [{"text": "a"}])+++def test_an_eval_split_with_images_stays_blocked():+ img = datasets.Dataset.from_dict({"text": ["a"], "images": [[0]]})+ assert _blocked(eval_dataset = {"a": TEXT, "b": img})+ assert not _blocked(eval_dataset = {"a": TEXT})+++def test_custom_collator_and_tokenizerless_processor_stay_blocked():+ assert _blocked(data_collator = lambda x: x)+ assert _blocked(processing_class = _Proc(tokenizer = None))+++def test_multimodal_rope_models_stay_blocked():+ """Qwen2-VL family builds the packed mask only from 4-row position_ids, so 2D ids leak."""+ cfg = _Cfg(+ model_type = "qwen2_5_vl", text_config = {"rope_parameters": {"mrope_section": [1, 1, 0]}}+ )+ assert _uses_multimodal_rope(cfg)+ assert _blocked(model = _Model(cfg))+ assert not _uses_multimodal_rope(+ _Cfg(text_config = {"rope_parameters": {"rope_type": "default"}})+ )+++def test_a_transformed_dataset_stays_blocked():+ """with_transform rows can add images the stored columns do not show."""+ ds = TEXT.with_transform(lambda batch: batch)+ assert not _is_text_only_dataset(ds)+ assert _blocked(train_dataset = ds)++[email protected]("version", ["0.22.2", "0.24.0", "1.9.2"])+def test_trl_before_1_10_never_tries_a_processor(monkeypatch, version):+ """TRL < 1.10 refuses packing / padding-free for any processor: no doomed __init__ first."""+ monkeypatch.setattr(trainer_module.trl, "__version__", version)+ module, calls = _patched(raise_on = "Packing is not supported for vision-language models.")+ t = module.SFTTrainer(+ model = _Model(), args = _config(True), train_dataset = TEXT, processing_class = _Proc()+ )+ assert calls == [(False, False)]+ assert t.args.padding_free is False and t.args.packing is False+++ "message",+ [+ "Packing is not supported for vision-language models. Please set `packing=False` in the SFTConfig.",+ "Padding-free training is yet not supported for vision-language models. Please set "+ "`padding_free=False` in the `SFTConfig`.",+ "Packing is not supported for vision datasets. Please set `packing=False` in the SFTConfig.",+ "Padding-free training is yet not supported for vision datasets. Please set `padding_free=False` in "+ "the `SFTConfig`.",+ ],+)+def test_a_trl_refusal_falls_back_to_the_old_block(message):+ """A refusal TRL still raises falls back to exactly the old behaviour."""+ module, calls = _patched(raise_on = message)+ packing = message.startswith("Packing")+ t = module.SFTTrainer(+ model = _Model(), args = _config(packing), train_dataset = TEXT, processing_class = _Proc()+ )+ assert len(calls) == 2+ assert calls[-1] == (False, False)+ assert t.args.padding_free is False and t.args.packing is False+++def test_an_unrelated_value_error_still_propagates():+ module, _ = _patched(raise_on = "something else entirely")+ with pytest.raises(ValueError, match = "something else"):+ module.SFTTrainer(+ model = _Model(), args = _config(False), train_dataset = TEXT, processing_class = _Proc()+ )+++if __name__ == "__main__":+ raise SystemExit(pytest.main([__file__, "-q"]))diff --git a/tests/utils/test_flash_impl.py b/tests/utils/test_flash_impl.pyindex 1b410925a68..80888a51b3b 100644--- a/tests/utils/test_flash_impl.py+++ b/tests/utils/test_flash_impl.py@@ -536,9 +536,12 @@ def test_fa4_hd256_window_keeps_segments_route(real_fa4):not has_real_cuda(), reason = "needs CUDA tensors (the wrapper only takes CUDA rows)")@pytest.mark.skipif(torch.version.hip is not None, reason = "torch varlen is off on ROCm")[email protected]("efficient", [True, False])@pytest.mark.parametrize("window", [None, 2])@pytest.mark.parametrize("allowed", [True, False])-def test_hf_wrapper_skips_torch_varlen_for_shapes_the_impl_lacks(monkeypatch, allowed, window):+def test_hf_wrapper_skips_torch_varlen_for_shapes_the_impl_lacks(+ monkeypatch, allowed, window, efficient+):from unsloth.utils import hf_packed_attention as hpafrom unsloth.utils import attention_dispatch as ad@@ -569,8 +572,11 @@ def test_hf_wrapper_skips_torch_varlen_for_shapes_the_impl_lacks(monkeypatch, almonkeypatch.setattr(hpa, "_TORCH_VARLEN", [fake_varlen])monkeypatch.setattr(hpa, "_TORCH_VARLEN_DEVICES", {})monkeypatch.setattr(hpa, "_sdpa_requested", lambda q, k: False)- # No FA2 / xformers either, so a refused torch varlen goes back to the wrapped sdpa.+ # No FA2 / xformers either: a refused torch varlen takes the efficient kernel, else the wrapped sdpa.monkeypatch.setattr(ad, "select_attention_backend", lambda use_varlen = False: ad.SDPA)+ if not efficient:+ monkeypatch.setattr(hpa, "_EFFICIENT_VARLEN", [False])+ before = hpa.HF_PACKED_ATTENTION_STATS.get("efficient", 0)monkeypatch.setattr(fi, "flash_impl_allows", lambda head_dim, window, device: asked.append(window) or allowed)@@ -596,5 +602,8 @@ def test_hf_wrapper_skips_torch_varlen_for_shapes_the_impl_lacks(monkeypatch, alassert asked == [window]if allowed:assert used and not seen+ elif efficient:+ assert not used and not seen+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before + 1else:assert out == ("orig", None) and not useddiff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex f5a7fad720a..444ec1d4771 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -552,7 +552,13 @@ _FLEX_LARGE_HEAD_DIM_ENV_VAR = "UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM"# gemma2 measured slower under flex. gemma4 shares one _attn_implementation across all layers, so# flex would also pull its 25 sliding layers off unsloth_zoo's faster gemma4_flash_sliding router,# which is registered under "sdpa".-_FLEX_LARGE_HEAD_DIM_EXCLUDED_MODELS = ("gemma2", "gemma4", "gemma4_text")+_FLEX_LARGE_HEAD_DIM_EXCLUDED_MODELS = (+ "gemma2",+ "gemma4",+ "gemma4_text",+ "gemma4_unified",+ "gemma4_unified_text",+)# Interface dispatch, and a BlockMask from create_causal_mask rather than a hand-built 4D mask._FLEX_INTERFACE_MARKERS = ("ALL_ATTENTION_FUNCTIONS", "create_causal_mask")_FLEX_SUPPORT_FORCED = set()diff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex dee005012c5..0e4dfd42cc9 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -290,6 +290,148 @@ def _is_vision_eval_dataset(dataset, *, unknown_is_vision = False) -> bool:return _is_vision_dataset(dataset, unknown_is_vision = unknown_is_vision)+_MULTIMODAL_DATASET_KEYS = _VISION_DATASET_KEYS | frozenset(+ {+ "audio",+ "audios",+ "audio_values",+ "input_features",+ "input_features_mask",+ "pixel_values_images",+ }+)+_MULTIMODAL_FEATURE_TYPES = ("Image", "Audio", "Video", "Pdf")+# Media-named keys (image_path, audio_url, ...) may be loaded by the processor even as strings.+_MEDIA_NAME_PARTS = ("image", "audio", "video", "pixel", "input_values", "input_features")+++def _is_media_name(name) -> bool:+ if not isinstance(name, str):+ return False+ lowered = name.lower()+ return lowered in _MULTIMODAL_DATASET_KEYS or any(part in lowered for part in _MEDIA_NAME_PARTS)+++def _features_have_media(features) -> bool:+ stack = [(features, False)]+ seen = 0+ while stack:+ node, nested = stack.pop()+ seen += 1+ if seen > 10_000:+ return True # pathological schema: assume media rather than guess+ if type(node).__name__ in _MULTIMODAL_FEATURE_TYPES:+ return True+ if isinstance(node, dict):+ if any(_is_media_name(key) for key in node):+ return True+ # Only {"type", "text"} content parts are provably text.+ if nested and "type" in node and set(node) - {"type", "text"}:+ return True+ stack.extend((value, True) for value in node.values())+ elif isinstance(node, (list, tuple)):+ stack.extend((value, True) for value in node)+ else:+ inner = getattr(node, "feature", None) # Sequence / List / LargeList+ if inner is not None:+ stack.append((inner, True))+ return False+++def _is_text_only_dataset(dataset) -> bool:+ """Positively text-only: unknown schemas (lists, feature-less streams) count as media."""+ if dataset is None:+ return False+ column_names = getattr(dataset, "column_names", None)+ if column_names is None:+ return False+ if any(_is_media_name(name) for name in column_names):+ return False+ # with_transform / set_transform rows can carry images the stored columns do not show.+ if (getattr(dataset, "format", None) or {}).get("type") == "custom":+ return False+ features = getattr(dataset, "features", None)+ if features is None:+ # Streaming datasets lack types, so embedded images cannot be ruled out.+ return False+ return not _features_have_media(features)+++def _is_text_only_eval_dataset(dataset) -> bool:+ if dataset is None:+ return True+ if isinstance(dataset, dict):+ return all(_is_text_only_dataset(split) for split in dataset.values())+ return _is_text_only_dataset(dataset)+++def _uses_multimodal_rope(model_config) -> bool:+ """mRoPE models build a packed mask only from 4-row position_ids; 2D ids leak across docs."""+ if model_config is None:+ return True # unknown: assume the worst+ try:+ stack = [model_config.to_dict()]+ except Exception:+ return True+ seen = 0+ while stack:+ node = stack.pop()+ seen += 1+ if seen > 10_000:+ return True+ if isinstance(node, dict):+ for key, value in node.items():+ if key in ("mrope_section", "mrope_interleaved"):+ return True+ if key in ("rope_type", "type") and value == "mrope":+ return True+ if isinstance(value, (dict, list, tuple)):+ stack.append(value)+ elif isinstance(node, (list, tuple)):+ stack.extend(x for x in node if isinstance(x, (dict, list, tuple)))+ return False+++_PROCESSOR_TEXT_PACKING_MODELS = frozenset(+ {"gemma3", "gemma3_text", "gemma4", "gemma4_text", "gemma4_unified", "gemma4_unified_text"}+)+++def _processor_text_only(+ processing_class, data_collator, train_dataset, eval_dataset, model_config+) -> bool:+ """A processor on text-only rows tokenizes like its tokenizer, so packing is safe."""+ if os.environ.get("UNSLOTH_PROCESSOR_TEXT_PACKING", "1").lower() in ("0", "false", "no", "off"):+ return False+ # Only families whose packed text rows are verified to equal each document alone.+ if getattr(model_config, "model_type", None) not in _PROCESSOR_TEXT_PACKING_MODELS:+ return False+ # TRL < 1.10 refuses packing / padding-free for any processor; skip a doomed __init__.+ if Version(trl.__version__) < Version("1.10.0"):+ return False+ if data_collator is not None:+ return False+ if _uses_multimodal_rope(model_config):+ return False+ if getattr(processing_class, "tokenizer", None) is None:+ return False+ return _is_text_only_dataset(train_dataset) and _is_text_only_eval_dataset(eval_dataset)+++# TRL < 1.10 raises the first two for any processor; 1.10+ the last two when a sample holds images.+_TRL_VLM_PACKING_REJECTIONS = (+ "packing is not supported for vision-language models",+ "padding-free training is yet not supported for vision-language models",+ "packing is not supported for vision datasets",+ "padding-free training is yet not supported for vision datasets",+)+++def _is_trl_vlm_packing_rejection(exc: Exception) -> bool:+ message = str(exc).lower()+ return any(msg in message for msg in _TRL_VLM_PACKING_REJECTIONS)++_HYBRID_CONFIG_MARKERS = ("linear_conv_kernel_dim","linear_key_head_dim",@@ -1577,6 +1719,9 @@ def _patch_sft_trainer_auto_packing(trl_module):train_dataset = args[3] if len(args) >= 4 else kwargs.get("train_dataset")eval_dataset = args[4] if len(args) >= 5 else kwargs.get("eval_dataset")is_processor = isinstance(processing_class, ProcessorMixin)+ processor_text_only = is_processor and _processor_text_only(+ processing_class, data_collator, train_dataset, eval_dataset, model_config+ )is_auto_processor_vlm = is_vlm and processing_class is Noneis_vision_dataset = (data_collator is None@@ -1609,7 +1754,7 @@ def _patch_sft_trainer_auto_packing(trl_module):# condition the post-init check skips on, so leaving it armed is free.blocked = ((data_collator is not None)- or is_processor+ or (is_processor and not processor_text_only)or is_auto_processor_vlmor is_vision_datasetor is_unsupported_model@@ -1676,7 +1821,23 @@ def _patch_sft_trainer_auto_packing(trl_module):try:original_init(self, *args, **kwargs)except ValueError as exc:- if packing_active and _should_skip_auto_packing_error(exc):+ if processor_text_only and not blocked and _is_trl_vlm_packing_rejection(exc):+ if requested_pack:+ logger.warning(+ "Unsloth: packing=True ignored (this TRL version refuses it for processors)."+ )+ else:+ logger.info("Unsloth: padding-free disabled for the processor (%s).", exc)+ for _attr in ("packing", "padding_free"):+ if hasattr(config_arg, _attr):+ setattr(config_arg, _attr, False)+ packing_active = False+ auto_padding_free_active = False+ blocked = True+ # Release the failed attempt's frames (may hold a loaded model) before retrying.+ exc.__traceback__ = None+ original_init(self, *args, **kwargs)+ elif packing_active and _should_skip_auto_packing_error(exc):logger.info("Unsloth: Auto sample packing failed because trainer reported an incompatible setup (%s).",exc,diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.pyindex a1c0b796fa5..63cf90bd5ab 100644--- a/unsloth/utils/hf_packed_attention.py+++ b/unsloth/utils/hf_packed_attention.py@@ -1,7 +1,9 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved."""Packed rows on the transformers modeling path run a varlen flash kernel (torch's, else xformers /-flash-attn) instead of SDPA over a dense block-causal mask; anything else reaches the wrapped "sdpa"."""+flash-attn) instead of SDPA over a dense block-causal mask; heads past flash's 256 (Gemma-4 global+layers, 512) and shapes an FA4 / FA3 impl refuses run SDPA's memory-efficient kernel over the same+cu_seqlens. Anything else reaches the wrapped "sdpa"."""import inspectimport logging@@ -15,7 +17,8 @@ __all__ = ["enable_hf_packed_attention", "HF_PACKED_ATTENTION_STATS"]_FLAG = "_unsloth_hf_packed_varlen"_ENV = "UNSLOTH_HF_PACKED_VARLEN"_ORIG_SDPA = [None]-HF_PACKED_ATTENTION_STATS = {"fast": 0, "fallback": 0, "mask_checks": 0}+# efficient: the subset of fast calls that ran SDPA's memory-efficient kernel over cu_seqlens.+HF_PACKED_ATTENTION_STATS = {"fast": 0, "fallback": 0, "mask_checks": 0, "efficient": 0}# Layers of one window share a mask object: window -> [weakref, (cu_seqlens, versions), verdict]._MASK_CHECK = {}_MASK_CHECK_ROWS = 1024@@ -149,6 +152,136 @@ def _torch_varlen(device):return fn if ok else None+_EFFICIENT_ENV = "UNSLOTH_HF_PACKED_EFFICIENT"+# Above flash's 256 cap (Gemma-4 global layers are 512); wider heads are untested.+_EFFICIENT_MAX_HEAD_DIM = 512+_EFFICIENT_VARLEN = []+_ACCELERATOR_ERROR = getattr(torch, "AcceleratorError", None) # torch 2.8++_EFFICIENT_DEVICES = {}+++def _efficient_varlen_takes(query, window) -> bool:+ """SDPA's memory-efficient kernel over cu_seqlens: no T x T bias. Off under any policy asking for SDPA."""+ if os.environ.get(_EFFICIENT_ENV, "1").lower() in ("0", "false", "no", "off"):+ return False+ if not _EFFICIENT_VARLEN:+ ok = True+ try:+ schema = torch.ops.aten._efficient_attention_forward.default._schema+ names = {argument.name for argument in schema.arguments}+ ok = ok and {"cu_seqlens_q", "scale", "window_size"} <= names+ schema = torch.ops.aten._efficient_attention_backward.default._schema+ ok = ok and "window_size" in {argument.name for argument in schema.arguments}+ except Exception:+ ok = False+ _EFFICIENT_VARLEN.append(ok)+ head_dim = query.shape[-1]+ if (+ not _EFFICIENT_VARLEN[0]+ or torch.version.hip is not None+ or head_dim > _EFFICIENT_MAX_HEAD_DIM+ or head_dim % 8 != 0+ # Its backward is nondeterministic.+ or torch.are_deterministic_algorithms_enabled()+ or not torch.backends.cuda.mem_efficient_sdp_enabled()+ ):+ return False+ ok = _EFFICIENT_DEVICES.get(query.device.index)+ if ok is None:+ ok = torch.cuda.get_device_capability(query.device)[0] >= 8+ _EFFICIENT_DEVICES[query.device.index] = ok+ if not ok:+ return False+ from torch.nn.attention import SDPBackend+ from .attention_dispatch import _sdpa_backend_state++ # A math-first priority (or a disabled efficient backend) for these shapes keeps SDPA.+ n_heads = query.shape[1]+ _, choice = _sdpa_backend_state(+ str(query.device), query.dtype, head_dim, n_heads, n_heads, True, False+ )+ return choice not in (int(SDPBackend.MATH), -1)+++class _EfficientVarlenAttention(torch.autograd.Function):+ """(1, T, H, D) Q / K / V over cu_seqlens, causal (and windowed) inside each segment."""++ @staticmethod+ def forward(ctx, query, key, value, cu_seqlens, max_seqlen, scale, window):+ out, logsumexp, seed, offset, _, _ = torch.ops.aten._efficient_attention_forward(+ query,+ key,+ value,+ None,+ cu_seqlens,+ cu_seqlens,+ max_seqlen,+ max_seqlen,+ 0.0,+ 1, # causal from the top left of each segment+ any(ctx.needs_input_grad[:3]),+ scale = scale,+ window_size = window,+ )+ ctx.save_for_backward(query, key, value, out, logsumexp, cu_seqlens, seed, offset)+ ctx.max_seqlen, ctx.scale, ctx.window = max_seqlen, scale, window+ return out++ @staticmethod+ def backward(ctx, grad_out):+ query, key, value, out, logsumexp, cu_seqlens, seed, offset = ctx.saved_tensors+ grad_q, grad_k, grad_v, _ = torch.ops.aten._efficient_attention_backward(+ grad_out.contiguous(),+ query,+ key,+ value,+ None,+ out,+ cu_seqlens,+ cu_seqlens,+ ctx.max_seqlen,+ ctx.max_seqlen,+ logsumexp,+ 0.0,+ seed,+ offset,+ 1,+ False,+ scale = ctx.scale,+ window_size = ctx.window,+ )+ return grad_q, grad_k, grad_v, None, None, None, None+++def _efficient_varlen(query, key, value, cu_seqlens, max_seqlen, scale, window):+ """Q (1, H, T, D), K / V (1, H_kv, T, D) -> (1, T, H, D)."""+ _, n_heads, total, head_dim = query.shape+ n_kv_heads = key.shape[1]++ def heads_last(x):+ x = x[0].transpose(0, 1).unsqueeze(0)+ # The kernel needs a unit stride on the head dim; a refusal would turn the route off for good.+ return x if x.stride(-1) == 1 else x.contiguous()++ q, k, v = heads_last(query), heads_last(key), heads_last(value)+ if n_kv_heads != n_heads:+ # The kernel has no GQA: repeat K / V heads as transformers' sdpa path does (autograd sums back).+ groups = n_heads // n_kv_heads+ k = (+ k.unsqueeze(3)+ .expand(1, total, n_kv_heads, groups, head_dim)+ .reshape(1, total, n_heads, head_dim)+ )+ v = (+ v.unsqueeze(3)+ .expand(1, total, n_kv_heads, groups, head_dim)+ .reshape(1, total, n_heads, head_dim)+ )+ return _EfficientVarlenAttention.apply(+ q, k, v, cu_seqlens.to(torch.int32), int(max_seqlen), float(scale), window+ )++_SDPA_MATH = []@@ -221,6 +354,7 @@ def _sdpa_packed_varlen(bsz, n_heads, q_len, head_dim = query.shapeseq_info = torch_varlen = backend = None+ efficient = Falsewindow = _sliding_window(module, kwargs)if (bsz == 1@@ -230,9 +364,8 @@ def _sdpa_packed_varlen(and key.shape[0] == value.shape[0] == 1and key.shape[1] == value.shape[1]and key.shape[2] == value.shape[2] == q_len- # Not MLA (192 / 128) nor Gemma-4 global (512).+ # Not MLA (192 / 128).and key.shape[-1] == value.shape[-1] == head_dim- and head_dim <= 256and head_dim % 8 == 0and query.dtype in (torch.float16, torch.bfloat16)and query.dtype == key.dtype == value.dtype@@ -246,27 +379,35 @@ def _sdpa_packed_varlen(and n_heads % key.shape[1] == 0and window is not _NO_WINDOWand not _softcapped(module, kwargs)- and not _sdpa_requested(query, key)):- torch_varlen = _torch_varlen(query.device)- # An FA4 / FA3 impl behind torch's flash op lacks some shapes (sm100: hd 136-248, hd256 + window).- if torch_varlen is not None and not flash_impl_allows(head_dim, window, query.device):- torch_varlen = None- backend = select_attention_backend(use_varlen = True)- # flash-attn's ROCm backend lists sliding windows as work in progress: windowed rows keep sdpa there.- if window is not None and torch.version.hip is not None:- backend = None- # Flash-2 varlen backward faults past int32 indexing. Keyed on training / grad mode, not- # requires_grad, so a reentrant checkpoint's no_grad forward and its recompute agree.- if (- not _VARLEN_INT32_GUARD_DISABLED- and (getattr(module, "training", False) or torch.is_grad_enabled())- and _varlen_backward_overflows_int32(- kwargs["packed_seq_lengths"].numel(), q_len, n_heads, head_dim+ if head_dim > 256:+ # Past every flash kernel (Gemma-4 global layers are 512): the efficient kernel, per segment.+ efficient = _efficient_varlen_takes(query, window)+ elif not _sdpa_requested(query, key):+ torch_varlen = _torch_varlen(query.device)+ # An FA4 / FA3 impl behind torch's flash op lacks some shapes (sm100: hd 136-248, hd256 + window).+ refused = torch_varlen is not None and not flash_impl_allows(+ head_dim, window, query.device)- ):- torch_varlen, backend = None, None- if torch_varlen is not None or backend in (FLASH_VARLEN, XFORMERS):+ if refused:+ torch_varlen = None+ backend = select_attention_backend(use_varlen = True)+ # flash-attn's ROCm backend lists sliding windows as work in progress: windowed rows keep sdpa there.+ if window is not None and torch.version.hip is not None:+ backend = None+ # Flash-2 varlen backward faults past int32; keyed on grad mode so checkpoint recompute agrees.+ if (+ not _VARLEN_INT32_GUARD_DISABLED+ and (getattr(module, "training", False) or torch.is_grad_enabled())+ and _varlen_backward_overflows_int32(+ kwargs["packed_seq_lengths"].numel(), q_len, n_heads, head_dim+ )+ ):+ torch_varlen, backend, refused = None, None, False+ # Refused shape and no flash-attn / xformers: the efficient kernel.+ if refused and backend not in (FLASH_VARLEN, XFORMERS):+ efficient = _efficient_varlen_takes(query, window)+ if efficient or torch_varlen is not None or backend in (FLASH_VARLEN, XFORMERS):seq_info = get_packed_info_from_kwargs(kwargs, query.device)if seq_info is None or not _mask_is_block_causal(attention_mask, seq_info[1], q_len, window):HF_PACKED_ATTENTION_STATS["fallback"] += 1@@ -285,6 +426,35 @@ def _sdpa_packed_varlen(scale = head_dim**-0.5 if scaling is None else scalingn_kv_heads = key.shape[1]_, cu_seqlens, max_seqlen = seq_info+ if efficient:+ try:+ out = _efficient_varlen(query, key, value, cu_seqlens, max_seqlen, scale, window)+ except torch.OutOfMemoryError:+ raise+ except RuntimeError as exc:+ # A CUDA fault is not a refusal: let it surface.+ if _ACCELERATOR_ERROR is not None and isinstance(exc, _ACCELERATOR_ERROR):+ raise+ # A build without this kernel shape refuses at forward: stop using it, this call goes to sdpa.+ _EFFICIENT_VARLEN[:] = [False]+ logging.getLogger(__name__).warning(+ f"Unsloth: efficient varlen attention unavailable, packed rows past head dim 256 use sdpa ({exc})"+ )+ HF_PACKED_ATTENTION_STATS["fast"] -= 1+ HF_PACKED_ATTENTION_STATS["fallback"] += 1+ return orig(+ module,+ query,+ key,+ value,+ attention_mask,+ dropout = dropout,+ scaling = scaling,+ is_causal = is_causal,+ **kwargs,+ )+ HF_PACKED_ATTENTION_STATS["efficient"] += 1+ return out, Noneif torch_varlen is not None:gqa, k_in, v_in = {}, key, valueif n_kv_heads != n_heads:@@ -310,6 +480,8 @@ def _sdpa_packed_varlen(except torch.OutOfMemoryError:raiseexcept RuntimeError as exc:+ if _ACCELERATOR_ERROR is not None and isinstance(exc, _ACCELERATOR_ERROR):+ raise# A build whose varlen picks a backend without causal GQA (cuDNN) refuses at forward: stop# using it and give this call to the wrapped sdpa._TORCH_VARLEN[:] = [None]