Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Gemma 4 packed SFT: efficient kernel for the 512-wide global layers, processor on text-only data (#13384)

Daniel Han37f908833a40
diff --git a/tests/test_flex_gemma4_sliding_scope.py b/tests/test_flex_gemma4_sliding_scope.py
index 3dca3852cbe..b9193e6dd54 100644
--- a/tests/test_flex_gemma4_sliding_scope.py
+++ b/tests/test_flex_gemma4_sliding_scope.py
@@ -51,6 +51,20 @@ def test_gemma4_text_only_is_not_routed_either():
assert u._prefers_flex_for_head_dim(text_only) is False
[email protected]("top", ["gemma4_unified", None])
+def test_gemma4_unified_is_not_routed_either(top):
+ text = _Cfg(
+ model_type = "gemma4_unified_text",
+ head_dim = 256,
+ global_head_dim = 512,
+ sliding_window = 1024,
+ num_attention_heads = 16,
+ )
+ cfg = text if top is None else _Cfg(model_type = top, text_config = text)
+ assert u._text_attention_head_dim(cfg) == 512
+ assert u._prefers_flex_for_head_dim(cfg) is False
+
+
def test_the_head_dim_is_still_large_so_this_really_is_the_exclusion_talking():
assert u._text_attention_head_dim(_gemma4()) == 512
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.py
index 5c0377bcd39..2ddb796fe47 100644
--- a/tests/test_hf_packed_attention.py
+++ b/tests/test_hf_packed_attention.py
@@ -179,7 +179,7 @@ _TORCH_VARLEN_HOST = pytest.mark.skipif(
"dropout",
"batched",
"cpu",
- "head_dim_512",
+ "head_dim_640",
"mla_value_width",
"autocast_dtype",
"paged_cache",
@@ -228,8 +228,9 @@ def test_packed_calls_that_varlen_cannot_express_fall_back(monkeypatch, why):
B = 2
elif why == "cpu":
device, mask = "cpu", _block_causal(lengths)
- elif why == "head_dim_512":
- D = v_width = 512
+ elif why == "head_dim_640":
+ # Past both flash (256) and the efficient route's tested width (512).
+ D = v_width = 640
elif why == "mla_value_width":
D, v_width = 192, 128
elif why == "paged_cache":
@@ -670,3 +671,443 @@ def test_tiny_mistral_packed_row_matches_the_masked_path(monkeypatch, window):
torch.testing.assert_close(fast_logits, ref_logits, atol = 3e-2, rtol = 3e-2)
for a, b in zip(fast_grads, ref_grads):
torch.testing.assert_close(a, b, atol = 3e-2, rtol = 5e-2)
+
+
+# Gemma-4 global layers: head_dim 512, past every flash kernel, take SDPA's efficient kernel over cu_seqlens.
+def _efficient_varlen_op() -> bool:
+ try:
+ forward = torch.ops.aten._efficient_attention_forward.default._schema
+ backward = torch.ops.aten._efficient_attention_backward.default._schema
+ except Exception:
+ return False
+ return {"cu_seqlens_q", "scale", "window_size"} <= {a.name for a in forward.arguments} and (
+ "window_size" in {a.name for a in backward.arguments}
+ )
+
+
+_EFFICIENT_HOST = pytest.mark.skipif(
+ torch.version.hip is not None or not _efficient_varlen_op(),
+ reason = "the efficient varlen route is off on ROCm and on torch without its varlen schema",
+)
+
+
+def _efficient_fresh(monkeypatch):
+ monkeypatch.setattr(hpa, "_EFFICIENT_VARLEN", [])
+ monkeypatch.setattr(hpa, "_EFFICIENT_DEVICES", {})
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ monkeypatch.delenv(hpa._EFFICIENT_ENV, raising = False)
+
+
+def _wide_inputs(
+ lengths,
+ H,
+ n_kv,
+ D,
+ scale = 0.1,
+):
+ T = sum(lengths)
+
+ def make(heads):
+ x = torch.randn(1, heads, T, D, device = "cuda", dtype = torch.bfloat16) * scale
+ return x.requires_grad_()
+
+ return make(H), make(n_kv), make(n_kv)
+
+
+@_CUDA
+@_EFFICIENT_HOST
[email protected]("window", [None, 5])
[email protected]("n_kv", [8, 2, 1])
+def test_wide_heads_match_sdpa_over_the_dense_mask(_efficient_fresh, n_kv, window):
+ """Output and dQ / dK / dV equal transformers' sdpa over the real mask, Gemma-4 scaling 1.0."""
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ hpa._ORIG_SDPA[0] = sdpa_attention_forward
+ torch.manual_seed(0)
+ lengths = [37, 100, 2, 5, 230]
+ H, D = 8, 512
+ module = types.SimpleNamespace(
+ is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = window
+ )
+ q, k, v = _wide_inputs(lengths, H, n_kv, D)
+ mask = (
+ _block_causal(lengths, "cuda")
+ if window is None
+ else _hf_window_mask(lengths, window, "cuda")
+ )
+ psl = torch.tensor(lengths, dtype = torch.int32)
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ fast, _ = hpa._sdpa_packed_varlen(module, q, k, v, mask, scaling = 1.0, packed_seq_lengths = psl)
+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"] + 1
+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"] + 1
+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 1.0)
+ assert fast.shape == ref.shape == (1, sum(lengths), H, D)
+ torch.testing.assert_close(fast.float(), ref.float(), atol = 1e-2, rtol = 1e-2)
+ g = torch.randn_like(ref)
+ for a, b in zip(
+ torch.autograd.grad(fast, (q, k, v), g), torch.autograd.grad(ref, (q, k, v), g)
+ ):
+ torch.testing.assert_close(a.float(), b.float(), atol = 2e-2, rtol = 2e-2)
+
+
+@_CUDA
+@_EFFICIENT_HOST
+def test_wide_heads_keep_documents_apart(_efficient_fresh):
+ """Each document of the packed row equals that document attended alone."""
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ hpa._ORIG_SDPA[0] = sdpa_attention_forward
+ torch.manual_seed(1)
+ lengths = [60, 3, 130]
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 4, training = True)
+ q, k, v = _wide_inputs(lengths, 8, 2, 512)
+ fast, _ = hpa._sdpa_packed_varlen(
+ module,
+ q,
+ k,
+ v,
+ _block_causal(lengths, "cuda"),
+ scaling = 1.0,
+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
+ )
+ start = 0
+ for n in lengths:
+ sl = slice(start, start + n)
+ alone, _ = sdpa_attention_forward(
+ module, q[:, :, sl], k[:, :, sl], v[:, :, sl], None, scaling = 1.0, is_causal = True
+ )
+ torch.testing.assert_close(fast[:, sl].float(), alone.float(), atol = 1e-2, rtol = 1e-2)
+ start += n
+
+
+@_CUDA
+@_EFFICIENT_HOST
+ "why",
+ [
+ "none",
+ "cross_document",
+ "wrong_window",
+ "softcap",
+ "sinks",
+ "dropout",
+ "fp32",
+ "head_dim_640",
+ "kill_switch",
+ "deterministic",
+ "math_only",
+ "efficient_disabled",
+ "no_op",
+ ],
+)
+def test_wide_heads_fall_back_to_the_wrapped_sdpa(_efficient_fresh, monkeypatch, why):
+ """Each reason alone sends a head_dim 512 call to the wrapped sdpa; "none" is the control."""
+ import contextlib
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ seen, orig = _calls()
+ hpa._ORIG_SDPA[0] = orig
+ lengths, D, dtype = [5, 7], 512, torch.bfloat16
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 2)
+ mask, dropout, context = _block_causal(lengths, "cuda"), 0.0, contextlib.nullcontext()
+ if why == "cross_document":
+ mask = mask.clone()
+ mask[0, 0, 6, 1] = True
+ elif why == "wrong_window":
+ module.sliding_window = 3 # the mask has no band
+ elif why == "softcap":
+ module.config = types.SimpleNamespace(attn_logit_softcapping = 50.0)
+ elif why == "sinks":
+ module.sinks = torch.zeros(4)
+ elif why == "dropout":
+ dropout = 0.1
+ elif why == "fp32":
+ dtype = torch.float32
+ elif why == "head_dim_640":
+ D = 640
+ elif why == "kill_switch":
+ monkeypatch.setenv(hpa._EFFICIENT_ENV, "0")
+ elif why == "math_only":
+ context = sdpa_kernel([SDPBackend.MATH])
+ elif why == "efficient_disabled":
+ context = sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.MATH])
+ elif why == "no_op":
+ monkeypatch.setattr(hpa, "_EFFICIENT_VARLEN", [False])
+ q = torch.randn(1, 4, 12, D, device = "cuda", dtype = dtype)
+ k = torch.randn(1, 2, 12, D, device = "cuda", dtype = dtype)
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ deterministic = torch.are_deterministic_algorithms_enabled()
+ if why == "deterministic":
+ torch.use_deterministic_algorithms(True)
+ try:
+ with context:
+ out = hpa._sdpa_packed_varlen(
+ module,
+ q,
+ k,
+ k,
+ mask,
+ dropout = dropout,
+ scaling = 1.0,
+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
+ )
+ finally:
+ torch.use_deterministic_algorithms(deterministic)
+ if why == "none":
+ assert out[0].shape == (1, 12, 4, D) and seen == []
+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"] + 1
+ else:
+ assert out == ("orig", None)
+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"]
+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1
+
+
+@_CUDA
+@_EFFICIENT_HOST
+def test_a_refusing_efficient_kernel_hands_the_call_back(_efficient_fresh, monkeypatch):
+ seen, orig = _calls()
+ hpa._ORIG_SDPA[0] = orig
+
+ def refuse(*args, **kwargs):
+ raise RuntimeError("No available kernel.")
+
+ monkeypatch.setattr(hpa._EfficientVarlenAttention, "apply", refuse)
+ lengths = [3, 3]
+ q = torch.randn(1, 4, 6, 512, device = "cuda", dtype = torch.bfloat16)
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ out = hpa._sdpa_packed_varlen(
+ types.SimpleNamespace(is_causal = True),
+ q,
+ q,
+ q,
+ _block_causal(lengths, "cuda"),
+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
+ )
+ assert out == ("orig", None) and len(seen) == 1
+ assert hpa._EFFICIENT_VARLEN == [False]
+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"]
+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1
+
+
+@_CUDA
+@_EFFICIENT_HOST
+def test_the_efficient_kill_switch_is_read_on_every_call(_efficient_fresh, monkeypatch):
+ q = torch.randn(1, 4, 12, 512, device = "cuda", dtype = torch.bfloat16)
+ assert hpa._efficient_varlen_takes(q, None)
+ monkeypatch.setenv(hpa._EFFICIENT_ENV, "0")
+ assert not hpa._efficient_varlen_takes(q, None)
+ monkeypatch.delenv(hpa._EFFICIENT_ENV)
+ assert hpa._efficient_varlen_takes(q, None)
+
+
+@_CUDA
+@_EFFICIENT_HOST
+def test_a_non_contiguous_head_dim_is_copied_not_refused(_efficient_fresh):
+ cu = torch.tensor([0, 5, 12], device = "cuda", dtype = torch.int32)
+ base = torch.randn(1, 4, 12, 1024, device = "cuda", dtype = torch.bfloat16)
+ q = base[..., ::2] # head dim 512 with stride 2
+ assert q.stride(-1) == 2
+ out = hpa._efficient_varlen(q, q, q, cu, 7, 512**-0.5, None)
+ ref = hpa._efficient_varlen(
+ q.contiguous(), q.contiguous(), q.contiguous(), cu, 7, 512**-0.5, None
+ )
+ torch.testing.assert_close(out, ref)
+
+
+@_CUDA
+@_EFFICIENT_HOST
+def test_a_cuda_fault_in_the_efficient_kernel_is_not_a_refusal(_efficient_fresh, monkeypatch):
+ if hpa._ACCELERATOR_ERROR is None:
+ pytest.skip("this torch has no AcceleratorError")
+ seen, orig = _calls()
+ hpa._ORIG_SDPA[0] = orig
+
+ def fault(*args, **kwargs):
+ raise hpa._ACCELERATOR_ERROR("CUDA error: an illegal memory access was encountered")
+
+ monkeypatch.setattr(hpa._EfficientVarlenAttention, "apply", fault)
+ lengths = [3, 3]
+ q = torch.randn(1, 4, 6, 512, device = "cuda", dtype = torch.bfloat16)
+ with pytest.raises(hpa._ACCELERATOR_ERROR):
+ hpa._sdpa_packed_varlen(
+ types.SimpleNamespace(is_causal = True),
+ q,
+ q,
+ q,
+ _block_causal(lengths, "cuda"),
+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
+ )
+ assert seen == [] and hpa._EFFICIENT_VARLEN != [False]
+
+
+@_CUDA
+@_EFFICIENT_HOST
[email protected]("backend", ["sdpa", "flash_varlen"])
+def test_a_shape_the_flash_impl_refuses_takes_the_efficient_kernel(
+ _efficient_fresh, monkeypatch, backend
+):
+ """FA4 on sm100 lacks hd256 + window: without flash-attn / xformers it takes the efficient kernel."""
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+ from unsloth.utils import attention_dispatch as ad
+ from unsloth.utils import flash_impl
+
+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:
+ pytest.skip("no torch varlen_attn to refuse")
+ monkeypatch.setattr(flash_impl, "flash_impl_allows", lambda head_dim, window, device: False)
+ monkeypatch.setattr(ad, "select_attention_backend", lambda use_varlen = False: backend)
+ dispatched = []
+ monkeypatch.setattr(
+ ad, "run_attention", lambda **kwargs: dispatched.append(1) or kwargs["Q"].transpose(1, 2)
+ )
+ hpa._ORIG_SDPA[0] = sdpa_attention_forward
+ torch.manual_seed(2)
+ lengths, window = [9, 20, 4], 6
+ module = types.SimpleNamespace(
+ is_causal = True, num_key_value_groups = 4, training = True, sliding_window = window
+ )
+ q, k, v = _wide_inputs(lengths, 8, 2, 256)
+ mask = _hf_window_mask(lengths, window, "cuda")
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ fast, _ = hpa._sdpa_packed_varlen(
+ module,
+ q,
+ k,
+ v,
+ mask,
+ scaling = 1.0,
+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
+ )
+ if backend == "flash_varlen":
+ assert dispatched == [1]
+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"]
+ return
+ assert dispatched == []
+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"] + 1
+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 1.0)
+ torch.testing.assert_close(fast.float(), ref.float(), atol = 1e-2, rtol = 1e-2)
+ g = torch.randn_like(ref)
+ for a, b in zip(
+ torch.autograd.grad(fast, (q, k, v), g), torch.autograd.grad(ref, (q, k, v), g)
+ ):
+ torch.testing.assert_close(a.float(), b.float(), atol = 2e-2, rtol = 2e-2)
+
+
+@_CUDA
+@_EFFICIENT_HOST
+def test_wide_heads_under_reentrant_checkpoint(_efficient_fresh):
+ from torch.utils.checkpoint import checkpoint
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ hpa._ORIG_SDPA[0] = sdpa_attention_forward
+ lengths = [40, 24]
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 4, training = True)
+ q, k, v = _wide_inputs(lengths, 8, 2, 512)
+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
+ fn = lambda q, k, v: hpa._sdpa_packed_varlen(
+ module, q, k, v, mask, scaling = 1.0, packed_seq_lengths = psl
+ )[0]
+ out = checkpoint(fn, q, k, v, use_reentrant = True)
+ g = torch.randn_like(out)
+ out.backward(g)
+ grads = [x.grad for x in (q, k, v)]
+ ref = sdpa_attention_forward(module, q, k, v, mask, scaling = 1.0)[0]
+ torch.testing.assert_close(out.float(), ref.float(), atol = 1e-2, rtol = 1e-2)
+ for a, b in zip(grads, torch.autograd.grad(ref, (q, k, v), g)):
+ torch.testing.assert_close(a.float(), b.float(), atol = 2e-2, rtol = 2e-2)
+
+
+@_CUDA
+@_EFFICIENT_HOST
+def test_wide_heads_leave_a_compiled_caller_alone(_efficient_fresh):
+ """Under fullgraph compile the wrapper returns the wrapped sdpa: no graph break, no counters."""
+ hpa._ORIG_SDPA[0] = _plain_sdpa
+ lengths = [3, 5]
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)
+ q = torch.randn(1, 4, 8, 512, device = "cuda", dtype = torch.bfloat16)
+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ fn = torch.compile(
+ lambda q: hpa._sdpa_packed_varlen(
+ module, q, q, q, mask, scaling = 1.0, packed_seq_lengths = psl
+ )[0],
+ backend = "eager",
+ fullgraph = True,
+ )
+ torch.testing.assert_close(fn(q), _plain_sdpa(module, q, q, q, mask, scaling = 1.0)[0])
+ assert hpa.HF_PACKED_ATTENTION_STATS == before
+
+
+@_CUDA
+@_EFFICIENT_HOST
[email protected]("k_eq_v", [False, True])
+def test_tiny_gemma4_packed_row_matches_each_document(_efficient_fresh, monkeypatch, k_eq_v):
+ """Gemma4 global_head_dim 512: efficient kernel; logits / grads match sdpa and each doc alone."""
+ transformers = pytest.importorskip("transformers")
+ if not hasattr(transformers, "Gemma4TextConfig"):
+ pytest.skip("this transformers has no Gemma 4")
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+
+ hpa._ORIG_SDPA[0] = None
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ # transformers' own entry: an earlier test can leave the installed wrapper in the mapping.
+ monkeypatch.setitem(ALL_ATTENTION_FUNCTIONS, "sdpa", sdpa_attention_forward)
+ assert hpa.enable_hf_packed_attention()
+ torch.manual_seed(0)
+ config = transformers.Gemma4TextConfig(
+ vocab_size = 128,
+ vocab_size_per_layer_input = 128,
+ hidden_size = 64,
+ hidden_size_per_layer_input = 8,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ layer_types = ["sliding_attention", "full_attention"],
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ num_global_key_value_heads = 1 if k_eq_v else None,
+ attention_k_eq_v = k_eq_v,
+ head_dim = 256,
+ global_head_dim = 512,
+ sliding_window = 4,
+ attn_implementation = "sdpa",
+ )
+ model = transformers.Gemma4ForCausalLM(config).cuda().to(torch.bfloat16)
+ global_attn = model.model.layers[1].self_attn
+ assert global_attn.head_dim == 512 and model.model.layers[0].self_attn.head_dim == 256
+ lengths = [3, 11, 7]
+ ids = torch.randint(0, 128, (1, sum(lengths)), device = "cuda")
+ position_ids = torch.cat([torch.arange(n) for n in lengths])[None].cuda()
+ psl = torch.tensor(lengths, dtype = torch.int32)
+
+ def run():
+ model.zero_grad()
+ out = model(
+ input_ids = ids, position_ids = position_ids, packed_seq_lengths = psl, use_cache = False
+ )
+ out.logits.float().square().mean().backward()
+ return out.logits.detach().float(), [
+ p.grad.float() for p in model.parameters() if p.grad is not None
+ ]
+
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ fast_logits, fast_grads = run()
+ stats = hpa.HF_PACKED_ATTENTION_STATS
+ assert stats["efficient"] == before["efficient"] + 1
+ # Without torch varlen (torch 2.10 and older) the windowed sliding layer may have no varlen route.
+ assert stats["fast"] + stats["fallback"] == before["fast"] + before["fallback"] + 2
+ if hpa._torch_varlen(ids.device) is not None:
+ assert stats["fast"] == before["fast"] + 2
+ monkeypatch.setenv("UNSLOTH_HF_PACKED_VARLEN", "0")
+ ref_logits, ref_grads = run()
+ torch.testing.assert_close(fast_logits, ref_logits, atol = 3e-2, rtol = 3e-2)
+ for a, b in zip(fast_grads, ref_grads):
+ torch.testing.assert_close(a, b, atol = 3e-2, rtol = 5e-2)
+ with torch.no_grad():
+ start = 0
+ for n in lengths:
+ alone = model(input_ids = ids[:, start : start + n], use_cache = False).logits.float()
+ torch.testing.assert_close(
+ fast_logits[:, start : start + n], alone, atol = 3e-2, rtol = 3e-2
+ )
+ start += n
diff --git a/tests/test_processor_text_packing_gate.py b/tests/test_processor_text_packing_gate.py
new file mode 100644
index 00000000000..2a3bea66468
--- /dev/null
+++ b/tests/test_processor_text_packing_gate.py
@@ -0,0 +1,289 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A processor `processing_class` on a text-only dataset may pack; media or mRoPE keeps the block."""
+
+from types import SimpleNamespace
+
+import pytest
+
+pytest.importorskip("torch")
+datasets = pytest.importorskip("datasets")
+from torch import nn # noqa: E402
+from transformers import ProcessorMixin # noqa: E402
+
+try:
+ import unsloth.trainer as trainer_module # noqa: E402
+ from unsloth.trainer import _is_text_only_dataset, _uses_multimodal_rope # noqa: E402
+except ImportError:
+ import unsloth
+ if getattr(unsloth, "DEVICE_TYPE", None) != "mlx":
+ raise
+ pytest.skip("unsloth.trainer is the MLX shim here", allow_module_level = True)
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)
+ monkeypatch.delenv("UNSLOTH_PROCESSOR_TEXT_PACKING", raising = False)
+ monkeypatch.setattr(trainer_module.trl, "__version__", "1.13.0")
+ for name in ("enable_padding_free_metadata", "enable_sample_packing"):
+ monkeypatch.setattr(trainer_module, name, lambda model, trainer: None)
+
+
+class _Proc(ProcessorMixin):
+ """A processor without running ProcessorMixin.__init__ (no real sub-processors needed)."""
+
+ def __init__(self, tokenizer = object()):
+ self.tokenizer = tokenizer
+
+
+class _Cfg:
+ def __init__(self, **extra):
+ self.model_type = "gemma4"
+ self.vision_config = {}
+ self.__dict__.update(extra)
+
+ def to_dict(self):
+ return dict(self.__dict__)
+
+
+class _Model(nn.Module):
+ def __init__(self, config = None):
+ super().__init__()
+ self.config = config if config is not None else _Cfg()
+
+ def forward(
+ self,
+ input_ids = None,
+ **kwargs,
+ ):
+ return None
+
+
+def _patched(raise_on = None):
+ calls = []
+
+ class _StubSFTTrainer:
+ def __init__(
+ self,
+ model = None,
+ args = None,
+ **kwargs,
+ ):
+ calls.append(
+ (bool(getattr(args, "packing", False)), getattr(args, "padding_free", None))
+ )
+ if raise_on is not None and (args.packing or args.padding_free):
+ raise ValueError(raise_on)
+ self.model = model
+ self.args = args
+
+ module = SimpleNamespace(SFTTrainer = _StubSFTTrainer)
+ trainer_module._patch_sft_trainer_auto_packing(module)
+ return module, calls
+
+
+def _config(packing = False):
+ return SimpleNamespace(packing = packing, padding_free = None, max_length = 512)
+
+
+TEXT = datasets.Dataset.from_dict({"text": ["a b c", "d e"]})
+
+
[email protected]("packing", [False, True])
+def test_text_only_processor_is_unblocked(packing):
+ module, _ = _patched()
+ t = module.SFTTrainer(
+ model = _Model(), args = _config(packing), train_dataset = TEXT, processing_class = _Proc()
+ )
+ assert t.args.padding_free is True
+ assert bool(t.args.packing) is packing
+
+
+def test_kill_switch_restores_the_block(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_PROCESSOR_TEXT_PACKING", "0")
+ module, _ = _patched()
+ t = module.SFTTrainer(
+ model = _Model(), args = _config(True), train_dataset = TEXT, processing_class = _Proc()
+ )
+ assert t.args.padding_free is False and t.args.packing is False
+
+
+def _blocked(**kwargs):
+ module, _ = _patched()
+ kw = dict(model = _Model(), args = _config(True), train_dataset = TEXT, processing_class = _Proc())
+ kw.update(kwargs)
+ t = module.SFTTrainer(**kw)
+ return t.args.padding_free is False and t.args.packing is False
+
+
+ "column",
+ ["image", "images", "audio", "input_features", "pixel_values", "video", "mm_token_type_ids"],
+)
+def test_media_columns_stay_blocked(column):
+ ds = datasets.Dataset.from_dict({"text": ["a"], column: [[0]]})
+ assert _blocked(train_dataset = ds)
+
+
+def test_an_image_feature_nested_in_messages_stays_blocked():
+ feats = datasets.Features(
+ {
+ "messages": [
+ {
+ "role": datasets.Value("string"),
+ "content": [{"type": datasets.Value("string"), "image": datasets.Image()}],
+ }
+ ]
+ }
+ )
+ ds = datasets.Dataset.from_dict(
+ {"messages": [[{"role": "user", "content": [{"type": "text", "image": None}]}]]},
+ features = feats,
+ )
+ assert not _is_text_only_dataset(ds)
+ assert _blocked(train_dataset = ds)
+
+
+ "column", ["image_path", "audio_url", "input_values", "image_patches", "Video_File"]
+)
+def test_media_named_string_columns_stay_blocked(column):
+ ds = datasets.Dataset.from_dict({"text": ["a"], column: ["x"]})
+ assert not _is_text_only_dataset(ds)
+ assert _blocked(train_dataset = ds)
+
+
+def test_an_image_path_string_inside_messages_stays_blocked():
+ ds = datasets.Dataset.from_dict(
+ {"messages": [[{"role": "user", "content": [{"type": "image", "image": "cat.png"}]}]]}
+ )
+ assert not _is_text_only_dataset(ds)
+ assert _blocked(train_dataset = ds)
+
+
+def _chat(*parts):
+ return datasets.Dataset.from_dict({"messages": [[{"role": "user", "content": list(parts)}]]})
+
+
+ "part",
+ [
+ {"type": "audio", "url": "speech.wav"},
+ {"type": "image", "url": "https://example.com/cat.png"},
+ {"type": "video", "path": "clip.mp4"},
+ {"type": "image", "base64": "aGk="},
+ ],
+)
+def test_a_media_part_named_only_by_its_type_stays_blocked(part):
+ ds = _chat({"type": "text", "text": "hi"}, part)
+ assert not _is_text_only_dataset(ds)
+ assert _blocked(train_dataset = ds)
+
+
+def test_text_parts_and_a_top_level_type_column_stay_text():
+ assert _is_text_only_dataset(_chat({"type": "text", "text": "hi"}))
+ assert _is_text_only_dataset(
+ datasets.Dataset.from_dict({"text": ["a"], "type": ["news"], "source": ["web"]})
+ )
+
+
[email protected]("model_type", ["mllama", "paligemma", "llava", "mistral3", "qwen2_vl"])
+def test_processors_outside_the_verified_families_stay_blocked(model_type):
+ assert _blocked(model = _Model(_Cfg(model_type = model_type)))
+
+
+ "model_type",
+ ["gemma3", "gemma3_text", "gemma4", "gemma4_text", "gemma4_unified", "gemma4_unified_text"],
+)
+def test_verified_families_are_unblocked(model_type):
+ assert not _blocked(model = _Model(_Cfg(model_type = model_type)))
+
+
[email protected]("value", ["false", "OFF", "no"])
+def test_kill_switch_spellings(monkeypatch, value):
+ monkeypatch.setenv("UNSLOTH_PROCESSOR_TEXT_PACKING", value)
+ assert _blocked()
+
+
+def test_unknown_schema_stays_blocked():
+ assert _blocked(train_dataset = [{"text": "a"}])
+
+
+def test_an_eval_split_with_images_stays_blocked():
+ img = datasets.Dataset.from_dict({"text": ["a"], "images": [[0]]})
+ assert _blocked(eval_dataset = {"a": TEXT, "b": img})
+ assert not _blocked(eval_dataset = {"a": TEXT})
+
+
+def test_custom_collator_and_tokenizerless_processor_stay_blocked():
+ assert _blocked(data_collator = lambda x: x)
+ assert _blocked(processing_class = _Proc(tokenizer = None))
+
+
+def test_multimodal_rope_models_stay_blocked():
+ """Qwen2-VL family builds the packed mask only from 4-row position_ids, so 2D ids leak."""
+ cfg = _Cfg(
+ model_type = "qwen2_5_vl", text_config = {"rope_parameters": {"mrope_section": [1, 1, 0]}}
+ )
+ assert _uses_multimodal_rope(cfg)
+ assert _blocked(model = _Model(cfg))
+ assert not _uses_multimodal_rope(
+ _Cfg(text_config = {"rope_parameters": {"rope_type": "default"}})
+ )
+
+
+def test_a_transformed_dataset_stays_blocked():
+ """with_transform rows can add images the stored columns do not show."""
+ ds = TEXT.with_transform(lambda batch: batch)
+ assert not _is_text_only_dataset(ds)
+ assert _blocked(train_dataset = ds)
+
+
[email protected]("version", ["0.22.2", "0.24.0", "1.9.2"])
+def test_trl_before_1_10_never_tries_a_processor(monkeypatch, version):
+ """TRL < 1.10 refuses packing / padding-free for any processor: no doomed __init__ first."""
+ monkeypatch.setattr(trainer_module.trl, "__version__", version)
+ module, calls = _patched(raise_on = "Packing is not supported for vision-language models.")
+ t = module.SFTTrainer(
+ model = _Model(), args = _config(True), train_dataset = TEXT, processing_class = _Proc()
+ )
+ assert calls == [(False, False)]
+ assert t.args.padding_free is False and t.args.packing is False
+
+
+ "message",
+ [
+ "Packing is not supported for vision-language models. Please set `packing=False` in the SFTConfig.",
+ "Padding-free training is yet not supported for vision-language models. Please set "
+ "`padding_free=False` in the `SFTConfig`.",
+ "Packing is not supported for vision datasets. Please set `packing=False` in the SFTConfig.",
+ "Padding-free training is yet not supported for vision datasets. Please set `padding_free=False` in "
+ "the `SFTConfig`.",
+ ],
+)
+def test_a_trl_refusal_falls_back_to_the_old_block(message):
+ """A refusal TRL still raises falls back to exactly the old behaviour."""
+ module, calls = _patched(raise_on = message)
+ packing = message.startswith("Packing")
+ t = module.SFTTrainer(
+ model = _Model(), args = _config(packing), train_dataset = TEXT, processing_class = _Proc()
+ )
+ assert len(calls) == 2
+ assert calls[-1] == (False, False)
+ assert t.args.padding_free is False and t.args.packing is False
+
+
+def test_an_unrelated_value_error_still_propagates():
+ module, _ = _patched(raise_on = "something else entirely")
+ with pytest.raises(ValueError, match = "something else"):
+ module.SFTTrainer(
+ model = _Model(), args = _config(False), train_dataset = TEXT, processing_class = _Proc()
+ )
+
+
+if __name__ == "__main__":
+ raise SystemExit(pytest.main([__file__, "-q"]))
diff --git a/tests/utils/test_flash_impl.py b/tests/utils/test_flash_impl.py
index 1b410925a68..80888a51b3b 100644
--- a/tests/utils/test_flash_impl.py
+++ b/tests/utils/test_flash_impl.py
@@ -536,9 +536,12 @@ def test_fa4_hd256_window_keeps_segments_route(real_fa4):
not has_real_cuda(), reason = "needs CUDA tensors (the wrapper only takes CUDA rows)"
)
@pytest.mark.skipif(torch.version.hip is not None, reason = "torch varlen is off on ROCm")
[email protected]("efficient", [True, False])
@pytest.mark.parametrize("window", [None, 2])
@pytest.mark.parametrize("allowed", [True, False])
-def test_hf_wrapper_skips_torch_varlen_for_shapes_the_impl_lacks(monkeypatch, allowed, window):
+def test_hf_wrapper_skips_torch_varlen_for_shapes_the_impl_lacks(
+ monkeypatch, allowed, window, efficient
+):
from unsloth.utils import hf_packed_attention as hpa
from unsloth.utils import attention_dispatch as ad
@@ -569,8 +572,11 @@ def test_hf_wrapper_skips_torch_varlen_for_shapes_the_impl_lacks(monkeypatch, al
monkeypatch.setattr(hpa, "_TORCH_VARLEN", [fake_varlen])
monkeypatch.setattr(hpa, "_TORCH_VARLEN_DEVICES", {})
monkeypatch.setattr(hpa, "_sdpa_requested", lambda q, k: False)
- # No FA2 / xformers either, so a refused torch varlen goes back to the wrapped sdpa.
+ # No FA2 / xformers either: a refused torch varlen takes the efficient kernel, else the wrapped sdpa.
monkeypatch.setattr(ad, "select_attention_backend", lambda use_varlen = False: ad.SDPA)
+ if not efficient:
+ monkeypatch.setattr(hpa, "_EFFICIENT_VARLEN", [False])
+ before = hpa.HF_PACKED_ATTENTION_STATS.get("efficient", 0)
monkeypatch.setattr(
fi, "flash_impl_allows", lambda head_dim, window, device: asked.append(window) or allowed
)
@@ -596,5 +602,8 @@ def test_hf_wrapper_skips_torch_varlen_for_shapes_the_impl_lacks(monkeypatch, al
assert asked == [window]
if allowed:
assert used and not seen
+ elif efficient:
+ assert not used and not seen
+ assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before + 1
else:
assert out == ("orig", None) and not used
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index f5a7fad720a..444ec1d4771 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -552,7 +552,13 @@ _FLEX_LARGE_HEAD_DIM_ENV_VAR = "UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM"
# gemma2 measured slower under flex. gemma4 shares one _attn_implementation across all layers, so
# flex would also pull its 25 sliding layers off unsloth_zoo's faster gemma4_flash_sliding router,
# which is registered under "sdpa".
-_FLEX_LARGE_HEAD_DIM_EXCLUDED_MODELS = ("gemma2", "gemma4", "gemma4_text")
+_FLEX_LARGE_HEAD_DIM_EXCLUDED_MODELS = (
+ "gemma2",
+ "gemma4",
+ "gemma4_text",
+ "gemma4_unified",
+ "gemma4_unified_text",
+)
# Interface dispatch, and a BlockMask from create_causal_mask rather than a hand-built 4D mask.
_FLEX_INTERFACE_MARKERS = ("ALL_ATTENTION_FUNCTIONS", "create_causal_mask")
_FLEX_SUPPORT_FORCED = set()
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index dee005012c5..0e4dfd42cc9 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -290,6 +290,148 @@ def _is_vision_eval_dataset(dataset, *, unknown_is_vision = False) -> bool:
return _is_vision_dataset(dataset, unknown_is_vision = unknown_is_vision)
+_MULTIMODAL_DATASET_KEYS = _VISION_DATASET_KEYS | frozenset(
+ {
+ "audio",
+ "audios",
+ "audio_values",
+ "input_features",
+ "input_features_mask",
+ "pixel_values_images",
+ }
+)
+_MULTIMODAL_FEATURE_TYPES = ("Image", "Audio", "Video", "Pdf")
+# Media-named keys (image_path, audio_url, ...) may be loaded by the processor even as strings.
+_MEDIA_NAME_PARTS = ("image", "audio", "video", "pixel", "input_values", "input_features")
+
+
+def _is_media_name(name) -> bool:
+ if not isinstance(name, str):
+ return False
+ lowered = name.lower()
+ return lowered in _MULTIMODAL_DATASET_KEYS or any(part in lowered for part in _MEDIA_NAME_PARTS)
+
+
+def _features_have_media(features) -> bool:
+ stack = [(features, False)]
+ seen = 0
+ while stack:
+ node, nested = stack.pop()
+ seen += 1
+ if seen > 10_000:
+ return True # pathological schema: assume media rather than guess
+ if type(node).__name__ in _MULTIMODAL_FEATURE_TYPES:
+ return True
+ if isinstance(node, dict):
+ if any(_is_media_name(key) for key in node):
+ return True
+ # Only {"type", "text"} content parts are provably text.
+ if nested and "type" in node and set(node) - {"type", "text"}:
+ return True
+ stack.extend((value, True) for value in node.values())
+ elif isinstance(node, (list, tuple)):
+ stack.extend((value, True) for value in node)
+ else:
+ inner = getattr(node, "feature", None) # Sequence / List / LargeList
+ if inner is not None:
+ stack.append((inner, True))
+ return False
+
+
+def _is_text_only_dataset(dataset) -> bool:
+ """Positively text-only: unknown schemas (lists, feature-less streams) count as media."""
+ if dataset is None:
+ return False
+ column_names = getattr(dataset, "column_names", None)
+ if column_names is None:
+ return False
+ if any(_is_media_name(name) for name in column_names):
+ return False
+ # with_transform / set_transform rows can carry images the stored columns do not show.
+ if (getattr(dataset, "format", None) or {}).get("type") == "custom":
+ return False
+ features = getattr(dataset, "features", None)
+ if features is None:
+ # Streaming datasets lack types, so embedded images cannot be ruled out.
+ return False
+ return not _features_have_media(features)
+
+
+def _is_text_only_eval_dataset(dataset) -> bool:
+ if dataset is None:
+ return True
+ if isinstance(dataset, dict):
+ return all(_is_text_only_dataset(split) for split in dataset.values())
+ return _is_text_only_dataset(dataset)
+
+
+def _uses_multimodal_rope(model_config) -> bool:
+ """mRoPE models build a packed mask only from 4-row position_ids; 2D ids leak across docs."""
+ if model_config is None:
+ return True # unknown: assume the worst
+ try:
+ stack = [model_config.to_dict()]
+ except Exception:
+ return True
+ seen = 0
+ while stack:
+ node = stack.pop()
+ seen += 1
+ if seen > 10_000:
+ return True
+ if isinstance(node, dict):
+ for key, value in node.items():
+ if key in ("mrope_section", "mrope_interleaved"):
+ return True
+ if key in ("rope_type", "type") and value == "mrope":
+ return True
+ if isinstance(value, (dict, list, tuple)):
+ stack.append(value)
+ elif isinstance(node, (list, tuple)):
+ stack.extend(x for x in node if isinstance(x, (dict, list, tuple)))
+ return False
+
+
+_PROCESSOR_TEXT_PACKING_MODELS = frozenset(
+ {"gemma3", "gemma3_text", "gemma4", "gemma4_text", "gemma4_unified", "gemma4_unified_text"}
+)
+
+
+def _processor_text_only(
+ processing_class, data_collator, train_dataset, eval_dataset, model_config
+) -> bool:
+ """A processor on text-only rows tokenizes like its tokenizer, so packing is safe."""
+ if os.environ.get("UNSLOTH_PROCESSOR_TEXT_PACKING", "1").lower() in ("0", "false", "no", "off"):
+ return False
+ # Only families whose packed text rows are verified to equal each document alone.
+ if getattr(model_config, "model_type", None) not in _PROCESSOR_TEXT_PACKING_MODELS:
+ return False
+ # TRL < 1.10 refuses packing / padding-free for any processor; skip a doomed __init__.
+ if Version(trl.__version__) < Version("1.10.0"):
+ return False
+ if data_collator is not None:
+ return False
+ if _uses_multimodal_rope(model_config):
+ return False
+ if getattr(processing_class, "tokenizer", None) is None:
+ return False
+ return _is_text_only_dataset(train_dataset) and _is_text_only_eval_dataset(eval_dataset)
+
+
+# TRL < 1.10 raises the first two for any processor; 1.10+ the last two when a sample holds images.
+_TRL_VLM_PACKING_REJECTIONS = (
+ "packing is not supported for vision-language models",
+ "padding-free training is yet not supported for vision-language models",
+ "packing is not supported for vision datasets",
+ "padding-free training is yet not supported for vision datasets",
+)
+
+
+def _is_trl_vlm_packing_rejection(exc: Exception) -> bool:
+ message = str(exc).lower()
+ return any(msg in message for msg in _TRL_VLM_PACKING_REJECTIONS)
+
+
_HYBRID_CONFIG_MARKERS = (
"linear_conv_kernel_dim",
"linear_key_head_dim",
@@ -1577,6 +1719,9 @@ def _patch_sft_trainer_auto_packing(trl_module):
train_dataset = args[3] if len(args) >= 4 else kwargs.get("train_dataset")
eval_dataset = args[4] if len(args) >= 5 else kwargs.get("eval_dataset")
is_processor = isinstance(processing_class, ProcessorMixin)
+ processor_text_only = is_processor and _processor_text_only(
+ processing_class, data_collator, train_dataset, eval_dataset, model_config
+ )
is_auto_processor_vlm = is_vlm and processing_class is None
is_vision_dataset = (
data_collator is None
@@ -1609,7 +1754,7 @@ def _patch_sft_trainer_auto_packing(trl_module):
# condition the post-init check skips on, so leaving it armed is free.
blocked = (
(data_collator is not None)
- or is_processor
+ or (is_processor and not processor_text_only)
or is_auto_processor_vlm
or is_vision_dataset
or is_unsupported_model
@@ -1676,7 +1821,23 @@ def _patch_sft_trainer_auto_packing(trl_module):
try:
original_init(self, *args, **kwargs)
except ValueError as exc:
- if packing_active and _should_skip_auto_packing_error(exc):
+ if processor_text_only and not blocked and _is_trl_vlm_packing_rejection(exc):
+ if requested_pack:
+ logger.warning(
+ "Unsloth: packing=True ignored (this TRL version refuses it for processors)."
+ )
+ else:
+ logger.info("Unsloth: padding-free disabled for the processor (%s).", exc)
+ for _attr in ("packing", "padding_free"):
+ if hasattr(config_arg, _attr):
+ setattr(config_arg, _attr, False)
+ packing_active = False
+ auto_padding_free_active = False
+ blocked = True
+ # Release the failed attempt's frames (may hold a loaded model) before retrying.
+ exc.__traceback__ = None
+ original_init(self, *args, **kwargs)
+ elif packing_active and _should_skip_auto_packing_error(exc):
logger.info(
"Unsloth: Auto sample packing failed because trainer reported an incompatible setup (%s).",
exc,
diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.py
index a1c0b796fa5..63cf90bd5ab 100644
--- a/unsloth/utils/hf_packed_attention.py
+++ b/unsloth/utils/hf_packed_attention.py
@@ -1,7 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Packed rows on the transformers modeling path run a varlen flash kernel (torch's, else xformers /
-flash-attn) instead of SDPA over a dense block-causal mask; anything else reaches the wrapped "sdpa"."""
+flash-attn) instead of SDPA over a dense block-causal mask; heads past flash's 256 (Gemma-4 global
+layers, 512) and shapes an FA4 / FA3 impl refuses run SDPA's memory-efficient kernel over the same
+cu_seqlens. Anything else reaches the wrapped "sdpa"."""
import inspect
import logging
@@ -15,7 +17,8 @@ __all__ = ["enable_hf_packed_attention", "HF_PACKED_ATTENTION_STATS"]
_FLAG = "_unsloth_hf_packed_varlen"
_ENV = "UNSLOTH_HF_PACKED_VARLEN"
_ORIG_SDPA = [None]
-HF_PACKED_ATTENTION_STATS = {"fast": 0, "fallback": 0, "mask_checks": 0}
+# efficient: the subset of fast calls that ran SDPA's memory-efficient kernel over cu_seqlens.
+HF_PACKED_ATTENTION_STATS = {"fast": 0, "fallback": 0, "mask_checks": 0, "efficient": 0}
# Layers of one window share a mask object: window -> [weakref, (cu_seqlens, versions), verdict].
_MASK_CHECK = {}
_MASK_CHECK_ROWS = 1024
@@ -149,6 +152,136 @@ def _torch_varlen(device):
return fn if ok else None
+_EFFICIENT_ENV = "UNSLOTH_HF_PACKED_EFFICIENT"
+# Above flash's 256 cap (Gemma-4 global layers are 512); wider heads are untested.
+_EFFICIENT_MAX_HEAD_DIM = 512
+_EFFICIENT_VARLEN = []
+_ACCELERATOR_ERROR = getattr(torch, "AcceleratorError", None) # torch 2.8+
+_EFFICIENT_DEVICES = {}
+
+
+def _efficient_varlen_takes(query, window) -> bool:
+ """SDPA's memory-efficient kernel over cu_seqlens: no T x T bias. Off under any policy asking for SDPA."""
+ if os.environ.get(_EFFICIENT_ENV, "1").lower() in ("0", "false", "no", "off"):
+ return False
+ if not _EFFICIENT_VARLEN:
+ ok = True
+ try:
+ schema = torch.ops.aten._efficient_attention_forward.default._schema
+ names = {argument.name for argument in schema.arguments}
+ ok = ok and {"cu_seqlens_q", "scale", "window_size"} <= names
+ schema = torch.ops.aten._efficient_attention_backward.default._schema
+ ok = ok and "window_size" in {argument.name for argument in schema.arguments}
+ except Exception:
+ ok = False
+ _EFFICIENT_VARLEN.append(ok)
+ head_dim = query.shape[-1]
+ if (
+ not _EFFICIENT_VARLEN[0]
+ or torch.version.hip is not None
+ or head_dim > _EFFICIENT_MAX_HEAD_DIM
+ or head_dim % 8 != 0
+ # Its backward is nondeterministic.
+ or torch.are_deterministic_algorithms_enabled()
+ or not torch.backends.cuda.mem_efficient_sdp_enabled()
+ ):
+ return False
+ ok = _EFFICIENT_DEVICES.get(query.device.index)
+ if ok is None:
+ ok = torch.cuda.get_device_capability(query.device)[0] >= 8
+ _EFFICIENT_DEVICES[query.device.index] = ok
+ if not ok:
+ return False
+ from torch.nn.attention import SDPBackend
+ from .attention_dispatch import _sdpa_backend_state
+
+ # A math-first priority (or a disabled efficient backend) for these shapes keeps SDPA.
+ n_heads = query.shape[1]
+ _, choice = _sdpa_backend_state(
+ str(query.device), query.dtype, head_dim, n_heads, n_heads, True, False
+ )
+ return choice not in (int(SDPBackend.MATH), -1)
+
+
+class _EfficientVarlenAttention(torch.autograd.Function):
+ """(1, T, H, D) Q / K / V over cu_seqlens, causal (and windowed) inside each segment."""
+
+ @staticmethod
+ def forward(ctx, query, key, value, cu_seqlens, max_seqlen, scale, window):
+ out, logsumexp, seed, offset, _, _ = torch.ops.aten._efficient_attention_forward(
+ query,
+ key,
+ value,
+ None,
+ cu_seqlens,
+ cu_seqlens,
+ max_seqlen,
+ max_seqlen,
+ 0.0,
+ 1, # causal from the top left of each segment
+ any(ctx.needs_input_grad[:3]),
+ scale = scale,
+ window_size = window,
+ )
+ ctx.save_for_backward(query, key, value, out, logsumexp, cu_seqlens, seed, offset)
+ ctx.max_seqlen, ctx.scale, ctx.window = max_seqlen, scale, window
+ return out
+
+ @staticmethod
+ def backward(ctx, grad_out):
+ query, key, value, out, logsumexp, cu_seqlens, seed, offset = ctx.saved_tensors
+ grad_q, grad_k, grad_v, _ = torch.ops.aten._efficient_attention_backward(
+ grad_out.contiguous(),
+ query,
+ key,
+ value,
+ None,
+ out,
+ cu_seqlens,
+ cu_seqlens,
+ ctx.max_seqlen,
+ ctx.max_seqlen,
+ logsumexp,
+ 0.0,
+ seed,
+ offset,
+ 1,
+ False,
+ scale = ctx.scale,
+ window_size = ctx.window,
+ )
+ return grad_q, grad_k, grad_v, None, None, None, None
+
+
+def _efficient_varlen(query, key, value, cu_seqlens, max_seqlen, scale, window):
+ """Q (1, H, T, D), K / V (1, H_kv, T, D) -> (1, T, H, D)."""
+ _, n_heads, total, head_dim = query.shape
+ n_kv_heads = key.shape[1]
+
+ def heads_last(x):
+ x = x[0].transpose(0, 1).unsqueeze(0)
+ # The kernel needs a unit stride on the head dim; a refusal would turn the route off for good.
+ return x if x.stride(-1) == 1 else x.contiguous()
+
+ q, k, v = heads_last(query), heads_last(key), heads_last(value)
+ if n_kv_heads != n_heads:
+ # The kernel has no GQA: repeat K / V heads as transformers' sdpa path does (autograd sums back).
+ groups = n_heads // n_kv_heads
+ k = (
+ k.unsqueeze(3)
+ .expand(1, total, n_kv_heads, groups, head_dim)
+ .reshape(1, total, n_heads, head_dim)
+ )
+ v = (
+ v.unsqueeze(3)
+ .expand(1, total, n_kv_heads, groups, head_dim)
+ .reshape(1, total, n_heads, head_dim)
+ )
+ return _EfficientVarlenAttention.apply(
+ q, k, v, cu_seqlens.to(torch.int32), int(max_seqlen), float(scale), window
+ )
+
+
_SDPA_MATH = []
@@ -221,6 +354,7 @@ def _sdpa_packed_varlen(
bsz, n_heads, q_len, head_dim = query.shape
seq_info = torch_varlen = backend = None
+ efficient = False
window = _sliding_window(module, kwargs)
if (
bsz == 1
@@ -230,9 +364,8 @@ def _sdpa_packed_varlen(
and key.shape[0] == value.shape[0] == 1
and key.shape[1] == value.shape[1]
and key.shape[2] == value.shape[2] == q_len
- # Not MLA (192 / 128) nor Gemma-4 global (512).
+ # Not MLA (192 / 128).
and key.shape[-1] == value.shape[-1] == head_dim
- and head_dim <= 256
and head_dim % 8 == 0
and query.dtype in (torch.float16, torch.bfloat16)
and query.dtype == key.dtype == value.dtype
@@ -246,27 +379,35 @@ def _sdpa_packed_varlen(
and n_heads % key.shape[1] == 0
and window is not _NO_WINDOW
and not _softcapped(module, kwargs)
- and not _sdpa_requested(query, key)
):
- torch_varlen = _torch_varlen(query.device)
- # An FA4 / FA3 impl behind torch's flash op lacks some shapes (sm100: hd 136-248, hd256 + window).
- if torch_varlen is not None and not flash_impl_allows(head_dim, window, query.device):
- torch_varlen = None
- backend = select_attention_backend(use_varlen = True)
- # flash-attn's ROCm backend lists sliding windows as work in progress: windowed rows keep sdpa there.
- if window is not None and torch.version.hip is not None:
- backend = None
- # Flash-2 varlen backward faults past int32 indexing. Keyed on training / grad mode, not
- # requires_grad, so a reentrant checkpoint's no_grad forward and its recompute agree.
- if (
- not _VARLEN_INT32_GUARD_DISABLED
- and (getattr(module, "training", False) or torch.is_grad_enabled())
- and _varlen_backward_overflows_int32(
- kwargs["packed_seq_lengths"].numel(), q_len, n_heads, head_dim
+ if head_dim > 256:
+ # Past every flash kernel (Gemma-4 global layers are 512): the efficient kernel, per segment.
+ efficient = _efficient_varlen_takes(query, window)
+ elif not _sdpa_requested(query, key):
+ torch_varlen = _torch_varlen(query.device)
+ # An FA4 / FA3 impl behind torch's flash op lacks some shapes (sm100: hd 136-248, hd256 + window).
+ refused = torch_varlen is not None and not flash_impl_allows(
+ head_dim, window, query.device
)
- ):
- torch_varlen, backend = None, None
- if torch_varlen is not None or backend in (FLASH_VARLEN, XFORMERS):
+ if refused:
+ torch_varlen = None
+ backend = select_attention_backend(use_varlen = True)
+ # flash-attn's ROCm backend lists sliding windows as work in progress: windowed rows keep sdpa there.
+ if window is not None and torch.version.hip is not None:
+ backend = None
+ # Flash-2 varlen backward faults past int32; keyed on grad mode so checkpoint recompute agrees.
+ if (
+ not _VARLEN_INT32_GUARD_DISABLED
+ and (getattr(module, "training", False) or torch.is_grad_enabled())
+ and _varlen_backward_overflows_int32(
+ kwargs["packed_seq_lengths"].numel(), q_len, n_heads, head_dim
+ )
+ ):
+ torch_varlen, backend, refused = None, None, False
+ # Refused shape and no flash-attn / xformers: the efficient kernel.
+ if refused and backend not in (FLASH_VARLEN, XFORMERS):
+ efficient = _efficient_varlen_takes(query, window)
+ if efficient or torch_varlen is not None or backend in (FLASH_VARLEN, XFORMERS):
seq_info = get_packed_info_from_kwargs(kwargs, query.device)
if seq_info is None or not _mask_is_block_causal(attention_mask, seq_info[1], q_len, window):
HF_PACKED_ATTENTION_STATS["fallback"] += 1
@@ -285,6 +426,35 @@ def _sdpa_packed_varlen(
scale = head_dim**-0.5 if scaling is None else scaling
n_kv_heads = key.shape[1]
_, cu_seqlens, max_seqlen = seq_info
+ if efficient:
+ try:
+ out = _efficient_varlen(query, key, value, cu_seqlens, max_seqlen, scale, window)
+ except torch.OutOfMemoryError:
+ raise
+ except RuntimeError as exc:
+ # A CUDA fault is not a refusal: let it surface.
+ if _ACCELERATOR_ERROR is not None and isinstance(exc, _ACCELERATOR_ERROR):
+ raise
+ # A build without this kernel shape refuses at forward: stop using it, this call goes to sdpa.
+ _EFFICIENT_VARLEN[:] = [False]
+ logging.getLogger(__name__).warning(
+ f"Unsloth: efficient varlen attention unavailable, packed rows past head dim 256 use sdpa ({exc})"
+ )
+ HF_PACKED_ATTENTION_STATS["fast"] -= 1
+ HF_PACKED_ATTENTION_STATS["fallback"] += 1
+ return orig(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ dropout = dropout,
+ scaling = scaling,
+ is_causal = is_causal,
+ **kwargs,
+ )
+ HF_PACKED_ATTENTION_STATS["efficient"] += 1
+ return out, None
if torch_varlen is not None:
gqa, k_in, v_in = {}, key, value
if n_kv_heads != n_heads:
@@ -310,6 +480,8 @@ def _sdpa_packed_varlen(
except torch.OutOfMemoryError:
raise
except RuntimeError as exc:
+ if _ACCELERATOR_ERROR is not None and isinstance(exc, _ACCELERATOR_ERROR):
+ raise
# A build whose varlen picks a backend without causal GQA (cuDNN) refuses at forward: stop
# using it and give this call to the wrapped sdpa.
_TORCH_VARLEN[:] = [None]