Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Run packed rows of head-dim-routed flex decoders through varlen SDPA (#15514)

Daniel Han92ecb5219160
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.py
index 1eb50bfe3ff..4e6f6e3550e 100644
--- a/tests/test_hf_packed_attention.py
+++ b/tests/test_hf_packed_attention.py
@@ -662,6 +662,141 @@ def test_int32_guard_follows_training_or_grad_mode(monkeypatch, mode):
assert out == ("orig", None)
+# (L + 128 * n) * H * round32(D) at H = 8, D = 64: 400 tokens' worth per group.
+_SMALL_INT32_LIMIT = 400 * 8 * 64
+
+
+def test_int32_safe_groups_split_at_document_boundaries(monkeypatch):
+ import unsloth.utils.attention_dispatch as ad
+
+ monkeypatch.setattr(ad, "_INT32_ELEMENTS", _SMALL_INT32_LIMIT)
+ lengths = [37, 100, 2, 5, 230, 64, 9]
+ groups = hpa._int32_safe_groups(lengths, 8, 64)
+ assert groups[0][0] == 0 and groups[-1][1] == len(lengths) and groups[-1][3] == sum(lengths)
+ for (first, end, start, stop, longest), following in zip(groups, groups[1:] + [None]):
+ assert stop - start == sum(lengths[first:end]) and longest == max(lengths[first:end])
+ assert not ad._varlen_backward_overflows_int32(end - first, stop - start, 8, 64)
+ if following is not None:
+ assert following[0] == end and following[2] == stop
+ assert len(groups) > 1
+ # One document alone past the limit cannot be split.
+ assert hpa._int32_safe_groups([37, 300], 8, 64) is None
+
+
+@_CUDA
[email protected]("window", [None, 4])
[email protected]("route", ["torch", "dispatch"])
+def test_int32_guard_splits_documents_instead_of_the_dense_fallback(monkeypatch, route, window):
+ """A row past the varlen backward's int32 limit runs the kernel per group of documents, and the
+ output and gradients are the unsplit kernel's; SDPA over the dense mask is never reached."""
+ import unsloth.utils.attention_dispatch as ad
+
+ if route == "torch" and hpa._torch_varlen(torch.device("cuda", 0)) is None:
+ pytest.skip("this torch has no window_size varlen_attn")
+ if route == "dispatch":
+ if not (ad.HAS_FLASH_ATTENTION or ad.HAS_XFORMERS):
+ pytest.skip("needs xformers or flash-attn")
+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)
+ # Where torch's packed kernels beat xformers the dispatcher picks sdpa: pin the varlen one.
+ backend = ad.FLASH_VARLEN if ad.HAS_FLASH_ATTENTION else ad.XFORMERS
+ monkeypatch.setattr(ad, "select_attention_backend", lambda **kwargs: backend)
+
+ def _dense(*args, **kwargs):
+ raise AssertionError("packed row fell back to SDPA over the dense mask")
+
+ hpa._ORIG_SDPA[0] = _dense
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ torch.manual_seed(0)
+ lengths = [37, 100, 2, 5, 230, 64, 9]
+ T, H, n_kv, D = sum(lengths), 8, 2, 64
+ module = types.SimpleNamespace(
+ is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = window
+ )
+ q, k, v = (
+ torch.randn(1, h, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ for h in (H, n_kv, n_kv)
+ )
+ mask = (
+ _block_causal(lengths, "cuda")
+ if window is None
+ else _hf_window_mask(lengths, window, "cuda")
+ )
+ psl = torch.tensor(lengths, dtype = torch.int32)
+ g = torch.randn(1, T, H, D, device = "cuda", dtype = torch.bfloat16)
+
+ def run():
+ out, _ = hpa._sdpa_packed_varlen(
+ module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl
+ )
+ return (out, *torch.autograd.grad(out, (q, k, v), g))
+
+ whole = run()
+ monkeypatch.setattr(ad, "_INT32_ELEMENTS", _SMALL_INT32_LIMIT)
+ assert ad._varlen_backward_overflows_int32(len(lengths), T, H, D)
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ split = run()
+ assert hpa.HF_PACKED_ATTENTION_STATS["int32_split"] == before["int32_split"] + 1
+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"]
+ for a, b in zip(split, whole):
+ torch.testing.assert_close(a, b, atol = 0, rtol = 0)
+
+
+def test_block_varlen_kwargs_sends_rows_past_int32_to_the_eager_wrapper(monkeypatch):
+ """The traced kernel cannot split a row, so the eager wrapper (which can) runs it; a document that
+ alone overflows keeps the previous dense path."""
+ import unsloth.utils.attention_dispatch as ad
+
+ hpa._ORIG_SDPA[0] = _plain_sdpa
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
+ monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: False)
+ monkeypatch.setattr(hpa, "_BLOCK_PREP", {})
+ monkeypatch.setattr(ad, "_INT32_ELEMENTS", 200 * 4 * 32)
+ layer = _block_layer(head_dim = 32, n_heads = 4)
+ for lengths, expected in (([20, 30, 40], hpa._LEAVE_BLOCK), ([20, 80], None)):
+ hidden = torch.randn(1, sum(lengths), 8)
+ kwargs = {
+ "packed_seq_lengths": torch.tensor(lengths),
+ "attention_mask": _block_causal(lengths),
+ }
+ before = hpa.HF_PACKED_ATTENTION_STATS["compiled"]
+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == expected
+ assert hpa.HF_PACKED_ATTENTION_STATS["compiled"] == before
+
+
+def test_traced_leave_block_call_runs_the_eager_wrapper(monkeypatch):
+ seen = []
+
+ @torch.compiler.disable
+ def eager(module, q, k, v, mask, **kwargs):
+ seen.append(set(kwargs))
+ return q.transpose(1, 2) * 3, None
+
+ hpa._ORIG_SDPA[0] = lambda *args, **kwargs: pytest.fail("the dense path stayed in the graph")
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ monkeypatch.setattr(hpa, "_sdpa_packed_varlen_eager", eager)
+ lengths = [3, 3]
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = True)
+ q = torch.randn(1, 4, 6, 16, dtype = torch.bfloat16)
+ mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)
+ torch._dynamo.reset()
+ fn = torch.compile(
+ lambda q: hpa._sdpa_packed_varlen(
+ module,
+ q,
+ q,
+ q,
+ mask,
+ packed_seq_lengths = psl,
+ **{hpa.BLOCK_VARLEN_KWARG: hpa._LEAVE_BLOCK},
+ )[0],
+ backend = "eager",
+ )
+ torch.testing.assert_close(fn(q), q.transpose(1, 2) * 3)
+ assert seen and all(hpa.BLOCK_VARLEN_KWARG not in kwargs for kwargs in seen)
+
+
@_CUDA
@_TORCH_VARLEN_HOST
def test_a_refusing_torch_varlen_hands_the_call_back(monkeypatch):
diff --git a/tests/test_packed_varlen_over_flex.py b/tests/test_packed_varlen_over_flex.py
new file mode 100644
index 00000000000..625d098e5b5
--- /dev/null
+++ b/tests/test_packed_varlen_over_flex.py
@@ -0,0 +1,309 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""Packed / padding-free training moves a decoder routed to flex only for its head dim onto "sdpa",
+where the packed varlen wrapper runs; every other routing is left alone."""
+
+import types
+
+import pytest
+import torch
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+
+import unsloth # noqa: F401 (must precede transformers)
+import unsloth.models._utils as u
+from unsloth.utils import hf_packed_attention as hpa
+
+transformers = pytest.importorskip("transformers")
+
+
+def _gemma(
+ head_dim = 256,
+ impl = "flex_attention",
+ model_type = None,
+ dtype = torch.bfloat16,
+):
+ config = transformers.GemmaConfig(
+ vocab_size = 128,
+ hidden_size = 256,
+ intermediate_size = 256,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = head_dim,
+ )
+ config._attn_implementation = impl
+ config.dtype = dtype
+ if model_type is not None:
+ config.model_type = model_type
+ return config
+
+
+def routed(monkeypatch):
+ """A box where head_dim 256 routes to flex and torch's varlen kernel runs."""
+ for name in ("UNSLOTH_PACKED_VARLEN_OVER_FLEX", "UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM"):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setattr(u, "_flex_kernels_fit_large_head_dim", lambda: True)
+ monkeypatch.setattr(u, "_sdpa_reaches_cudnn_at_head_dim_256", lambda: False)
+ monkeypatch.setattr(hpa, "_ORIG_SDPA", [lambda *args, **kwargs: None])
+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ # SDPA backend state for these shapes: flash enabled, flash picked (not math).
+ monkeypatch.setattr(hpa, "_SDPA_MATH", [-1, lambda *args: (True, 0)])
+
+
+def test_head_dim_flex_moves_to_sdpa(routed):
+ config = _gemma()
+ assert u.use_sdpa_for_packed_head_dim_flex(types.SimpleNamespace(config = config))
+ assert config._attn_implementation == "sdpa"
+
+
+ "why",
+ [
+ "kill_switch",
+ "flex_forced",
+ "flex_refused",
+ "gemma3_family",
+ "head_dim_512",
+ "head_dim_128",
+ "already_sdpa",
+ "no_torch_varlen",
+ "one_device_without_varlen",
+ "wrapper_not_installed",
+ "cudnn_takes_head_dim_256",
+ ],
+)
+def test_every_other_routing_is_left_alone(routed, monkeypatch, why):
+ config = _gemma()
+ if why == "kill_switch":
+ monkeypatch.setenv("UNSLOTH_PACKED_VARLEN_OVER_FLEX", "0")
+ elif why == "flex_forced":
+ monkeypatch.setenv("UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM", "1")
+ elif why == "flex_refused":
+ monkeypatch.setenv("UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM", "0")
+ elif why == "gemma3_family":
+ config = _gemma(model_type = "gemma3_text")
+ elif why == "head_dim_512":
+ config = _gemma(head_dim = 512)
+ elif why == "head_dim_128":
+ config = _gemma(head_dim = 128)
+ elif why == "already_sdpa":
+ config = _gemma(impl = "sdpa")
+ elif why == "no_torch_varlen":
+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)
+ elif why == "one_device_without_varlen":
+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None if device.index else object())
+ elif why == "wrapper_not_installed":
+ monkeypatch.setattr(hpa, "_ORIG_SDPA", [None])
+ elif why == "cudnn_takes_head_dim_256":
+ monkeypatch.setattr(u, "_sdpa_reaches_cudnn_at_head_dim_256", lambda: True)
+ before = config._attn_implementation
+ assert not u.use_sdpa_for_packed_head_dim_flex(types.SimpleNamespace(config = config))
+ assert config._attn_implementation == before
+
+
+def deterministic():
+ previous = torch.are_deterministic_algorithms_enabled()
+ torch.use_deterministic_algorithms(True, warn_only = True)
+ yield
+ torch.use_deterministic_algorithms(previous)
+
+
+ "why",
+ [
+ "float32",
+ "float32_embeddings",
+ "dtype_unknown",
+ "deterministic",
+ "flash_sdpa_disabled",
+ "math_first_priority",
+ "softcap_config",
+ "sinks_module",
+ "no_sdpa_model_type",
+ ],
+)
+def test_rows_the_varlen_wrapper_declines_stay_on_flex(routed, monkeypatch, request, why):
+ """The wrapper would send these to dense-mask SDPA (slower than flex, ~2x memory at head_dim 256)."""
+ config = _gemma()
+ model = types.SimpleNamespace(config = config)
+ if why == "float32":
+ config = _gemma(dtype = torch.float32)
+ model = types.SimpleNamespace(config = config)
+ elif why == "float32_embeddings":
+ embedding = torch.nn.Embedding(4, 4)
+ model.get_input_embeddings = lambda: embedding
+ elif why == "dtype_unknown":
+ config = _gemma(dtype = None)
+ model = types.SimpleNamespace(config = config)
+ elif why == "deterministic":
+ request.getfixturevalue("deterministic")
+ elif why == "flash_sdpa_disabled":
+ monkeypatch.setattr(hpa, "_SDPA_MATH", [-1, lambda *args: (False, 0)])
+ elif why == "math_first_priority":
+ monkeypatch.setattr(hpa, "_SDPA_MATH", [-1, lambda *args: (True, -1)])
+ elif why == "softcap_config":
+ config.attn_logit_softcapping = 50.0
+ elif why == "sinks_module":
+ attention = types.SimpleNamespace(sinks = torch.zeros(2))
+ model.modules = lambda: iter([attention])
+ elif why == "no_sdpa_model_type":
+ monkeypatch.setattr(u, "_NO_SDPA_MODEL_TYPES", {"gemma"})
+ assert not u.use_sdpa_for_packed_head_dim_flex(model)
+ assert config._attn_implementation == "flex_attention"
+
+
+def test_explicit_flex_request_stays_flex(routed):
+ assert u.requests_flex_attention("flex_attention")
+ assert u.requests_flex_attention({"text_config": "flex_attention"})
+ assert not u.requests_flex_attention(None) and not u.requests_flex_attention("sdpa")
+ config = _gemma()
+ model = types.SimpleNamespace(config = config)
+ setattr(model, u._EXPLICIT_FLEX_ATTR, True)
+ assert not u.use_sdpa_for_packed_head_dim_flex(model)
+ assert config._attn_implementation == "flex_attention"
+
+
+def test_explicit_flex_is_per_model_not_per_type(routed):
+ # Two models of one type: the one loaded with an explicit flex request keeps flex, the other switches.
+ explicit, routed_model = (
+ types.SimpleNamespace(config = _gemma()),
+ types.SimpleNamespace(config = _gemma()),
+ )
+ setattr(explicit, u._EXPLICIT_FLEX_ATTR, True)
+ assert u.use_sdpa_for_packed_head_dim_flex(routed_model)
+ assert not u.use_sdpa_for_packed_head_dim_flex(explicit)
+ assert explicit.config._attn_implementation == "flex_attention"
+ assert routed_model.config._attn_implementation == "sdpa"
+
+
[email protected]("compiled", ["torch_compile_arg", "compiled_model"])
+def test_whole_model_compile_keeps_flex(routed, monkeypatch, compiled):
+ # The outer compile inlines the blocks, where the varlen wrapper falls back to dense-mask SDPA.
+ from unsloth.utils import packing
+
+ monkeypatch.setattr(hpa, "enable_hf_packed_attention", lambda: True)
+ config = _gemma()
+ model = types.SimpleNamespace(config = config)
+ args = types.SimpleNamespace()
+ if compiled == "torch_compile_arg":
+ args.torch_compile = True
+ else:
+ model._orig_mod = types.SimpleNamespace(config = config)
+ collator = types.SimpleNamespace(padding_free = True, torch_call = lambda examples: {})
+ trainer = types.SimpleNamespace(data_collator = collator, args = args)
+ packing.enable_padding_free_metadata(model, trainer)
+ assert config._attn_implementation == "flex_attention"
+
+
+ "batch,max_length,switched", [(2, 4096, True), (8, 4096, False), (1, 32768, False)]
+)
+def test_long_flattened_rows_keep_flex(routed, monkeypatch, batch, max_length, switched):
+ # SDPA builds a dense [T, T] mask for the whole flattened row before the varlen wrapper runs.
+ from unsloth.utils import packing
+
+ monkeypatch.setattr(hpa, "enable_hf_packed_attention", lambda: True)
+ config = _gemma()
+ args = types.SimpleNamespace(max_length = max_length, per_device_train_batch_size = batch)
+ collator = types.SimpleNamespace(padding_free = True, torch_call = lambda examples: {})
+ trainer = types.SimpleNamespace(data_collator = collator, args = args)
+ packing.enable_padding_free_metadata(types.SimpleNamespace(config = config), trainer)
+ assert config._attn_implementation == ("sdpa" if switched else "flex_attention")
+
+
+ "eval_dataset,eval_batch,pad_multiple,switched",
+ [(None, 8, None, True), ([1], 8, None, False), ([1], 1, None, True), (None, 8, 8, False)],
+)
+def test_eval_rows_and_aligned_tails_keep_flex(
+ routed, monkeypatch, eval_dataset, eval_batch, pad_multiple, switched
+):
+ # Eval batches flatten into one row too; a padded-to-multiple tail is declined by the varlen wrapper.
+ from unsloth.utils import packing
+
+ monkeypatch.setattr(hpa, "enable_hf_packed_attention", lambda: True)
+ config = _gemma()
+ args = types.SimpleNamespace(
+ max_length = 8192, per_device_train_batch_size = 1, per_device_eval_batch_size = eval_batch
+ )
+ collator = types.SimpleNamespace(
+ padding_free = True, torch_call = lambda examples: {}, pad_to_multiple_of = pad_multiple
+ )
+ trainer = types.SimpleNamespace(data_collator = collator, args = args, eval_dataset = eval_dataset)
+ packing.enable_padding_free_metadata(types.SimpleNamespace(config = config), trainer)
+ assert config._attn_implementation == ("sdpa" if switched else "flex_attention")
+
+
+def test_padding_free_metadata_makes_the_switch(routed, monkeypatch):
+ from unsloth.utils import packing
+
+ monkeypatch.setattr(hpa, "enable_hf_packed_attention", lambda: True)
+ config = _gemma()
+ collator = types.SimpleNamespace(padding_free = True, torch_call = lambda examples: {})
+ trainer = types.SimpleNamespace(data_collator = collator, args = types.SimpleNamespace())
+ packing.enable_padding_free_metadata(types.SimpleNamespace(config = config), trainer)
+ assert config._attn_implementation == "sdpa"
+
+
+def test_a_document_too_long_to_split_under_int32_stays_on_flex(routed):
+ """The wrapper splits many short documents under flash's int32 varlen backward limit, but one
+ document past it alone would go to dense-mask SDPA: (L + 128) * 2 heads * 256 >= 2**31."""
+ over = 2**31 // (2 * 256) - 128
+ config = _gemma()
+ assert not u.use_sdpa_for_packed_head_dim_flex(types.SimpleNamespace(config = config), over)
+ assert config._attn_implementation == "flex_attention"
+ assert u.use_sdpa_for_packed_head_dim_flex(types.SimpleNamespace(config = config), over - 1)
+ assert config._attn_implementation == "sdpa"
+
+
+def test_packing_passes_the_trainer_max_length(routed, monkeypatch):
+ from unsloth.utils import packing
+ monkeypatch.setattr(hpa, "enable_hf_packed_attention", lambda: True)
+ for max_length, impl in ((2**22, "flex_attention"), (8192, "sdpa")):
+ config = _gemma()
+ collator = types.SimpleNamespace(padding_free = True, torch_call = lambda examples: {})
+ trainer = types.SimpleNamespace(
+ data_collator = collator, args = types.SimpleNamespace(max_length = max_length)
+ )
+ packing.enable_padding_free_metadata(types.SimpleNamespace(config = config), trainer)
+ assert config._attn_implementation == impl
+
+
[email protected](not has_real_cuda(), reason = "needs a CUDA device")
+def test_switched_model_runs_packed_rows_on_varlen_like_flex(monkeypatch):
+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:
+ pytest.skip("this torch has no window_size varlen_attn")
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+
+ monkeypatch.setitem(ALL_ATTENTION_FUNCTIONS, "sdpa", ALL_ATTENTION_FUNCTIONS["sdpa"])
+ monkeypatch.setattr(hpa, "_ORIG_SDPA", [None])
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ assert hpa.enable_hf_packed_attention()
+ monkeypatch.setattr(u, "_flex_kernels_fit_large_head_dim", lambda: True)
+ monkeypatch.setattr(u, "_sdpa_reaches_cudnn_at_head_dim_256", lambda: False)
+ for name in ("UNSLOTH_PACKED_VARLEN_OVER_FLEX", "UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM"):
+ monkeypatch.delenv(name, raising = False)
+ torch.manual_seed(0)
+ model = transformers.GemmaForCausalLM(_gemma()).cuda().to(torch.bfloat16)
+ model.config._attn_implementation = "flex_attention"
+ lengths = [5, 70, 53]
+ ids = torch.randint(0, 128, (1, sum(lengths)), device = "cuda")
+ position_ids = torch.cat([torch.arange(n) for n in lengths])[None].cuda()
+ psl = torch.tensor(lengths, dtype = torch.int32)
+
+ def logits():
+ with torch.no_grad():
+ return model(
+ input_ids = ids, position_ids = position_ids, packed_seq_lengths = psl, use_cache = False
+ ).logits.float()
+
+ flex = logits()
+ assert u.use_sdpa_for_packed_head_dim_flex(model)
+ before = hpa.HF_PACKED_ATTENTION_STATS["fast"]
+ varlen = logits()
+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before + 2
+ torch.testing.assert_close(varlen, flex, atol = 3e-2, rtol = 3e-2)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 4cbb2e22bc7..d99b7439f5d 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -731,6 +731,140 @@ def _prefers_flex_for_head_dim(config):
return not _sdpa_reaches_cudnn_at_head_dim_256()
+_PACKED_VARLEN_OVER_FLEX_ENV_VAR = "UNSLOTH_PACKED_VARLEN_OVER_FLEX"
+# Set on a loaded model whose loader call asked for flex_attention: packing keeps it on flex.
+_EXPLICIT_FLEX_ATTR = "_unsloth_explicit_flex_attention"
+
+
+def requests_flex_attention(requested_attn_implementation):
+ requested = requested_attn_implementation
+ return requested == "flex_attention" or (
+ isinstance(requested, dict) and "flex_attention" in requested.values()
+ )
+
+
+def _packed_attention_dtype(model, config):
+ """The dtype attention runs in without autocast: the input embeddings, else the config's."""
+ try:
+ dtype = model.get_input_embeddings().weight.dtype
+ except Exception:
+ dtype = None
+ if isinstance(dtype, torch.dtype):
+ return dtype
+ for attention_config in [config] + _text_attention_configs(config):
+ for name in ("dtype", "torch_dtype"):
+ dtype = _config_get(attention_config, name, None)
+ if isinstance(dtype, str):
+ dtype = getattr(torch, dtype.removeprefix("torch."), None)
+ if isinstance(dtype, torch.dtype):
+ return dtype
+ return None
+
+
+def _packed_varlen_takes_model(model, config, target, head_dim, device_count):
+ """The packed varlen wrapper's call-independent checks, at setup: fp16 / bf16, no softcap or
+ sinks, an SDPA-capable class, and no policy asking for SDPA itself (deterministic algorithms, a
+ disabled flash backend, a math-first priority). Positional-arg calls are only known per call."""
+ from ..utils.hf_packed_attention import _sdpa_policy_requested, _softcapped
+
+ dtype = _packed_attention_dtype(model, config)
+ if dtype not in (torch.float16, torch.bfloat16):
+ return False
+ if not _maskless_causal_sdpa_accepts(config):
+ return False
+ base = model.get_base_model() if hasattr(model, "get_base_model") else model
+ if _declares_no_sdpa(type(base)) or _native_class_refuses_sdpa(type(base)):
+ return False
+ modules = getattr(model, "modules", None)
+ if callable(modules) and any(_softcapped(module, {}) for module in modules()):
+ return False
+ n_heads = _config_get(target, "num_attention_heads", None)
+ n_kv_heads = _config_get(target, "num_key_value_heads", None) or n_heads
+ if not isinstance(n_heads, int) or not isinstance(n_kv_heads, int):
+ return False
+ return not any(
+ _sdpa_policy_requested(torch.device("cuda", index), dtype, head_dim, n_heads, n_kv_heads)
+ for index in range(device_count)
+ )
+
+
+# Longest flattened packed row moved to SDPA: its dense bool mask is T^2 bytes (16384 tokens: 256 MiB).
+_PACKED_SDPA_MAX_ROW_TOKENS = 16384
+
+
+def use_sdpa_for_packed_head_dim_flex(
+ model,
+ max_length = None,
+ row_tokens = None,
+):
+ """Packed / padding-free rows: a decoder on flex only for its head dim (<= 256) moves to "sdpa",
+ whose packed varlen wrapper runs them through torch's varlen flash kernel exactly (B200 head_dim
+ 256: 6-10% faster at a fixed packed length, 15-24% with varying lengths, and no flex compile).
+ True when switched. Not the gemma3 family, not head_dim 512, not a forced
+ UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM, not an explicit flex_attention request, only where
+ torch's varlen kernel runs and only when the wrapper would take the rows (else dense-mask SDPA).
+ max_length: the longest document; one past flash's int32 varlen backward limit stays on flex.
+ row_tokens: tokens in one flattened packed row; SDPA builds a dense [T, T] mask before the varlen
+ wrapper runs (flex keeps a block mask), so rows past _PACKED_SDPA_MAX_ROW_TOKENS stay on flex."""
+ if os.environ.get(_PACKED_VARLEN_OVER_FLEX_ENV_VAR, "1").strip().lower() in (
+ "0",
+ "false",
+ "no",
+ "off",
+ ):
+ return False
+ if _flex_large_head_dim_override() is not None:
+ return False
+ config = getattr(model, "config", None)
+ if config is None or _config_prefers_flex_attention(config):
+ return False
+ base = model.get_base_model() if hasattr(model, "get_base_model") else model
+ if getattr(base, _EXPLICIT_FLEX_ATTR, False) or getattr(model, _EXPLICIT_FLEX_ATTR, False):
+ return False
+ head_dim = _text_attention_head_dim(config)
+ if head_dim is None or head_dim > _FLEX_KERNEL_OPTIONS_SAFE_HEAD_DIM:
+ return False
+ if not _prefers_flex_for_head_dim(config):
+ return False
+ text_config = _text_sub_config(config)
+ target = config if text_config is None else text_config
+ if _config_get(target, "_attn_implementation", None) != "flex_attention":
+ return False
+ try:
+ from ..utils.hf_packed_attention import _ORIG_SDPA, _torch_varlen
+
+ if _ORIG_SDPA[0] is None or not torch.cuda.is_available():
+ return False
+ device_count = torch.cuda.device_count()
+ if any(_torch_varlen(torch.device("cuda", index)) is None for index in range(device_count)):
+ return False
+ if not _packed_varlen_takes_model(model, config, target, head_dim, device_count):
+ return False
+ from ..utils.attention_dispatch import (
+ _VARLEN_INT32_GUARD_DISABLED,
+ _varlen_backward_overflows_int32,
+ )
+
+ if isinstance(row_tokens, int) and row_tokens > _PACKED_SDPA_MAX_ROW_TOKENS:
+ return False
+ # The wrapper splits many documents under the limit, but not one document alone.
+ n_heads = _config_get(target, "num_attention_heads", None)
+ if (
+ not _VARLEN_INT32_GUARD_DISABLED
+ and isinstance(max_length, int)
+ and _varlen_backward_overflows_int32(1, max_length, n_heads, head_dim)
+ ):
+ return False
+ except Exception:
+ return False
+ _write_attn_impl(target, "sdpa")
+ logging.getLogger(__name__).info(
+ f"Unsloth: packed rows at head_dim {head_dim} use varlen flash attention via sdpa "
+ f"instead of flex_attention ({_PACKED_VARLEN_OVER_FLEX_ENV_VAR}=0 keeps flex)."
+ )
+ return True
+
+
# Above head_dim 256 the default flex template faults with misaligned address on torch 2.13 and 2.14; 32x32 blocks pass
_FLEX_KERNEL_OPTIONS_SAFE_HEAD_DIM = 256
# BLOCK_M/N drive the forward template, BLOCK_M1/N1/M2/N2 the two backward passes.
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 1cb6f3501db..1b50b868606 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -255,6 +255,8 @@ from ._utils import (
attention_class_for_load,
_REMOTE_CLASS_HUB_OPTIONS,
resolve_attention_implementation,
+ requests_flex_attention,
+ _EXPLICIT_FLEX_ATTR,
_get_text_only_config,
_is_family_text_decoder,
_config_get,
@@ -3755,6 +3757,7 @@ class FastBaseModel:
supports_sdpa = _attn_supports_sdpa,
dtype = attn_dtype,
)
+ _explicit_flex = requests_flex_attention(kwargs.get("attn_implementation", None))
# FP8 models: get_model_name already redirected to the BF16 sibling if the model ships FP8 weights, so just update it here for sanity.
auto_config.model_name = model_name
@@ -4902,6 +4905,8 @@ class FastBaseModel:
_mark_loaded_revision(tokenizer, _tokenizer_revision)
model = _mark_forced_float32(model, do_forced_float32)
model = _mark_full_finetuning(model, full_finetuning)
+ if _explicit_flex:
+ setattr(model, _EXPLICIT_FLEX_ATTR, True)
# Last, so the host copies carry the fp32 recasts the passes above make.
finish_block_swap_load(
model, _block_swap_state, planned_prefetch_depth(device_map_planner_kwargs)
diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.py
index 8cd43e11416..e2ed7e6ec7a 100644
--- a/unsloth/utils/hf_packed_attention.py
+++ b/unsloth/utils/hf_packed_attention.py
@@ -18,15 +18,19 @@ _ENV = "UNSLOTH_HF_PACKED_VARLEN"
_ORIG_SDPA = [None]
# True inside _block_compile's own compiled blocks (not a user's outer compile); see _leaves_block.
IN_COMPILED_BLOCK = [False]
-# efficient: the subset of fast calls that ran SDPA's memory-efficient kernel over cu_seqlens.
+# efficient: the subset of fast calls that ran SDPA's memory-efficient kernel over cu_seqlens;
+# int32_split: fast calls run in document groups to keep flash's varlen backward under int32.
HF_PACKED_ATTENTION_STATS = {
"fast": 0,
"fallback": 0,
"mask_checks": 0,
"efficient": 0,
"compiled": 0,
+ "int32_split": 0,
}
BLOCK_VARLEN_KWARG = "_unsloth_block_varlen"
+# BLOCK_VARLEN_KWARG value for rows the eager wrapper must split under the int32 limit.
+_LEAVE_BLOCK = "leave"
# Layers of one window share a mask object: window -> [weakref, (cu_seqlens, versions), verdict].
_MASK_CHECK = {}
_MASK_CHECK_ROWS = 1024
@@ -415,6 +419,39 @@ def _flash_varlen(query, key, value, cu_seqlens, max_seqlen, scale, window):
return out.unsqueeze(0)
+def _int32_safe_groups(lengths, n_heads, head_dim):
+ """Consecutive documents in groups whose varlen backward stays under the int32 limit, as
+ (first, end, start token, stop token, longest); None when one document alone overflows."""
+ from .attention_dispatch import _varlen_backward_overflows_int32
+
+ groups, first, start, tokens, longest = [], 0, 0, 0, 0
+ for index, length in enumerate(lengths):
+ if index > first and _varlen_backward_overflows_int32(
+ index - first + 1, tokens + length, n_heads, head_dim
+ ):
+ groups.append((first, index, start, start + tokens, longest))
+ first, start, tokens, longest = index, start + tokens, 0, 0
+ if index == first and _varlen_backward_overflows_int32(1, length, n_heads, head_dim):
+ return None
+ tokens += length
+ longest = max(longest, length)
+ groups.append((first, len(lengths), start, start + tokens, longest))
+ return groups
+
+
+def _run_document_groups(run, query, key, value, seq_info, groups):
+ """Documents attend only inside themselves: each group runs on its own rows and rebased
+ cu_seqlens, and the rows are joined back (autograd splits the backward the same way)."""
+ lengths, cu_seqlens, _ = seq_info
+ outs = []
+ for first, end, start, stop, longest in groups:
+ rows = slice(start, stop)
+ info = (lengths[first:end], cu_seqlens[first : end + 1] - start, longest)
+ outs.append(run(query[:, :, rows], key[:, :, rows], value[:, :, rows], info))
+ HF_PACKED_ATTENTION_STATS["int32_split"] += 1
+ return torch.cat(outs, dim = 1)
+
+
def _sdpa_packed_varlen(
module,
query,
@@ -437,6 +474,20 @@ def _sdpa_packed_varlen(
HF_PACKED_ATTENTION_STATS["fallback"] += 1
return orig(module, query, key, value, attention_mask, *args, **named, **kwargs)
block_varlen = kwargs.pop(BLOCK_VARLEN_KWARG, None)
+ if isinstance(block_varlen, str):
+ if torch.compiler.is_compiling() and not _disabled():
+ return _sdpa_packed_varlen_eager(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ dropout = dropout,
+ scaling = scaling,
+ is_causal = is_causal,
+ **kwargs,
+ )
+ block_varlen = None
# Fallback gets the uncast Q / K / V so GQA's dK / dV reduce in fp32 after repeat_kv.
given = (query, key, value)
if kwargs.get("packed_seq_lengths") is not None and not _disabled():
@@ -503,7 +554,7 @@ def _sdpa_packed_varlen(
from .packing import get_packed_info_from_kwargs
bsz, n_heads, q_len, head_dim = query.shape
- seq_info = torch_varlen = backend = None
+ seq_info = torch_varlen = backend = groups = None
efficient = False
window = _static_window(module, query, key, value, dropout, is_causal, kwargs)
if window is not _NO_WINDOW:
@@ -530,7 +581,17 @@ def _sdpa_packed_varlen(
kwargs["packed_seq_lengths"].numel(), q_len, n_heads, head_dim
)
):
- torch_varlen, backend, refused = None, None, False
+ from .packing import _host_segment_lengths
+
+ # Documents are independent: groups of them stay under the limit, exactly.
+ refused, groups = (
+ False,
+ _int32_safe_groups(
+ _host_segment_lengths(kwargs["packed_seq_lengths"]), n_heads, head_dim
+ ),
+ )
+ if groups is None:
+ torch_varlen, backend = None, None
# Refused shape and no flash-attn / xformers: the efficient kernel.
if refused and backend not in (FLASH_VARLEN, XFORMERS):
efficient = _efficient_varlen_takes(query, window)
@@ -580,9 +641,21 @@ def _sdpa_packed_varlen(
return out, None
if torch_varlen is not None:
try:
- out = _call_torch_varlen(
- torch_varlen, query, key, value, cu_seqlens, max_seqlen, scale, window
- )
+ if groups is None:
+ out = _call_torch_varlen(
+ torch_varlen, query, key, value, cu_seqlens, max_seqlen, scale, window
+ )
+ else:
+ out = _run_document_groups(
+ lambda q, k, v, info: _call_torch_varlen(
+ torch_varlen, q, k, v, info[1], info[2], scale, window
+ ),
+ query,
+ key,
+ value,
+ seq_info,
+ groups,
+ )
except torch.OutOfMemoryError:
raise
except RuntimeError as exc:
@@ -619,21 +692,28 @@ def _sdpa_packed_varlen(
sdpa_kwargs = {"scale": scale},
xformers_kwargs = {"scale": scale},
)
- context = AttentionContext(
- bsz = 1,
- q_len = q_len,
- kv_seq_len = q_len,
- n_heads = n_heads,
- head_dim = head_dim,
- # Not grad mode: xformers needs a backward op for grad tensors even under no_grad (reentrant
- # checkpointing), and that op rejects the grouped 5D layout.
- requires_grad = query.requires_grad or key.requires_grad or value.requires_grad,
- seq_info = seq_info,
- attention_mask = None,
- causal_mask = None,
- sliding_window = window,
- )
- out = run_attention(config = config, context = context, Q = query, K = key, V = value)
+
+ def dispatch(query, key, value, seq_info):
+ context = AttentionContext(
+ bsz = 1,
+ q_len = query.shape[2],
+ kv_seq_len = query.shape[2],
+ n_heads = n_heads,
+ head_dim = head_dim,
+ # Not grad mode: xformers needs a backward op for grad tensors even under no_grad
+ # (reentrant checkpointing), and that op rejects the grouped 5D layout.
+ requires_grad = query.requires_grad or key.requires_grad or value.requires_grad,
+ seq_info = seq_info,
+ attention_mask = None,
+ causal_mask = None,
+ sliding_window = window,
+ )
+ return run_attention(config = config, context = context, Q = query, K = key, V = value)
+
+ if groups is None:
+ out = dispatch(query, key, value, seq_info)
+ else:
+ out = _run_document_groups(dispatch, query, key, value, seq_info, groups)
return out.contiguous(), None
@@ -741,7 +821,8 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):
"""Eager, before _block_compile runs a compiled decoder block: (cu_seqlens, max_seqlen) for its
packed row, so attention runs torch's varlen op inside the graph. Building them there needs a host
read of the longest segment and the mask check a host sync, which is why the traced wrapper used to
- fall back to SDPA over the dense mask. None keeps that path."""
+ fall back to SDPA over the dense mask. None keeps that path; _LEAVE_BLOCK runs the eager
+ wrapper."""
seq_lengths, mask = kwargs.get("packed_seq_lengths"), kwargs.get("attention_mask")
attention = getattr(layer, "self_attn", None)
# Layers of one forward (and its checkpoint recompute) share the lengths and their window's mask.
@@ -760,7 +841,7 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):
# sdpa_kernel / deterministic mode can change between calls on the same tensors.
and (entry[5] is None or _sdpa_policy_requested(*entry[5]) == entry[6])
):
- if entry[3] is not None:
+ if isinstance(entry[3], tuple):
HF_PACKED_ATTENTION_STATS["compiled"] += 1
return entry[3]
policy = []
@@ -833,7 +914,12 @@ def _block_varlen_prepare(
if not _VARLEN_INT32_GUARD_DISABLED and _varlen_backward_overflows_int32(
kwargs["packed_seq_lengths"].numel(), total, n_heads, head_dim
):
- return None
+ from .packing import _host_segment_lengths
+
+ # Splitting at document boundaries in the graph would need per-step shapes: the eager wrapper
+ # splits instead (a graph break for these rows only). One document alone keeps the dense path.
+ lengths = _host_segment_lengths(kwargs["packed_seq_lengths"])
+ return None if _int32_safe_groups(lengths, n_heads, head_dim) is None else _LEAVE_BLOCK
seq_info = get_packed_info_from_kwargs(kwargs, hidden_states.device)
if seq_info is None or not _mask_is_block_causal(
kwargs.get("attention_mask"), seq_info[1], total, window
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index ae83c2dd991..004e3b311cc 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -201,11 +201,56 @@ def configure_padding_free(config):
setattr(config, "remove_unused_columns", False)
-def _enable_hf_packed_attention() -> None:
+def _longest_document(model, trainer) -> Optional[int]:
+ args = getattr(trainer, "args", None)
+ for value in (
+ getattr(args, "max_length", None),
+ getattr(args, "max_seq_length", None),
+ getattr(model, "max_seq_length", None),
+ ):
+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:
+ return value
+ return None
+
+
+def _packed_row_tokens(model, trainer) -> Optional[int]:
+ """Tokens in one flattened row: the padding-free collator joins a whole device batch, the eval
+ batch too when there is an eval dataset (transformers' default eval batch is 8)."""
+ longest = _longest_document(model, trainer)
+ if longest is None:
+ return None
+ args = getattr(trainer, "args", None)
+ names = ["per_device_train_batch_size"]
+ if getattr(trainer, "eval_dataset", None) is not None:
+ names.append("per_device_eval_batch_size")
+ batches = [getattr(args, name, None) for name in names]
+ batches = [b for b in batches if isinstance(b, int) and not isinstance(b, bool) and b > 0]
+ return longest * max(batches, default = 1)
+
+
+def _enable_hf_packed_attention(model = None, trainer = None) -> None:
# packed rows on the transformers modeling path run varlen instead of SDPA over a dense mask.
try:
from .hf_packed_attention import enable_hf_packed_attention
- enable_hf_packed_attention()
+
+ # A whole-model torch.compile (torch_compile=True, or an already compiled model) inlines the
+ # blocks, where the varlen wrapper falls back to dense-mask SDPA: those keep flex.
+ outer_compile = bool(getattr(getattr(trainer, "args", None), "torch_compile", False)) or (
+ hasattr(model, "_orig_mod")
+ )
+ # A collator padding the flattened row to a multiple leaves a tail the varlen wrapper declines.
+ pad_multiple = getattr(getattr(trainer, "data_collator", None), "pad_to_multiple_of", None)
+ aligned_tail = isinstance(pad_multiple, int) and pad_multiple > 1
+ if (
+ enable_hf_packed_attention()
+ and model is not None
+ and not outer_compile
+ and not aligned_tail
+ ):
+ from ..models._utils import use_sdpa_for_packed_head_dim_flex
+ use_sdpa_for_packed_head_dim_flex(
+ model, _longest_document(model, trainer), _packed_row_tokens(model, trainer)
+ )
except Exception as exc:
logging.getLogger(__name__).info(f"Unsloth: packed varlen attention not enabled ({exc})")
@@ -247,7 +292,7 @@ def enable_sample_packing(
collator = getattr(trainer, "data_collator", None)
if collator is None or not hasattr(collator, "torch_call"):
return
- _enable_hf_packed_attention()
+ _enable_hf_packed_attention(model, trainer)
_scope_flash_attention_impl(model, trainer, sequence_lengths_key)
if getattr(collator, "_unsloth_packing_wrapped", False):
return
@@ -294,7 +339,7 @@ def enable_padding_free_metadata(model, trainer):
return
mark_allow_overlength(model)
- _enable_hf_packed_attention()
+ _enable_hf_packed_attention(model, trainer)
if hasattr(collator, "return_position_ids"):
collator.return_position_ids = True
if hasattr(trainer, "args") and hasattr(trainer.args, "remove_unused_columns"):