Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Run packed rows of head-dim-routed flex decoders through varlen SDPA (#15514)
Daniel Han92ecb5219160
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.pyindex 1eb50bfe3ff..4e6f6e3550e 100644--- a/tests/test_hf_packed_attention.py+++ b/tests/test_hf_packed_attention.py@@ -662,6 +662,141 @@ def test_int32_guard_follows_training_or_grad_mode(monkeypatch, mode):assert out == ("orig", None)+# (L + 128 * n) * H * round32(D) at H = 8, D = 64: 400 tokens' worth per group.+_SMALL_INT32_LIMIT = 400 * 8 * 64+++def test_int32_safe_groups_split_at_document_boundaries(monkeypatch):+ import unsloth.utils.attention_dispatch as ad++ monkeypatch.setattr(ad, "_INT32_ELEMENTS", _SMALL_INT32_LIMIT)+ lengths = [37, 100, 2, 5, 230, 64, 9]+ groups = hpa._int32_safe_groups(lengths, 8, 64)+ assert groups[0][0] == 0 and groups[-1][1] == len(lengths) and groups[-1][3] == sum(lengths)+ for (first, end, start, stop, longest), following in zip(groups, groups[1:] + [None]):+ assert stop - start == sum(lengths[first:end]) and longest == max(lengths[first:end])+ assert not ad._varlen_backward_overflows_int32(end - first, stop - start, 8, 64)+ if following is not None:+ assert following[0] == end and following[2] == stop+ assert len(groups) > 1+ # One document alone past the limit cannot be split.+ assert hpa._int32_safe_groups([37, 300], 8, 64) is None+++@_CUDA[email protected]("window", [None, 4])[email protected]("route", ["torch", "dispatch"])+def test_int32_guard_splits_documents_instead_of_the_dense_fallback(monkeypatch, route, window):+ """A row past the varlen backward's int32 limit runs the kernel per group of documents, and the+ output and gradients are the unsplit kernel's; SDPA over the dense mask is never reached."""+ import unsloth.utils.attention_dispatch as ad++ if route == "torch" and hpa._torch_varlen(torch.device("cuda", 0)) is None:+ pytest.skip("this torch has no window_size varlen_attn")+ if route == "dispatch":+ if not (ad.HAS_FLASH_ATTENTION or ad.HAS_XFORMERS):+ pytest.skip("needs xformers or flash-attn")+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)+ # Where torch's packed kernels beat xformers the dispatcher picks sdpa: pin the varlen one.+ backend = ad.FLASH_VARLEN if ad.HAS_FLASH_ATTENTION else ad.XFORMERS+ monkeypatch.setattr(ad, "select_attention_backend", lambda **kwargs: backend)++ def _dense(*args, **kwargs):+ raise AssertionError("packed row fell back to SDPA over the dense mask")++ hpa._ORIG_SDPA[0] = _dense+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ torch.manual_seed(0)+ lengths = [37, 100, 2, 5, 230, 64, 9]+ T, H, n_kv, D = sum(lengths), 8, 2, 64+ module = types.SimpleNamespace(+ is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = window+ )+ q, k, v = (+ torch.randn(1, h, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ for h in (H, n_kv, n_kv)+ )+ mask = (+ _block_causal(lengths, "cuda")+ if window is None+ else _hf_window_mask(lengths, window, "cuda")+ )+ psl = torch.tensor(lengths, dtype = torch.int32)+ g = torch.randn(1, T, H, D, device = "cuda", dtype = torch.bfloat16)++ def run():+ out, _ = hpa._sdpa_packed_varlen(+ module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl+ )+ return (out, *torch.autograd.grad(out, (q, k, v), g))++ whole = run()+ monkeypatch.setattr(ad, "_INT32_ELEMENTS", _SMALL_INT32_LIMIT)+ assert ad._varlen_backward_overflows_int32(len(lengths), T, H, D)+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ split = run()+ assert hpa.HF_PACKED_ATTENTION_STATS["int32_split"] == before["int32_split"] + 1+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"]+ for a, b in zip(split, whole):+ torch.testing.assert_close(a, b, atol = 0, rtol = 0)+++def test_block_varlen_kwargs_sends_rows_past_int32_to_the_eager_wrapper(monkeypatch):+ """The traced kernel cannot split a row, so the eager wrapper (which can) runs it; a document that+ alone overflows keeps the previous dense path."""+ import unsloth.utils.attention_dispatch as ad++ hpa._ORIG_SDPA[0] = _plain_sdpa+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())+ monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: False)+ monkeypatch.setattr(hpa, "_BLOCK_PREP", {})+ monkeypatch.setattr(ad, "_INT32_ELEMENTS", 200 * 4 * 32)+ layer = _block_layer(head_dim = 32, n_heads = 4)+ for lengths, expected in (([20, 30, 40], hpa._LEAVE_BLOCK), ([20, 80], None)):+ hidden = torch.randn(1, sum(lengths), 8)+ kwargs = {+ "packed_seq_lengths": torch.tensor(lengths),+ "attention_mask": _block_causal(lengths),+ }+ before = hpa.HF_PACKED_ATTENTION_STATS["compiled"]+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == expected+ assert hpa.HF_PACKED_ATTENTION_STATS["compiled"] == before+++def test_traced_leave_block_call_runs_the_eager_wrapper(monkeypatch):+ seen = []++ @torch.compiler.disable+ def eager(module, q, k, v, mask, **kwargs):+ seen.append(set(kwargs))+ return q.transpose(1, 2) * 3, None++ hpa._ORIG_SDPA[0] = lambda *args, **kwargs: pytest.fail("the dense path stayed in the graph")+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ monkeypatch.setattr(hpa, "_sdpa_packed_varlen_eager", eager)+ lengths = [3, 3]+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = True)+ q = torch.randn(1, 4, 6, 16, dtype = torch.bfloat16)+ mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)+ torch._dynamo.reset()+ fn = torch.compile(+ lambda q: hpa._sdpa_packed_varlen(+ module,+ q,+ q,+ q,+ mask,+ packed_seq_lengths = psl,+ **{hpa.BLOCK_VARLEN_KWARG: hpa._LEAVE_BLOCK},+ )[0],+ backend = "eager",+ )+ torch.testing.assert_close(fn(q), q.transpose(1, 2) * 3)+ assert seen and all(hpa.BLOCK_VARLEN_KWARG not in kwargs for kwargs in seen)++@_CUDA@_TORCH_VARLEN_HOSTdef test_a_refusing_torch_varlen_hands_the_call_back(monkeypatch):diff --git a/tests/test_packed_varlen_over_flex.py b/tests/test_packed_varlen_over_flex.pynew file mode 100644index 00000000000..625d098e5b5--- /dev/null+++ b/tests/test_packed_varlen_over_flex.py@@ -0,0 +1,309 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""Packed / padding-free training moves a decoder routed to flex only for its head dim onto "sdpa",+where the packed varlen wrapper runs; every other routing is left alone."""++import types++import pytest+import torch+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py++import unsloth # noqa: F401 (must precede transformers)+import unsloth.models._utils as u+from unsloth.utils import hf_packed_attention as hpa++transformers = pytest.importorskip("transformers")+++def _gemma(+ head_dim = 256,+ impl = "flex_attention",+ model_type = None,+ dtype = torch.bfloat16,+):+ config = transformers.GemmaConfig(+ vocab_size = 128,+ hidden_size = 256,+ intermediate_size = 256,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = head_dim,+ )+ config._attn_implementation = impl+ config.dtype = dtype+ if model_type is not None:+ config.model_type = model_type+ return config+++def routed(monkeypatch):+ """A box where head_dim 256 routes to flex and torch's varlen kernel runs."""+ for name in ("UNSLOTH_PACKED_VARLEN_OVER_FLEX", "UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM"):+ monkeypatch.delenv(name, raising = False)+ monkeypatch.setattr(u, "_flex_kernels_fit_large_head_dim", lambda: True)+ monkeypatch.setattr(u, "_sdpa_reaches_cudnn_at_head_dim_256", lambda: False)+ monkeypatch.setattr(hpa, "_ORIG_SDPA", [lambda *args, **kwargs: None])+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ # SDPA backend state for these shapes: flash enabled, flash picked (not math).+ monkeypatch.setattr(hpa, "_SDPA_MATH", [-1, lambda *args: (True, 0)])+++def test_head_dim_flex_moves_to_sdpa(routed):+ config = _gemma()+ assert u.use_sdpa_for_packed_head_dim_flex(types.SimpleNamespace(config = config))+ assert config._attn_implementation == "sdpa"+++ "why",+ [+ "kill_switch",+ "flex_forced",+ "flex_refused",+ "gemma3_family",+ "head_dim_512",+ "head_dim_128",+ "already_sdpa",+ "no_torch_varlen",+ "one_device_without_varlen",+ "wrapper_not_installed",+ "cudnn_takes_head_dim_256",+ ],+)+def test_every_other_routing_is_left_alone(routed, monkeypatch, why):+ config = _gemma()+ if why == "kill_switch":+ monkeypatch.setenv("UNSLOTH_PACKED_VARLEN_OVER_FLEX", "0")+ elif why == "flex_forced":+ monkeypatch.setenv("UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM", "1")+ elif why == "flex_refused":+ monkeypatch.setenv("UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM", "0")+ elif why == "gemma3_family":+ config = _gemma(model_type = "gemma3_text")+ elif why == "head_dim_512":+ config = _gemma(head_dim = 512)+ elif why == "head_dim_128":+ config = _gemma(head_dim = 128)+ elif why == "already_sdpa":+ config = _gemma(impl = "sdpa")+ elif why == "no_torch_varlen":+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)+ elif why == "one_device_without_varlen":+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None if device.index else object())+ elif why == "wrapper_not_installed":+ monkeypatch.setattr(hpa, "_ORIG_SDPA", [None])+ elif why == "cudnn_takes_head_dim_256":+ monkeypatch.setattr(u, "_sdpa_reaches_cudnn_at_head_dim_256", lambda: True)+ before = config._attn_implementation+ assert not u.use_sdpa_for_packed_head_dim_flex(types.SimpleNamespace(config = config))+ assert config._attn_implementation == before+++def deterministic():+ previous = torch.are_deterministic_algorithms_enabled()+ torch.use_deterministic_algorithms(True, warn_only = True)+ yield+ torch.use_deterministic_algorithms(previous)+++ "why",+ [+ "float32",+ "float32_embeddings",+ "dtype_unknown",+ "deterministic",+ "flash_sdpa_disabled",+ "math_first_priority",+ "softcap_config",+ "sinks_module",+ "no_sdpa_model_type",+ ],+)+def test_rows_the_varlen_wrapper_declines_stay_on_flex(routed, monkeypatch, request, why):+ """The wrapper would send these to dense-mask SDPA (slower than flex, ~2x memory at head_dim 256)."""+ config = _gemma()+ model = types.SimpleNamespace(config = config)+ if why == "float32":+ config = _gemma(dtype = torch.float32)+ model = types.SimpleNamespace(config = config)+ elif why == "float32_embeddings":+ embedding = torch.nn.Embedding(4, 4)+ model.get_input_embeddings = lambda: embedding+ elif why == "dtype_unknown":+ config = _gemma(dtype = None)+ model = types.SimpleNamespace(config = config)+ elif why == "deterministic":+ request.getfixturevalue("deterministic")+ elif why == "flash_sdpa_disabled":+ monkeypatch.setattr(hpa, "_SDPA_MATH", [-1, lambda *args: (False, 0)])+ elif why == "math_first_priority":+ monkeypatch.setattr(hpa, "_SDPA_MATH", [-1, lambda *args: (True, -1)])+ elif why == "softcap_config":+ config.attn_logit_softcapping = 50.0+ elif why == "sinks_module":+ attention = types.SimpleNamespace(sinks = torch.zeros(2))+ model.modules = lambda: iter([attention])+ elif why == "no_sdpa_model_type":+ monkeypatch.setattr(u, "_NO_SDPA_MODEL_TYPES", {"gemma"})+ assert not u.use_sdpa_for_packed_head_dim_flex(model)+ assert config._attn_implementation == "flex_attention"+++def test_explicit_flex_request_stays_flex(routed):+ assert u.requests_flex_attention("flex_attention")+ assert u.requests_flex_attention({"text_config": "flex_attention"})+ assert not u.requests_flex_attention(None) and not u.requests_flex_attention("sdpa")+ config = _gemma()+ model = types.SimpleNamespace(config = config)+ setattr(model, u._EXPLICIT_FLEX_ATTR, True)+ assert not u.use_sdpa_for_packed_head_dim_flex(model)+ assert config._attn_implementation == "flex_attention"+++def test_explicit_flex_is_per_model_not_per_type(routed):+ # Two models of one type: the one loaded with an explicit flex request keeps flex, the other switches.+ explicit, routed_model = (+ types.SimpleNamespace(config = _gemma()),+ types.SimpleNamespace(config = _gemma()),+ )+ setattr(explicit, u._EXPLICIT_FLEX_ATTR, True)+ assert u.use_sdpa_for_packed_head_dim_flex(routed_model)+ assert not u.use_sdpa_for_packed_head_dim_flex(explicit)+ assert explicit.config._attn_implementation == "flex_attention"+ assert routed_model.config._attn_implementation == "sdpa"++[email protected]("compiled", ["torch_compile_arg", "compiled_model"])+def test_whole_model_compile_keeps_flex(routed, monkeypatch, compiled):+ # The outer compile inlines the blocks, where the varlen wrapper falls back to dense-mask SDPA.+ from unsloth.utils import packing++ monkeypatch.setattr(hpa, "enable_hf_packed_attention", lambda: True)+ config = _gemma()+ model = types.SimpleNamespace(config = config)+ args = types.SimpleNamespace()+ if compiled == "torch_compile_arg":+ args.torch_compile = True+ else:+ model._orig_mod = types.SimpleNamespace(config = config)+ collator = types.SimpleNamespace(padding_free = True, torch_call = lambda examples: {})+ trainer = types.SimpleNamespace(data_collator = collator, args = args)+ packing.enable_padding_free_metadata(model, trainer)+ assert config._attn_implementation == "flex_attention"+++ "batch,max_length,switched", [(2, 4096, True), (8, 4096, False), (1, 32768, False)]+)+def test_long_flattened_rows_keep_flex(routed, monkeypatch, batch, max_length, switched):+ # SDPA builds a dense [T, T] mask for the whole flattened row before the varlen wrapper runs.+ from unsloth.utils import packing++ monkeypatch.setattr(hpa, "enable_hf_packed_attention", lambda: True)+ config = _gemma()+ args = types.SimpleNamespace(max_length = max_length, per_device_train_batch_size = batch)+ collator = types.SimpleNamespace(padding_free = True, torch_call = lambda examples: {})+ trainer = types.SimpleNamespace(data_collator = collator, args = args)+ packing.enable_padding_free_metadata(types.SimpleNamespace(config = config), trainer)+ assert config._attn_implementation == ("sdpa" if switched else "flex_attention")+++ "eval_dataset,eval_batch,pad_multiple,switched",+ [(None, 8, None, True), ([1], 8, None, False), ([1], 1, None, True), (None, 8, 8, False)],+)+def test_eval_rows_and_aligned_tails_keep_flex(+ routed, monkeypatch, eval_dataset, eval_batch, pad_multiple, switched+):+ # Eval batches flatten into one row too; a padded-to-multiple tail is declined by the varlen wrapper.+ from unsloth.utils import packing++ monkeypatch.setattr(hpa, "enable_hf_packed_attention", lambda: True)+ config = _gemma()+ args = types.SimpleNamespace(+ max_length = 8192, per_device_train_batch_size = 1, per_device_eval_batch_size = eval_batch+ )+ collator = types.SimpleNamespace(+ padding_free = True, torch_call = lambda examples: {}, pad_to_multiple_of = pad_multiple+ )+ trainer = types.SimpleNamespace(data_collator = collator, args = args, eval_dataset = eval_dataset)+ packing.enable_padding_free_metadata(types.SimpleNamespace(config = config), trainer)+ assert config._attn_implementation == ("sdpa" if switched else "flex_attention")+++def test_padding_free_metadata_makes_the_switch(routed, monkeypatch):+ from unsloth.utils import packing++ monkeypatch.setattr(hpa, "enable_hf_packed_attention", lambda: True)+ config = _gemma()+ collator = types.SimpleNamespace(padding_free = True, torch_call = lambda examples: {})+ trainer = types.SimpleNamespace(data_collator = collator, args = types.SimpleNamespace())+ packing.enable_padding_free_metadata(types.SimpleNamespace(config = config), trainer)+ assert config._attn_implementation == "sdpa"+++def test_a_document_too_long_to_split_under_int32_stays_on_flex(routed):+ """The wrapper splits many short documents under flash's int32 varlen backward limit, but one+ document past it alone would go to dense-mask SDPA: (L + 128) * 2 heads * 256 >= 2**31."""+ over = 2**31 // (2 * 256) - 128+ config = _gemma()+ assert not u.use_sdpa_for_packed_head_dim_flex(types.SimpleNamespace(config = config), over)+ assert config._attn_implementation == "flex_attention"+ assert u.use_sdpa_for_packed_head_dim_flex(types.SimpleNamespace(config = config), over - 1)+ assert config._attn_implementation == "sdpa"+++def test_packing_passes_the_trainer_max_length(routed, monkeypatch):+ from unsloth.utils import packing+ monkeypatch.setattr(hpa, "enable_hf_packed_attention", lambda: True)+ for max_length, impl in ((2**22, "flex_attention"), (8192, "sdpa")):+ config = _gemma()+ collator = types.SimpleNamespace(padding_free = True, torch_call = lambda examples: {})+ trainer = types.SimpleNamespace(+ data_collator = collator, args = types.SimpleNamespace(max_length = max_length)+ )+ packing.enable_padding_free_metadata(types.SimpleNamespace(config = config), trainer)+ assert config._attn_implementation == impl++[email protected](not has_real_cuda(), reason = "needs a CUDA device")+def test_switched_model_runs_packed_rows_on_varlen_like_flex(monkeypatch):+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:+ pytest.skip("this torch has no window_size varlen_attn")+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS++ monkeypatch.setitem(ALL_ATTENTION_FUNCTIONS, "sdpa", ALL_ATTENTION_FUNCTIONS["sdpa"])+ monkeypatch.setattr(hpa, "_ORIG_SDPA", [None])+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ assert hpa.enable_hf_packed_attention()+ monkeypatch.setattr(u, "_flex_kernels_fit_large_head_dim", lambda: True)+ monkeypatch.setattr(u, "_sdpa_reaches_cudnn_at_head_dim_256", lambda: False)+ for name in ("UNSLOTH_PACKED_VARLEN_OVER_FLEX", "UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM"):+ monkeypatch.delenv(name, raising = False)+ torch.manual_seed(0)+ model = transformers.GemmaForCausalLM(_gemma()).cuda().to(torch.bfloat16)+ model.config._attn_implementation = "flex_attention"+ lengths = [5, 70, 53]+ ids = torch.randint(0, 128, (1, sum(lengths)), device = "cuda")+ position_ids = torch.cat([torch.arange(n) for n in lengths])[None].cuda()+ psl = torch.tensor(lengths, dtype = torch.int32)++ def logits():+ with torch.no_grad():+ return model(+ input_ids = ids, position_ids = position_ids, packed_seq_lengths = psl, use_cache = False+ ).logits.float()++ flex = logits()+ assert u.use_sdpa_for_packed_head_dim_flex(model)+ before = hpa.HF_PACKED_ATTENTION_STATS["fast"]+ varlen = logits()+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before + 2+ torch.testing.assert_close(varlen, flex, atol = 3e-2, rtol = 3e-2)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 4cbb2e22bc7..d99b7439f5d 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -731,6 +731,140 @@ def _prefers_flex_for_head_dim(config):return not _sdpa_reaches_cudnn_at_head_dim_256()+_PACKED_VARLEN_OVER_FLEX_ENV_VAR = "UNSLOTH_PACKED_VARLEN_OVER_FLEX"+# Set on a loaded model whose loader call asked for flex_attention: packing keeps it on flex.+_EXPLICIT_FLEX_ATTR = "_unsloth_explicit_flex_attention"+++def requests_flex_attention(requested_attn_implementation):+ requested = requested_attn_implementation+ return requested == "flex_attention" or (+ isinstance(requested, dict) and "flex_attention" in requested.values()+ )+++def _packed_attention_dtype(model, config):+ """The dtype attention runs in without autocast: the input embeddings, else the config's."""+ try:+ dtype = model.get_input_embeddings().weight.dtype+ except Exception:+ dtype = None+ if isinstance(dtype, torch.dtype):+ return dtype+ for attention_config in [config] + _text_attention_configs(config):+ for name in ("dtype", "torch_dtype"):+ dtype = _config_get(attention_config, name, None)+ if isinstance(dtype, str):+ dtype = getattr(torch, dtype.removeprefix("torch."), None)+ if isinstance(dtype, torch.dtype):+ return dtype+ return None+++def _packed_varlen_takes_model(model, config, target, head_dim, device_count):+ """The packed varlen wrapper's call-independent checks, at setup: fp16 / bf16, no softcap or+ sinks, an SDPA-capable class, and no policy asking for SDPA itself (deterministic algorithms, a+ disabled flash backend, a math-first priority). Positional-arg calls are only known per call."""+ from ..utils.hf_packed_attention import _sdpa_policy_requested, _softcapped++ dtype = _packed_attention_dtype(model, config)+ if dtype not in (torch.float16, torch.bfloat16):+ return False+ if not _maskless_causal_sdpa_accepts(config):+ return False+ base = model.get_base_model() if hasattr(model, "get_base_model") else model+ if _declares_no_sdpa(type(base)) or _native_class_refuses_sdpa(type(base)):+ return False+ modules = getattr(model, "modules", None)+ if callable(modules) and any(_softcapped(module, {}) for module in modules()):+ return False+ n_heads = _config_get(target, "num_attention_heads", None)+ n_kv_heads = _config_get(target, "num_key_value_heads", None) or n_heads+ if not isinstance(n_heads, int) or not isinstance(n_kv_heads, int):+ return False+ return not any(+ _sdpa_policy_requested(torch.device("cuda", index), dtype, head_dim, n_heads, n_kv_heads)+ for index in range(device_count)+ )+++# Longest flattened packed row moved to SDPA: its dense bool mask is T^2 bytes (16384 tokens: 256 MiB).+_PACKED_SDPA_MAX_ROW_TOKENS = 16384+++def use_sdpa_for_packed_head_dim_flex(+ model,+ max_length = None,+ row_tokens = None,+):+ """Packed / padding-free rows: a decoder on flex only for its head dim (<= 256) moves to "sdpa",+ whose packed varlen wrapper runs them through torch's varlen flash kernel exactly (B200 head_dim+ 256: 6-10% faster at a fixed packed length, 15-24% with varying lengths, and no flex compile).+ True when switched. Not the gemma3 family, not head_dim 512, not a forced+ UNSLOTH_FLEX_ATTENTION_FOR_LARGE_HEAD_DIM, not an explicit flex_attention request, only where+ torch's varlen kernel runs and only when the wrapper would take the rows (else dense-mask SDPA).+ max_length: the longest document; one past flash's int32 varlen backward limit stays on flex.+ row_tokens: tokens in one flattened packed row; SDPA builds a dense [T, T] mask before the varlen+ wrapper runs (flex keeps a block mask), so rows past _PACKED_SDPA_MAX_ROW_TOKENS stay on flex."""+ if os.environ.get(_PACKED_VARLEN_OVER_FLEX_ENV_VAR, "1").strip().lower() in (+ "0",+ "false",+ "no",+ "off",+ ):+ return False+ if _flex_large_head_dim_override() is not None:+ return False+ config = getattr(model, "config", None)+ if config is None or _config_prefers_flex_attention(config):+ return False+ base = model.get_base_model() if hasattr(model, "get_base_model") else model+ if getattr(base, _EXPLICIT_FLEX_ATTR, False) or getattr(model, _EXPLICIT_FLEX_ATTR, False):+ return False+ head_dim = _text_attention_head_dim(config)+ if head_dim is None or head_dim > _FLEX_KERNEL_OPTIONS_SAFE_HEAD_DIM:+ return False+ if not _prefers_flex_for_head_dim(config):+ return False+ text_config = _text_sub_config(config)+ target = config if text_config is None else text_config+ if _config_get(target, "_attn_implementation", None) != "flex_attention":+ return False+ try:+ from ..utils.hf_packed_attention import _ORIG_SDPA, _torch_varlen++ if _ORIG_SDPA[0] is None or not torch.cuda.is_available():+ return False+ device_count = torch.cuda.device_count()+ if any(_torch_varlen(torch.device("cuda", index)) is None for index in range(device_count)):+ return False+ if not _packed_varlen_takes_model(model, config, target, head_dim, device_count):+ return False+ from ..utils.attention_dispatch import (+ _VARLEN_INT32_GUARD_DISABLED,+ _varlen_backward_overflows_int32,+ )++ if isinstance(row_tokens, int) and row_tokens > _PACKED_SDPA_MAX_ROW_TOKENS:+ return False+ # The wrapper splits many documents under the limit, but not one document alone.+ n_heads = _config_get(target, "num_attention_heads", None)+ if (+ not _VARLEN_INT32_GUARD_DISABLED+ and isinstance(max_length, int)+ and _varlen_backward_overflows_int32(1, max_length, n_heads, head_dim)+ ):+ return False+ except Exception:+ return False+ _write_attn_impl(target, "sdpa")+ logging.getLogger(__name__).info(+ f"Unsloth: packed rows at head_dim {head_dim} use varlen flash attention via sdpa "+ f"instead of flex_attention ({_PACKED_VARLEN_OVER_FLEX_ENV_VAR}=0 keeps flex)."+ )+ return True++# Above head_dim 256 the default flex template faults with misaligned address on torch 2.13 and 2.14; 32x32 blocks pass_FLEX_KERNEL_OPTIONS_SAFE_HEAD_DIM = 256# BLOCK_M/N drive the forward template, BLOCK_M1/N1/M2/N2 the two backward passes.diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 1cb6f3501db..1b50b868606 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -255,6 +255,8 @@ from ._utils import (attention_class_for_load,_REMOTE_CLASS_HUB_OPTIONS,resolve_attention_implementation,+ requests_flex_attention,+ _EXPLICIT_FLEX_ATTR,_get_text_only_config,_is_family_text_decoder,_config_get,@@ -3755,6 +3757,7 @@ class FastBaseModel:supports_sdpa = _attn_supports_sdpa,dtype = attn_dtype,)+ _explicit_flex = requests_flex_attention(kwargs.get("attn_implementation", None))# FP8 models: get_model_name already redirected to the BF16 sibling if the model ships FP8 weights, so just update it here for sanity.auto_config.model_name = model_name@@ -4902,6 +4905,8 @@ class FastBaseModel:_mark_loaded_revision(tokenizer, _tokenizer_revision)model = _mark_forced_float32(model, do_forced_float32)model = _mark_full_finetuning(model, full_finetuning)+ if _explicit_flex:+ setattr(model, _EXPLICIT_FLEX_ATTR, True)# Last, so the host copies carry the fp32 recasts the passes above make.finish_block_swap_load(model, _block_swap_state, planned_prefetch_depth(device_map_planner_kwargs)diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.pyindex 8cd43e11416..e2ed7e6ec7a 100644--- a/unsloth/utils/hf_packed_attention.py+++ b/unsloth/utils/hf_packed_attention.py@@ -18,15 +18,19 @@ _ENV = "UNSLOTH_HF_PACKED_VARLEN"_ORIG_SDPA = [None]# True inside _block_compile's own compiled blocks (not a user's outer compile); see _leaves_block.IN_COMPILED_BLOCK = [False]-# efficient: the subset of fast calls that ran SDPA's memory-efficient kernel over cu_seqlens.+# efficient: the subset of fast calls that ran SDPA's memory-efficient kernel over cu_seqlens;+# int32_split: fast calls run in document groups to keep flash's varlen backward under int32.HF_PACKED_ATTENTION_STATS = {"fast": 0,"fallback": 0,"mask_checks": 0,"efficient": 0,"compiled": 0,+ "int32_split": 0,}BLOCK_VARLEN_KWARG = "_unsloth_block_varlen"+# BLOCK_VARLEN_KWARG value for rows the eager wrapper must split under the int32 limit.+_LEAVE_BLOCK = "leave"# Layers of one window share a mask object: window -> [weakref, (cu_seqlens, versions), verdict]._MASK_CHECK = {}_MASK_CHECK_ROWS = 1024@@ -415,6 +419,39 @@ def _flash_varlen(query, key, value, cu_seqlens, max_seqlen, scale, window):return out.unsqueeze(0)+def _int32_safe_groups(lengths, n_heads, head_dim):+ """Consecutive documents in groups whose varlen backward stays under the int32 limit, as+ (first, end, start token, stop token, longest); None when one document alone overflows."""+ from .attention_dispatch import _varlen_backward_overflows_int32++ groups, first, start, tokens, longest = [], 0, 0, 0, 0+ for index, length in enumerate(lengths):+ if index > first and _varlen_backward_overflows_int32(+ index - first + 1, tokens + length, n_heads, head_dim+ ):+ groups.append((first, index, start, start + tokens, longest))+ first, start, tokens, longest = index, start + tokens, 0, 0+ if index == first and _varlen_backward_overflows_int32(1, length, n_heads, head_dim):+ return None+ tokens += length+ longest = max(longest, length)+ groups.append((first, len(lengths), start, start + tokens, longest))+ return groups+++def _run_document_groups(run, query, key, value, seq_info, groups):+ """Documents attend only inside themselves: each group runs on its own rows and rebased+ cu_seqlens, and the rows are joined back (autograd splits the backward the same way)."""+ lengths, cu_seqlens, _ = seq_info+ outs = []+ for first, end, start, stop, longest in groups:+ rows = slice(start, stop)+ info = (lengths[first:end], cu_seqlens[first : end + 1] - start, longest)+ outs.append(run(query[:, :, rows], key[:, :, rows], value[:, :, rows], info))+ HF_PACKED_ATTENTION_STATS["int32_split"] += 1+ return torch.cat(outs, dim = 1)++def _sdpa_packed_varlen(module,query,@@ -437,6 +474,20 @@ def _sdpa_packed_varlen(HF_PACKED_ATTENTION_STATS["fallback"] += 1return orig(module, query, key, value, attention_mask, *args, **named, **kwargs)block_varlen = kwargs.pop(BLOCK_VARLEN_KWARG, None)+ if isinstance(block_varlen, str):+ if torch.compiler.is_compiling() and not _disabled():+ return _sdpa_packed_varlen_eager(+ module,+ query,+ key,+ value,+ attention_mask,+ dropout = dropout,+ scaling = scaling,+ is_causal = is_causal,+ **kwargs,+ )+ block_varlen = None# Fallback gets the uncast Q / K / V so GQA's dK / dV reduce in fp32 after repeat_kv.given = (query, key, value)if kwargs.get("packed_seq_lengths") is not None and not _disabled():@@ -503,7 +554,7 @@ def _sdpa_packed_varlen(from .packing import get_packed_info_from_kwargsbsz, n_heads, q_len, head_dim = query.shape- seq_info = torch_varlen = backend = None+ seq_info = torch_varlen = backend = groups = Noneefficient = Falsewindow = _static_window(module, query, key, value, dropout, is_causal, kwargs)if window is not _NO_WINDOW:@@ -530,7 +581,17 @@ def _sdpa_packed_varlen(kwargs["packed_seq_lengths"].numel(), q_len, n_heads, head_dim)):- torch_varlen, backend, refused = None, None, False+ from .packing import _host_segment_lengths++ # Documents are independent: groups of them stay under the limit, exactly.+ refused, groups = (+ False,+ _int32_safe_groups(+ _host_segment_lengths(kwargs["packed_seq_lengths"]), n_heads, head_dim+ ),+ )+ if groups is None:+ torch_varlen, backend = None, None# Refused shape and no flash-attn / xformers: the efficient kernel.if refused and backend not in (FLASH_VARLEN, XFORMERS):efficient = _efficient_varlen_takes(query, window)@@ -580,9 +641,21 @@ def _sdpa_packed_varlen(return out, Noneif torch_varlen is not None:try:- out = _call_torch_varlen(- torch_varlen, query, key, value, cu_seqlens, max_seqlen, scale, window- )+ if groups is None:+ out = _call_torch_varlen(+ torch_varlen, query, key, value, cu_seqlens, max_seqlen, scale, window+ )+ else:+ out = _run_document_groups(+ lambda q, k, v, info: _call_torch_varlen(+ torch_varlen, q, k, v, info[1], info[2], scale, window+ ),+ query,+ key,+ value,+ seq_info,+ groups,+ )except torch.OutOfMemoryError:raiseexcept RuntimeError as exc:@@ -619,21 +692,28 @@ def _sdpa_packed_varlen(sdpa_kwargs = {"scale": scale},xformers_kwargs = {"scale": scale},)- context = AttentionContext(- bsz = 1,- q_len = q_len,- kv_seq_len = q_len,- n_heads = n_heads,- head_dim = head_dim,- # Not grad mode: xformers needs a backward op for grad tensors even under no_grad (reentrant- # checkpointing), and that op rejects the grouped 5D layout.- requires_grad = query.requires_grad or key.requires_grad or value.requires_grad,- seq_info = seq_info,- attention_mask = None,- causal_mask = None,- sliding_window = window,- )- out = run_attention(config = config, context = context, Q = query, K = key, V = value)++ def dispatch(query, key, value, seq_info):+ context = AttentionContext(+ bsz = 1,+ q_len = query.shape[2],+ kv_seq_len = query.shape[2],+ n_heads = n_heads,+ head_dim = head_dim,+ # Not grad mode: xformers needs a backward op for grad tensors even under no_grad+ # (reentrant checkpointing), and that op rejects the grouped 5D layout.+ requires_grad = query.requires_grad or key.requires_grad or value.requires_grad,+ seq_info = seq_info,+ attention_mask = None,+ causal_mask = None,+ sliding_window = window,+ )+ return run_attention(config = config, context = context, Q = query, K = key, V = value)++ if groups is None:+ out = dispatch(query, key, value, seq_info)+ else:+ out = _run_document_groups(dispatch, query, key, value, seq_info, groups)return out.contiguous(), None@@ -741,7 +821,8 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):"""Eager, before _block_compile runs a compiled decoder block: (cu_seqlens, max_seqlen) for itspacked row, so attention runs torch's varlen op inside the graph. Building them there needs a hostread of the longest segment and the mask check a host sync, which is why the traced wrapper used to- fall back to SDPA over the dense mask. None keeps that path."""+ fall back to SDPA over the dense mask. None keeps that path; _LEAVE_BLOCK runs the eager+ wrapper."""seq_lengths, mask = kwargs.get("packed_seq_lengths"), kwargs.get("attention_mask")attention = getattr(layer, "self_attn", None)# Layers of one forward (and its checkpoint recompute) share the lengths and their window's mask.@@ -760,7 +841,7 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):# sdpa_kernel / deterministic mode can change between calls on the same tensors.and (entry[5] is None or _sdpa_policy_requested(*entry[5]) == entry[6])):- if entry[3] is not None:+ if isinstance(entry[3], tuple):HF_PACKED_ATTENTION_STATS["compiled"] += 1return entry[3]policy = []@@ -833,7 +914,12 @@ def _block_varlen_prepare(if not _VARLEN_INT32_GUARD_DISABLED and _varlen_backward_overflows_int32(kwargs["packed_seq_lengths"].numel(), total, n_heads, head_dim):- return None+ from .packing import _host_segment_lengths++ # Splitting at document boundaries in the graph would need per-step shapes: the eager wrapper+ # splits instead (a graph break for these rows only). One document alone keeps the dense path.+ lengths = _host_segment_lengths(kwargs["packed_seq_lengths"])+ return None if _int32_safe_groups(lengths, n_heads, head_dim) is None else _LEAVE_BLOCKseq_info = get_packed_info_from_kwargs(kwargs, hidden_states.device)if seq_info is None or not _mask_is_block_causal(kwargs.get("attention_mask"), seq_info[1], total, windowdiff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex ae83c2dd991..004e3b311cc 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -201,11 +201,56 @@ def configure_padding_free(config):setattr(config, "remove_unused_columns", False)-def _enable_hf_packed_attention() -> None:+def _longest_document(model, trainer) -> Optional[int]:+ args = getattr(trainer, "args", None)+ for value in (+ getattr(args, "max_length", None),+ getattr(args, "max_seq_length", None),+ getattr(model, "max_seq_length", None),+ ):+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:+ return value+ return None+++def _packed_row_tokens(model, trainer) -> Optional[int]:+ """Tokens in one flattened row: the padding-free collator joins a whole device batch, the eval+ batch too when there is an eval dataset (transformers' default eval batch is 8)."""+ longest = _longest_document(model, trainer)+ if longest is None:+ return None+ args = getattr(trainer, "args", None)+ names = ["per_device_train_batch_size"]+ if getattr(trainer, "eval_dataset", None) is not None:+ names.append("per_device_eval_batch_size")+ batches = [getattr(args, name, None) for name in names]+ batches = [b for b in batches if isinstance(b, int) and not isinstance(b, bool) and b > 0]+ return longest * max(batches, default = 1)+++def _enable_hf_packed_attention(model = None, trainer = None) -> None:# packed rows on the transformers modeling path run varlen instead of SDPA over a dense mask.try:from .hf_packed_attention import enable_hf_packed_attention- enable_hf_packed_attention()++ # A whole-model torch.compile (torch_compile=True, or an already compiled model) inlines the+ # blocks, where the varlen wrapper falls back to dense-mask SDPA: those keep flex.+ outer_compile = bool(getattr(getattr(trainer, "args", None), "torch_compile", False)) or (+ hasattr(model, "_orig_mod")+ )+ # A collator padding the flattened row to a multiple leaves a tail the varlen wrapper declines.+ pad_multiple = getattr(getattr(trainer, "data_collator", None), "pad_to_multiple_of", None)+ aligned_tail = isinstance(pad_multiple, int) and pad_multiple > 1+ if (+ enable_hf_packed_attention()+ and model is not None+ and not outer_compile+ and not aligned_tail+ ):+ from ..models._utils import use_sdpa_for_packed_head_dim_flex+ use_sdpa_for_packed_head_dim_flex(+ model, _longest_document(model, trainer), _packed_row_tokens(model, trainer)+ )except Exception as exc:logging.getLogger(__name__).info(f"Unsloth: packed varlen attention not enabled ({exc})")@@ -247,7 +292,7 @@ def enable_sample_packing(collator = getattr(trainer, "data_collator", None)if collator is None or not hasattr(collator, "torch_call"):return- _enable_hf_packed_attention()+ _enable_hf_packed_attention(model, trainer)_scope_flash_attention_impl(model, trainer, sequence_lengths_key)if getattr(collator, "_unsloth_packing_wrapped", False):return@@ -294,7 +339,7 @@ def enable_padding_free_metadata(model, trainer):returnmark_allow_overlength(model)- _enable_hf_packed_attention()+ _enable_hf_packed_attention(model, trainer)if hasattr(collator, "return_position_ids"):collator.return_position_ids = Trueif hasattr(trainer, "args") and hasattr(trainer.args, "remove_unused_columns"):