Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Flex wrapper: send short no-grad queries to the main kernel where flex_decoding cannot compile (#16092)

Daniel Hanfae3823d2e77
diff --git a/tests/test_flex_decode_guard_wrapper.py b/tests/test_flex_decode_guard_wrapper.py
new file mode 100644
index 00000000000..728afd93cc5
--- /dev/null
+++ b/tests/test_flex_decode_guard_wrapper.py
@@ -0,0 +1,166 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The registered Transformers flex wrapper keeps short no-grad queries off flex_decoding where unsloth_zoo found it
+cannot compile (Triton 3.9 removed the block pointers torch 2.9 - 2.12's decode template builds)."""
+
+import sys
+import types
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers)
+import unsloth.models._utils as u
+
+
+def _wrapped_call(
+ monkeypatch,
+ unavailable,
+ q_len = 1,
+ needs_backward = False,
+ small_ok = False,
+ head_dim = 64,
+ **kwargs,
+):
+ import torch
+
+ seen = {}
+
+ def fake_flex(module, query, key, value, *args, **kw):
+ seen["kernel_options"] = kw.get("kernel_options")
+ return query
+
+ monkeypatch.setattr(u, "_flex_decoding_unavailable", lambda: unavailable, raising = False)
+ monkeypatch.setattr(u, "_flex_tuned_bwd_device_ok", lambda: False)
+ monkeypatch.setattr(u, "_flex_small_smem_device_ok", lambda: small_ok)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: needs_backward)
+ wrapped = u._wrap_flex_attention_forward(fake_flex)
+
+ class M(torch.nn.Module):
+ is_causal = True
+
+ q = torch.randn(1, 2, q_len, head_dim)
+ k = torch.randn(1, 2, 128, head_dim)
+ wrapped(M(), q, k, k.clone(), object(), **kwargs)
+ return seen["kernel_options"]
+
+
[email protected]("q_len", [1, 5, 127])
+def test_short_no_grad_query_takes_the_main_kernel_when_flex_decoding_cannot_compile(
+ monkeypatch, q_len
+):
+ assert _wrapped_call(monkeypatch, True, q_len) == {"FORCE_USE_FLEX_ATTENTION": True}
+
+
[email protected]("q_len", [1, 5])
+def test_short_no_grad_query_keeps_auto_otherwise(monkeypatch, q_len):
+ assert _wrapped_call(monkeypatch, False, q_len) is None
+
+
+def test_backward_calls_are_unchanged(monkeypatch):
+ for unavailable in (False, True):
+ assert _wrapped_call(monkeypatch, unavailable, 5, needs_backward = True) == {
+ "FORCE_USE_FLEX_ATTENTION": True
+ }
+
+
+def test_caller_options_stay_on_top(monkeypatch):
+ opts = _wrapped_call(monkeypatch, True, 1, kernel_options = {"PRESCALE_QK": True})
+ assert opts == {"FORCE_USE_FLEX_ATTENTION": True, "PRESCALE_QK": True}
+ opts = _wrapped_call(monkeypatch, True, 1, kernel_options = {"FORCE_USE_FLEX_ATTENTION": False})
+ assert opts == {"FORCE_USE_FLEX_ATTENTION": False}
+
+
+def test_a_named_backend_is_left_alone(monkeypatch):
+ assert _wrapped_call(monkeypatch, True, 1, kernel_options = {"BACKEND": "TRITON_DECODE"}) == {
+ "BACKEND": "TRITON_DECODE"
+ }
+
+
+def test_flag_comes_from_unsloth_zoo(monkeypatch):
+ stub = types.ModuleType("unsloth_zoo.flex_attention.utils")
+ stub.FLEX_DECODING_UNAVAILABLE = True
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.flex_attention.utils", stub)
+ assert u._flex_decoding_unavailable() is True
+ stub.FLEX_DECODING_UNAVAILABLE = False
+ assert u._flex_decoding_unavailable() is False
+
+
+def test_an_unsloth_zoo_without_the_check_keeps_auto(monkeypatch):
+ stub = types.ModuleType("unsloth_zoo.flex_attention.utils")
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.flex_attention.utils", stub)
+ assert u._flex_decoding_unavailable() is False
+
+
+def _select_torch_2_9(options):
+ # torch 2.9 lowering: no BACKEND; flex_decoding unless FORCE_USE_FLEX_ATTENTION.
+ return "main" if options.get("FORCE_USE_FLEX_ATTENTION", False) else "decode"
+
+
+def _select_torch_2_12(options):
+ # torch 2.10 - 2.12: BACKEND plus the legacy knob is rejected; AUTO picks flex_decoding unless forced.
+ if "BACKEND" in options and options.get("FORCE_USE_FLEX_ATTENTION", False):
+ raise ValueError("BACKEND cannot be combined with legacy FORCE_USE_FLEX_ATTENTION")
+ backend = options.get("BACKEND", "AUTO")
+ if backend == "TRITON_DECODE" or (
+ backend == "AUTO" and not options.get("FORCE_USE_FLEX_ATTENTION", False)
+ ):
+ return "decode"
+ return "main"
+
+
[email protected]("select", [_select_torch_2_9, _select_torch_2_12])
[email protected]("needs_backward", [False, True])
+def test_an_explicit_auto_backend_takes_the_main_kernel_too(monkeypatch, needs_backward, select):
+ # BACKEND="TRITON" would not stop flex_decoding on torch 2.9, so AUTO is dropped for the legacy knob.
+ opts = _wrapped_call(monkeypatch, True, 1, needs_backward, kernel_options = {"BACKEND": "AUTO"})
+ assert opts == {"FORCE_USE_FLEX_ATTENTION": True} and select(opts) == "main"
+ opts = _wrapped_call(monkeypatch, False, 1, needs_backward, kernel_options = {"BACKEND": "AUTO"})
+ if needs_backward:
+ assert opts == {"FORCE_USE_FLEX_ATTENTION": True} and select(opts) == "main"
+ else:
+ assert opts == {"BACKEND": "AUTO"}
+ opts = _wrapped_call(
+ monkeypatch, True, 1, kernel_options = {"BACKEND": "AUTO", "FORCE_USE_FLEX_ATTENTION": False}
+ )
+ assert opts == {"BACKEND": "AUTO", "FORCE_USE_FLEX_ATTENTION": False}
+
+
[email protected]("needs_backward", [False, True])
[email protected]("backend", ["TRITON_DECODE", "FLASH", "TRITON"])
+def test_a_named_backend_in_any_mapping_is_left_alone(monkeypatch, needs_backward, backend):
+ # torch reads kernel_options through dict(); a read-only mapping must not hide its BACKEND.
+ from types import MappingProxyType
+ opts = _wrapped_call(
+ monkeypatch, True, 1, needs_backward, kernel_options = MappingProxyType({"BACKEND": backend})
+ )
+ assert opts == {"BACKEND": backend}
+
+
+def test_an_auto_backend_in_a_read_only_mapping_takes_the_main_kernel(monkeypatch):
+ from types import MappingProxyType
+
+ caller = MappingProxyType({"BACKEND": "AUTO", "PRESCALE_QK": True})
+ opts = _wrapped_call(monkeypatch, True, 1, kernel_options = caller)
+ assert opts == {"FORCE_USE_FLEX_ATTENTION": True, "PRESCALE_QK": True}
+ assert dict(caller) == {"BACKEND": "AUTO", "PRESCALE_QK": True}
+
+
+ "requested, main_kernel",
+ [
+ (None, True),
+ ({"BACKEND": "AUTO"}, True),
+ ({"FORCE_USE_FLEX_ATTENTION": False}, False),
+ ({"BACKEND": "TRITON_DECODE"}, False),
+ ],
+)
+def test_small_memory_tiles_follow_the_kernel_that_runs(monkeypatch, requested, main_kernel):
+ # The fitted tiles are main-kernel tiles: a caller that keeps flex_decoding must not get them.
+ kwargs = {} if requested is None else {"kernel_options": dict(requested)}
+ opts = _wrapped_call(monkeypatch, True, 1, small_ok = True, head_dim = 256, **kwargs) or {}
+ tiles = {k: opts.get(k) for k in u._FLEX_SMALL_SMEM_KERNEL_OPTIONS}
+ if main_kernel:
+ assert tiles == u._FLEX_SMALL_SMEM_KERNEL_OPTIONS
+ else:
+ assert all(v is None for v in tiles.values()) and opts == requested
diff --git a/tests/test_flex_large_head_dim_kernel_options.py b/tests/test_flex_large_head_dim_kernel_options.py
index eac8447cbcb..4fcaf81a77f 100644
--- a/tests/test_flex_large_head_dim_kernel_options.py
+++ b/tests/test_flex_large_head_dim_kernel_options.py
@@ -8,6 +8,12 @@ import unsloth # noqa: F401 (must precede transformers)
import unsloth.models._utils as u
[email protected](autouse = True)
+def _flex_decoding_available(monkeypatch):
+ # These pin the routing where flex_decoding compiles; test_flex_decode_guard_wrapper.py covers a host where it cannot.
+ monkeypatch.setattr(u, "_flex_decoding_unavailable", lambda: False, raising = False)
+
+
@pytest.mark.parametrize("head_dim", [32, 64, 128, 192, 256])
def test_no_kernel_options_at_or_below_256(head_dim):
assert u._flex_kernel_options_for_head_dim(head_dim) is None
diff --git a/tests/test_flex_small_smem_tiles.py b/tests/test_flex_small_smem_tiles.py
index f911d3c9bb2..be66f8f402e 100644
--- a/tests/test_flex_small_smem_tiles.py
+++ b/tests/test_flex_small_smem_tiles.py
@@ -10,6 +10,13 @@ import pytest
import unsloth # noqa: F401 (must precede transformers)
import unsloth.models._utils as u
+
[email protected](autouse = True)
+def _flex_decoding_available(monkeypatch):
+ # These pin the routing where flex_decoding compiles; test_flex_decode_guard_wrapper.py covers a host where it cannot.
+ monkeypatch.setattr(u, "_flex_decoding_unavailable", lambda: False, raising = False)
+
+
T4 = ((7, 5), 65536)
A10 = ((8, 6), 101376)
RTX3090 = ((8, 6), 101376)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 82dfedb5db6..17290f187e6 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -1033,6 +1033,15 @@ def _flex_call_needs_backward(query, key, value):
return False
+def _flex_decoding_unavailable():
+ """True when unsloth_zoo found flex_decoding cannot compile against the installed Triton."""
+ try:
+ from unsloth_zoo.flex_attention.utils import FLEX_DECODING_UNAVAILABLE
+ except Exception:
+ return False
+ return FLEX_DECODING_UNAVAILABLE is True
+
+
def _flex_main_kernel_forced(kwargs):
"""True when the caller's kernel_options force the main flex kernel over flex_decoding."""
requested = kwargs.get("kernel_options") or {}
@@ -1161,6 +1170,8 @@ def _wrap_flex_attention_forward(flex_attention_forward):
tuned_bwd_device_ok = _flex_tuned_bwd_device_ok()
small_smem_device_ok = _flex_small_smem_device_ok()
+ # A failed flex_decoding compile makes Dynamo run flex eagerly for the rest of the process (full score matrix).
+ decoding_unavailable = _flex_decoding_unavailable()
@functools.wraps(flex_attention_forward)
def unsloth_flex_attention_forward(module, query, key, value, *args, **kwargs):
@@ -1177,6 +1188,18 @@ def _wrap_flex_attention_forward(flex_attention_forward):
kwargs["attention_mask"] = _causal_block_mask(query, key)
elif _FLEX_MASKLESS_SDPA_ENABLED:
_count_flex_reroute("flex")
+ requested = kwargs.get("kernel_options") or {}
+ if not isinstance(requested, dict):
+ requested = dict(requested)
+ kwargs["kernel_options"] = requested
+ requested_backend = requested.get("BACKEND")
+ # Where flex_decoding cannot compile, the main kernel replaces it unless the caller pinned a kernel itself
+ # (a BACKEND other than AUTO, or FORCE_USE_FLEX_ATTENTION=False).
+ decode_replaced = (
+ decoding_unavailable
+ and requested_backend in (None, "AUTO")
+ and requested.get("FORCE_USE_FLEX_ATTENTION") is not False
+ )
try:
# Some vision callers reuse the interface with a non-4D query.
kernel_options = (
@@ -1199,6 +1222,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):
and (
_flex_call_needs_backward(query, key, value)
or _flex_main_kernel_forced(kwargs)
+ or decode_replaced
or query.shape[2] >= 128
)
and _flex_tuned_bwd_fits(
@@ -1217,15 +1241,20 @@ def _wrap_flex_attention_forward(flex_attention_forward):
kernel_options = _flex_tuned_bwd_kernel_options(query.shape[-1], tuned_bwd_device_ok)
# Inductor picks flex_decoding for a static query length below 128. Its logsumexp gets a
# padded batch stride (comprehensive_padding) that the backward template ignores, so every
- # batch row after the first reads a shifted LSE and the gradients blow up.
- if _flex_call_needs_backward(query, key, value) and "BACKEND" not in (
- kwargs.get("kernel_options") or {}
+ # batch row after the first reads a shifted LSE and the gradients blow up. Where flex_decoding cannot compile,
+ # no-grad queries take the main kernel too. BACKEND="AUTO" is dropped: torch 2.10+ rejects it with the legacy knob.
+ if requested_backend in (None, "AUTO") and (
+ decoding_unavailable or _flex_call_needs_backward(query, key, value)
):
kernel_options = kernel_options or {}
kernel_options["FORCE_USE_FLEX_ATTENTION"] = True
+ if (
+ requested_backend == "AUTO"
+ and requested.get("FORCE_USE_FLEX_ATTENTION") is not False
+ ):
+ requested = {k: v for k, v in requested.items() if k != "BACKEND"}
if kernel_options is not None:
# A caller that already asked for something keeps it: only fill the gaps.
- requested = kwargs.get("kernel_options") or {}
kernel_options.update(requested)
kwargs["kernel_options"] = kernel_options
return flex_attention_forward(module, query, key, value, *args, **kwargs)