Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Flex wrapper: send short no-grad queries to the main kernel where flex_decoding cannot compile (#16092)
Daniel Hanfae3823d2e77
diff --git a/tests/test_flex_decode_guard_wrapper.py b/tests/test_flex_decode_guard_wrapper.pynew file mode 100644index 00000000000..728afd93cc5--- /dev/null+++ b/tests/test_flex_decode_guard_wrapper.py@@ -0,0 +1,166 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The registered Transformers flex wrapper keeps short no-grad queries off flex_decoding where unsloth_zoo found it+cannot compile (Triton 3.9 removed the block pointers torch 2.9 - 2.12's decode template builds)."""++import sys+import types++import pytest++import unsloth # noqa: F401 (must precede transformers)+import unsloth.models._utils as u+++def _wrapped_call(+ monkeypatch,+ unavailable,+ q_len = 1,+ needs_backward = False,+ small_ok = False,+ head_dim = 64,+ **kwargs,+):+ import torch++ seen = {}++ def fake_flex(module, query, key, value, *args, **kw):+ seen["kernel_options"] = kw.get("kernel_options")+ return query++ monkeypatch.setattr(u, "_flex_decoding_unavailable", lambda: unavailable, raising = False)+ monkeypatch.setattr(u, "_flex_tuned_bwd_device_ok", lambda: False)+ monkeypatch.setattr(u, "_flex_small_smem_device_ok", lambda: small_ok)+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: needs_backward)+ wrapped = u._wrap_flex_attention_forward(fake_flex)++ class M(torch.nn.Module):+ is_causal = True++ q = torch.randn(1, 2, q_len, head_dim)+ k = torch.randn(1, 2, 128, head_dim)+ wrapped(M(), q, k, k.clone(), object(), **kwargs)+ return seen["kernel_options"]++[email protected]("q_len", [1, 5, 127])+def test_short_no_grad_query_takes_the_main_kernel_when_flex_decoding_cannot_compile(+ monkeypatch, q_len+):+ assert _wrapped_call(monkeypatch, True, q_len) == {"FORCE_USE_FLEX_ATTENTION": True}++[email protected]("q_len", [1, 5])+def test_short_no_grad_query_keeps_auto_otherwise(monkeypatch, q_len):+ assert _wrapped_call(monkeypatch, False, q_len) is None+++def test_backward_calls_are_unchanged(monkeypatch):+ for unavailable in (False, True):+ assert _wrapped_call(monkeypatch, unavailable, 5, needs_backward = True) == {+ "FORCE_USE_FLEX_ATTENTION": True+ }+++def test_caller_options_stay_on_top(monkeypatch):+ opts = _wrapped_call(monkeypatch, True, 1, kernel_options = {"PRESCALE_QK": True})+ assert opts == {"FORCE_USE_FLEX_ATTENTION": True, "PRESCALE_QK": True}+ opts = _wrapped_call(monkeypatch, True, 1, kernel_options = {"FORCE_USE_FLEX_ATTENTION": False})+ assert opts == {"FORCE_USE_FLEX_ATTENTION": False}+++def test_a_named_backend_is_left_alone(monkeypatch):+ assert _wrapped_call(monkeypatch, True, 1, kernel_options = {"BACKEND": "TRITON_DECODE"}) == {+ "BACKEND": "TRITON_DECODE"+ }+++def test_flag_comes_from_unsloth_zoo(monkeypatch):+ stub = types.ModuleType("unsloth_zoo.flex_attention.utils")+ stub.FLEX_DECODING_UNAVAILABLE = True+ monkeypatch.setitem(sys.modules, "unsloth_zoo.flex_attention.utils", stub)+ assert u._flex_decoding_unavailable() is True+ stub.FLEX_DECODING_UNAVAILABLE = False+ assert u._flex_decoding_unavailable() is False+++def test_an_unsloth_zoo_without_the_check_keeps_auto(monkeypatch):+ stub = types.ModuleType("unsloth_zoo.flex_attention.utils")+ monkeypatch.setitem(sys.modules, "unsloth_zoo.flex_attention.utils", stub)+ assert u._flex_decoding_unavailable() is False+++def _select_torch_2_9(options):+ # torch 2.9 lowering: no BACKEND; flex_decoding unless FORCE_USE_FLEX_ATTENTION.+ return "main" if options.get("FORCE_USE_FLEX_ATTENTION", False) else "decode"+++def _select_torch_2_12(options):+ # torch 2.10 - 2.12: BACKEND plus the legacy knob is rejected; AUTO picks flex_decoding unless forced.+ if "BACKEND" in options and options.get("FORCE_USE_FLEX_ATTENTION", False):+ raise ValueError("BACKEND cannot be combined with legacy FORCE_USE_FLEX_ATTENTION")+ backend = options.get("BACKEND", "AUTO")+ if backend == "TRITON_DECODE" or (+ backend == "AUTO" and not options.get("FORCE_USE_FLEX_ATTENTION", False)+ ):+ return "decode"+ return "main"++[email protected]("select", [_select_torch_2_9, _select_torch_2_12])[email protected]("needs_backward", [False, True])+def test_an_explicit_auto_backend_takes_the_main_kernel_too(monkeypatch, needs_backward, select):+ # BACKEND="TRITON" would not stop flex_decoding on torch 2.9, so AUTO is dropped for the legacy knob.+ opts = _wrapped_call(monkeypatch, True, 1, needs_backward, kernel_options = {"BACKEND": "AUTO"})+ assert opts == {"FORCE_USE_FLEX_ATTENTION": True} and select(opts) == "main"+ opts = _wrapped_call(monkeypatch, False, 1, needs_backward, kernel_options = {"BACKEND": "AUTO"})+ if needs_backward:+ assert opts == {"FORCE_USE_FLEX_ATTENTION": True} and select(opts) == "main"+ else:+ assert opts == {"BACKEND": "AUTO"}+ opts = _wrapped_call(+ monkeypatch, True, 1, kernel_options = {"BACKEND": "AUTO", "FORCE_USE_FLEX_ATTENTION": False}+ )+ assert opts == {"BACKEND": "AUTO", "FORCE_USE_FLEX_ATTENTION": False}++[email protected]("needs_backward", [False, True])[email protected]("backend", ["TRITON_DECODE", "FLASH", "TRITON"])+def test_a_named_backend_in_any_mapping_is_left_alone(monkeypatch, needs_backward, backend):+ # torch reads kernel_options through dict(); a read-only mapping must not hide its BACKEND.+ from types import MappingProxyType+ opts = _wrapped_call(+ monkeypatch, True, 1, needs_backward, kernel_options = MappingProxyType({"BACKEND": backend})+ )+ assert opts == {"BACKEND": backend}+++def test_an_auto_backend_in_a_read_only_mapping_takes_the_main_kernel(monkeypatch):+ from types import MappingProxyType++ caller = MappingProxyType({"BACKEND": "AUTO", "PRESCALE_QK": True})+ opts = _wrapped_call(monkeypatch, True, 1, kernel_options = caller)+ assert opts == {"FORCE_USE_FLEX_ATTENTION": True, "PRESCALE_QK": True}+ assert dict(caller) == {"BACKEND": "AUTO", "PRESCALE_QK": True}+++ "requested, main_kernel",+ [+ (None, True),+ ({"BACKEND": "AUTO"}, True),+ ({"FORCE_USE_FLEX_ATTENTION": False}, False),+ ({"BACKEND": "TRITON_DECODE"}, False),+ ],+)+def test_small_memory_tiles_follow_the_kernel_that_runs(monkeypatch, requested, main_kernel):+ # The fitted tiles are main-kernel tiles: a caller that keeps flex_decoding must not get them.+ kwargs = {} if requested is None else {"kernel_options": dict(requested)}+ opts = _wrapped_call(monkeypatch, True, 1, small_ok = True, head_dim = 256, **kwargs) or {}+ tiles = {k: opts.get(k) for k in u._FLEX_SMALL_SMEM_KERNEL_OPTIONS}+ if main_kernel:+ assert tiles == u._FLEX_SMALL_SMEM_KERNEL_OPTIONS+ else:+ assert all(v is None for v in tiles.values()) and opts == requesteddiff --git a/tests/test_flex_large_head_dim_kernel_options.py b/tests/test_flex_large_head_dim_kernel_options.pyindex eac8447cbcb..4fcaf81a77f 100644--- a/tests/test_flex_large_head_dim_kernel_options.py+++ b/tests/test_flex_large_head_dim_kernel_options.py@@ -8,6 +8,12 @@ import unsloth # noqa: F401 (must precede transformers)import unsloth.models._utils as u[email protected](autouse = True)+def _flex_decoding_available(monkeypatch):+ # These pin the routing where flex_decoding compiles; test_flex_decode_guard_wrapper.py covers a host where it cannot.+ monkeypatch.setattr(u, "_flex_decoding_unavailable", lambda: False, raising = False)++@pytest.mark.parametrize("head_dim", [32, 64, 128, 192, 256])def test_no_kernel_options_at_or_below_256(head_dim):assert u._flex_kernel_options_for_head_dim(head_dim) is Nonediff --git a/tests/test_flex_small_smem_tiles.py b/tests/test_flex_small_smem_tiles.pyindex f911d3c9bb2..be66f8f402e 100644--- a/tests/test_flex_small_smem_tiles.py+++ b/tests/test_flex_small_smem_tiles.py@@ -10,6 +10,13 @@ import pytestimport unsloth # noqa: F401 (must precede transformers)import unsloth.models._utils as u+[email protected](autouse = True)+def _flex_decoding_available(monkeypatch):+ # These pin the routing where flex_decoding compiles; test_flex_decode_guard_wrapper.py covers a host where it cannot.+ monkeypatch.setattr(u, "_flex_decoding_unavailable", lambda: False, raising = False)++T4 = ((7, 5), 65536)A10 = ((8, 6), 101376)RTX3090 = ((8, 6), 101376)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 82dfedb5db6..17290f187e6 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -1033,6 +1033,15 @@ def _flex_call_needs_backward(query, key, value):return False+def _flex_decoding_unavailable():+ """True when unsloth_zoo found flex_decoding cannot compile against the installed Triton."""+ try:+ from unsloth_zoo.flex_attention.utils import FLEX_DECODING_UNAVAILABLE+ except Exception:+ return False+ return FLEX_DECODING_UNAVAILABLE is True++def _flex_main_kernel_forced(kwargs):"""True when the caller's kernel_options force the main flex kernel over flex_decoding."""requested = kwargs.get("kernel_options") or {}@@ -1161,6 +1170,8 @@ def _wrap_flex_attention_forward(flex_attention_forward):tuned_bwd_device_ok = _flex_tuned_bwd_device_ok()small_smem_device_ok = _flex_small_smem_device_ok()+ # A failed flex_decoding compile makes Dynamo run flex eagerly for the rest of the process (full score matrix).+ decoding_unavailable = _flex_decoding_unavailable()@functools.wraps(flex_attention_forward)def unsloth_flex_attention_forward(module, query, key, value, *args, **kwargs):@@ -1177,6 +1188,18 @@ def _wrap_flex_attention_forward(flex_attention_forward):kwargs["attention_mask"] = _causal_block_mask(query, key)elif _FLEX_MASKLESS_SDPA_ENABLED:_count_flex_reroute("flex")+ requested = kwargs.get("kernel_options") or {}+ if not isinstance(requested, dict):+ requested = dict(requested)+ kwargs["kernel_options"] = requested+ requested_backend = requested.get("BACKEND")+ # Where flex_decoding cannot compile, the main kernel replaces it unless the caller pinned a kernel itself+ # (a BACKEND other than AUTO, or FORCE_USE_FLEX_ATTENTION=False).+ decode_replaced = (+ decoding_unavailable+ and requested_backend in (None, "AUTO")+ and requested.get("FORCE_USE_FLEX_ATTENTION") is not False+ )try:# Some vision callers reuse the interface with a non-4D query.kernel_options = (@@ -1199,6 +1222,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):and (_flex_call_needs_backward(query, key, value)or _flex_main_kernel_forced(kwargs)+ or decode_replacedor query.shape[2] >= 128)and _flex_tuned_bwd_fits(@@ -1217,15 +1241,20 @@ def _wrap_flex_attention_forward(flex_attention_forward):kernel_options = _flex_tuned_bwd_kernel_options(query.shape[-1], tuned_bwd_device_ok)# Inductor picks flex_decoding for a static query length below 128. Its logsumexp gets a# padded batch stride (comprehensive_padding) that the backward template ignores, so every- # batch row after the first reads a shifted LSE and the gradients blow up.- if _flex_call_needs_backward(query, key, value) and "BACKEND" not in (- kwargs.get("kernel_options") or {}+ # batch row after the first reads a shifted LSE and the gradients blow up. Where flex_decoding cannot compile,+ # no-grad queries take the main kernel too. BACKEND="AUTO" is dropped: torch 2.10+ rejects it with the legacy knob.+ if requested_backend in (None, "AUTO") and (+ decoding_unavailable or _flex_call_needs_backward(query, key, value)):kernel_options = kernel_options or {}kernel_options["FORCE_USE_FLEX_ATTENTION"] = True+ if (+ requested_backend == "AUTO"+ and requested.get("FORCE_USE_FLEX_ATTENTION") is not False+ ):+ requested = {k: v for k, v in requested.items() if k != "BACKEND"}if kernel_options is not None:# A caller that already asked for something keeps it: only fill the gaps.- requested = kwargs.get("kernel_options") or {}kernel_options.update(requested)kwargs["kernel_options"] = kernel_optionsreturn flex_attention_forward(module, query, key, value, *args, **kwargs)