Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let the compile cache keep denoiser graphs with sdpa_kernel blocks (restart first render 1-5.5 s faster) (#12510)

Daniel Han2eaf5b4ccff4
diff --git a/studio/backend/core/inference/diffusion_aot_cache.py b/studio/backend/core/inference/diffusion_aot_cache.py
new file mode 100644
index 00000000000..a414d2ee4ff
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_aot_cache.py
@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Let AOTAutogradCache store the denoiser graphs that hold an ``sdpa_kernel`` block.
+
+Dynamo bakes ``sdpa_kernel`` into the graph as ``_backend_from_string`` / ``_sdpa_kernel`` calls, which are not on
+AOTAutogradCache's allowlist, so every regional block of FLUX.1, Z-Image and Qwen-Image re-ran AOT tracing on each
+process start (pytorch/pytorch#194007; 2.6 / 4.7 / 1.8 s of a B200 restart's first render). Their arguments are graph
+constants, so the graph keys them apart; the one ambient input, SDPA's backend priority order, goes into the marked
+value torch folds into the cache key. UNSLOTH_DIFFUSION_SDPA_AOT_CACHE=0 disables it.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+from . import diffusion_compile_config as compile_config
+
+_ENV = "UNSLOTH_DIFFUSION_SDPA_AOT_CACHE"
+_INDUCTOR_MODULE = "torch._inductor.config"
+_KNOB = "unsafe_marked_cacheable_functions"
+# The names check_node_safe builds: f"{target.__module__}.{target.__name__}".
+SDPA_HELPERS = ("torch.nn.attention._backend_from_string", "torch.nn.attention._sdpa_kernel")
+
+
+def enabled() -> bool:
+ raw = (os.environ.get(_ENV) or "").strip().lower()
+ return raw not in ("0", "false", "no", "off")
+
+
+def _cache_key_value(torch: Any) -> str:
+ try:
+ priority = tuple(int(b) for b in torch._C._get_sdp_priority_order())
+ except Exception: # noqa: BLE001 - no priority API on this build: key on the version alone
+ priority = None
+ return f"{torch.__version__}|sdp_priority={priority}"
+
+
+def install(logger: Any = None) -> bool:
+ """Mark the two SDPA helpers cacheable (idempotent, process-wide). False when disabled or unsupported."""
+ if not enabled():
+ return False
+ try:
+ import torch
+ import torch.nn.attention as attention
+
+ if not all(
+ callable(getattr(attention, name.rsplit(".", 1)[1], None)) for name in SDPA_HELPERS
+ ):
+ return False
+ # Both the recorded knob (re-applied on render threads) and this thread's live value.
+ recorded = compile_config.get_knob(_INDUCTOR_MODULE, _KNOB)
+ live = getattr(getattr(torch._inductor, "config", None), _KNOB, None)
+ if not isinstance(recorded, dict) or not isinstance(live, dict):
+ return False
+ value = _cache_key_value(torch)
+ if all(recorded.get(n) == value and live.get(n) == value for n in SDPA_HELPERS):
+ return True
+ merged = {**recorded, **live}
+ merged.update({name: value for name in SDPA_HELPERS})
+ if not compile_config.set_knob(_INDUCTOR_MODULE, _KNOB, merged):
+ return False
+ if logger is not None:
+ logger.info(
+ "diffusion.aot_cache: sdpa_kernel graphs are AOTAutograd-cacheable (%s)", value
+ )
+ return True
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ if logger is not None:
+ logger.debug("diffusion.aot_cache: not installed (%s: %s)", type(exc).__name__, exc)
+ return False
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 977b487c82e..92572cc75f6 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -886,6 +886,12 @@ def _compile_repeated_blocks(
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "compile_repeated_blocks", exc)
return False
+ # A block whose attention runs inside sdpa_kernel would otherwise bypass AOTAutogradCache on every start.
+ try:
+ from . import diffusion_aot_cache
+ diffusion_aot_cache.install(logger)
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "sdpa aot cache", exc)
if unet is not None:
# Whole-module static compile for the U-Net classes above. fullgraph mirrors the regional decision; dynamic is
# ALWAYS False, so each new (height, width, batch) pays its own compile. ``Module.compile`` keeps the module
diff --git a/studio/backend/tests/test_diffusion_sdpa_aot_cache.py b/studio/backend/tests/test_diffusion_sdpa_aot_cache.py
new file mode 100644
index 00000000000..ff66451619c
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_sdpa_aot_cache.py
@@ -0,0 +1,128 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""AOTAutogradCache for denoiser graphs that hold an ``sdpa_kernel`` block.
+
+CPU only, checked with a real inductor compile (an AOTAutograd cache hit after a dynamo reset).
+"""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_compile_config as compile_config # noqa: E402
+from core.inference import diffusion_speed as speed # noqa: E402
+
+HELPERS = ("torch.nn.attention._backend_from_string", "torch.nn.attention._sdpa_kernel")
+
+
[email protected](autouse = True)
+def _reset_dynamo():
+ torch._dynamo.reset()
+ yield
+ torch._dynamo.reset()
+
+
+def _restore_cacheable():
+ cfg = torch._inductor.config
+ before = dict(cfg.unsafe_marked_cacheable_functions)
+ recorded = compile_config.recorded()
+ yield
+ cfg.unsafe_marked_cacheable_functions = before
+ compile_config._reset_for_tests()
+ for (module_name, attr), value in recorded.items():
+ compile_config.set_knob(module_name, attr, value)
+
+
+class _Block(torch.nn.Module):
+ def __init__(self) -> None:
+ super().__init__()
+ self.lin = torch.nn.Linear(16, 16)
+
+ def forward(self, x):
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ q = self.lin(x)
+ with sdpa_kernel(SDPBackend.MATH):
+ out = torch.nn.functional.scaled_dot_product_attention(q, q, q)
+ return out + x
+
+
+class _Dit(torch.nn.Module):
+ _repeated_blocks = ["_Block"]
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.blocks = torch.nn.ModuleList([_Block(), _Block()])
+ self.compiled_kwargs = None
+
+ def compile_repeated_blocks(self, **kwargs):
+ self.compiled_kwargs = kwargs
+
+
+def test_regional_compile_marks_the_sdpa_helpers_cacheable(_restore_cacheable, monkeypatch):
+ # Every regional compile goes through _compile_repeated_blocks; that is where the helpers must become cacheable.
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", raising = False)
+ torch._inductor.config.unsafe_marked_cacheable_functions = {}
+ pipe = types.SimpleNamespace(transformer = _Dit())
+ speed._compile_repeated_blocks(pipe, None)
+ marked = torch._inductor.config.unsafe_marked_cacheable_functions
+ for name in HELPERS:
+ assert name in marked
+ assert torch.__version__ in marked[name]
+
+
+def test_sdpa_aot_cache_kill_switch(_restore_cacheable, monkeypatch):
+ from core.inference import diffusion_aot_cache
+
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", "0")
+ torch._inductor.config.unsafe_marked_cacheable_functions = {}
+ assert diffusion_aot_cache.install() is False
+ assert torch._inductor.config.unsafe_marked_cacheable_functions == {}
+
+
+def test_sdpa_aot_cache_key_carries_the_priority_order(_restore_cacheable, monkeypatch):
+ from core.inference import diffusion_aot_cache
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", raising = False)
+ torch._inductor.config.unsafe_marked_cacheable_functions = {"user.fn": "1"}
+ assert diffusion_aot_cache.install() is True
+ marked = dict(torch._inductor.config.unsafe_marked_cacheable_functions)
+ assert marked["user.fn"] == "1" # a user's own entries survive
+ priority = tuple(int(b) for b in torch._C._get_sdp_priority_order())
+ assert f"sdp_priority={priority}" in marked[HELPERS[0]]
+
+
+def test_sdpa_kernel_graph_hits_the_aot_cache_after_a_restart(
+ _restore_cacheable, monkeypatch, tmp_path
+):
+ # A dynamo reset is a fresh process for dynamo; the second compile must be served by AOTAutogradCache, not bypass it.
+ from torch._dynamo.utils import counters
+ from core.inference import diffusion_aot_cache
+
+ monkeypatch.setenv("TORCHINDUCTOR_CACHE_DIR", str(tmp_path))
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", raising = False)
+ torch._inductor.config.unsafe_marked_cacheable_functions = {}
+ assert diffusion_aot_cache.install() is True
+ compile_config.apply()
+ block = _Block().eval()
+ x = torch.randn(1, 4, 16)
+ with torch.no_grad():
+ ref = block(x)
+ counters.clear()
+ first = torch.compile(block, backend = "inductor")(x)
+ assert counters["aot_autograd"]["autograd_cache_bypass"] == 0
+ assert counters["aot_autograd"]["autograd_cache_miss"] >= 1
+ torch._dynamo.reset()
+ counters.clear()
+ second = torch.compile(block, backend = "inductor")(x)
+ assert counters["aot_autograd"]["autograd_cache_hit"] >= 1
+ assert counters["aot_autograd"]["autograd_cache_bypass"] == 0
+ # The cache hit must serve the artifact the first compile built.
+ assert torch.equal(second, first)
+ torch.testing.assert_close(first, ref)