Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: let the compile cache keep denoiser graphs with sdpa_kernel blocks (restart first render 1-5.5 s faster) (#12510)
Daniel Han2eaf5b4ccff4
diff --git a/studio/backend/core/inference/diffusion_aot_cache.py b/studio/backend/core/inference/diffusion_aot_cache.pynew file mode 100644index 00000000000..a414d2ee4ff--- /dev/null+++ b/studio/backend/core/inference/diffusion_aot_cache.py@@ -0,0 +1,72 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Let AOTAutogradCache store the denoiser graphs that hold an ``sdpa_kernel`` block.++Dynamo bakes ``sdpa_kernel`` into the graph as ``_backend_from_string`` / ``_sdpa_kernel`` calls, which are not on+AOTAutogradCache's allowlist, so every regional block of FLUX.1, Z-Image and Qwen-Image re-ran AOT tracing on each+process start (pytorch/pytorch#194007; 2.6 / 4.7 / 1.8 s of a B200 restart's first render). Their arguments are graph+constants, so the graph keys them apart; the one ambient input, SDPA's backend priority order, goes into the marked+value torch folds into the cache key. UNSLOTH_DIFFUSION_SDPA_AOT_CACHE=0 disables it.+"""++from __future__ import annotations++import os+from typing import Any++from . import diffusion_compile_config as compile_config++_ENV = "UNSLOTH_DIFFUSION_SDPA_AOT_CACHE"+_INDUCTOR_MODULE = "torch._inductor.config"+_KNOB = "unsafe_marked_cacheable_functions"+# The names check_node_safe builds: f"{target.__module__}.{target.__name__}".+SDPA_HELPERS = ("torch.nn.attention._backend_from_string", "torch.nn.attention._sdpa_kernel")+++def enabled() -> bool:+ raw = (os.environ.get(_ENV) or "").strip().lower()+ return raw not in ("0", "false", "no", "off")+++def _cache_key_value(torch: Any) -> str:+ try:+ priority = tuple(int(b) for b in torch._C._get_sdp_priority_order())+ except Exception: # noqa: BLE001 - no priority API on this build: key on the version alone+ priority = None+ return f"{torch.__version__}|sdp_priority={priority}"+++def install(logger: Any = None) -> bool:+ """Mark the two SDPA helpers cacheable (idempotent, process-wide). False when disabled or unsupported."""+ if not enabled():+ return False+ try:+ import torch+ import torch.nn.attention as attention++ if not all(+ callable(getattr(attention, name.rsplit(".", 1)[1], None)) for name in SDPA_HELPERS+ ):+ return False+ # Both the recorded knob (re-applied on render threads) and this thread's live value.+ recorded = compile_config.get_knob(_INDUCTOR_MODULE, _KNOB)+ live = getattr(getattr(torch._inductor, "config", None), _KNOB, None)+ if not isinstance(recorded, dict) or not isinstance(live, dict):+ return False+ value = _cache_key_value(torch)+ if all(recorded.get(n) == value and live.get(n) == value for n in SDPA_HELPERS):+ return True+ merged = {**recorded, **live}+ merged.update({name: value for name in SDPA_HELPERS})+ if not compile_config.set_knob(_INDUCTOR_MODULE, _KNOB, merged):+ return False+ if logger is not None:+ logger.info(+ "diffusion.aot_cache: sdpa_kernel graphs are AOTAutograd-cacheable (%s)", value+ )+ return True+ except Exception as exc: # noqa: BLE001 - optimisation only+ if logger is not None:+ logger.debug("diffusion.aot_cache: not installed (%s: %s)", type(exc).__name__, exc)+ return Falsediff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 977b487c82e..92572cc75f6 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -886,6 +886,12 @@ def _compile_repeated_blocks(except Exception as exc: # noqa: BLE001 - optimisation only_warn(logger, "compile_repeated_blocks", exc)return False+ # A block whose attention runs inside sdpa_kernel would otherwise bypass AOTAutogradCache on every start.+ try:+ from . import diffusion_aot_cache+ diffusion_aot_cache.install(logger)+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "sdpa aot cache", exc)if unet is not None:# Whole-module static compile for the U-Net classes above. fullgraph mirrors the regional decision; dynamic is# ALWAYS False, so each new (height, width, batch) pays its own compile. ``Module.compile`` keeps the modulediff --git a/studio/backend/tests/test_diffusion_sdpa_aot_cache.py b/studio/backend/tests/test_diffusion_sdpa_aot_cache.pynew file mode 100644index 00000000000..ff66451619c--- /dev/null+++ b/studio/backend/tests/test_diffusion_sdpa_aot_cache.py@@ -0,0 +1,128 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""AOTAutogradCache for denoiser graphs that hold an ``sdpa_kernel`` block.++CPU only, checked with a real inductor compile (an AOTAutograd cache hit after a dynamo reset).+"""++from __future__ import annotations++import types++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_compile_config as compile_config # noqa: E402+from core.inference import diffusion_speed as speed # noqa: E402++HELPERS = ("torch.nn.attention._backend_from_string", "torch.nn.attention._sdpa_kernel")++[email protected](autouse = True)+def _reset_dynamo():+ torch._dynamo.reset()+ yield+ torch._dynamo.reset()+++def _restore_cacheable():+ cfg = torch._inductor.config+ before = dict(cfg.unsafe_marked_cacheable_functions)+ recorded = compile_config.recorded()+ yield+ cfg.unsafe_marked_cacheable_functions = before+ compile_config._reset_for_tests()+ for (module_name, attr), value in recorded.items():+ compile_config.set_knob(module_name, attr, value)+++class _Block(torch.nn.Module):+ def __init__(self) -> None:+ super().__init__()+ self.lin = torch.nn.Linear(16, 16)++ def forward(self, x):+ from torch.nn.attention import SDPBackend, sdpa_kernel++ q = self.lin(x)+ with sdpa_kernel(SDPBackend.MATH):+ out = torch.nn.functional.scaled_dot_product_attention(q, q, q)+ return out + x+++class _Dit(torch.nn.Module):+ _repeated_blocks = ["_Block"]++ def __init__(self) -> None:+ super().__init__()+ self.blocks = torch.nn.ModuleList([_Block(), _Block()])+ self.compiled_kwargs = None++ def compile_repeated_blocks(self, **kwargs):+ self.compiled_kwargs = kwargs+++def test_regional_compile_marks_the_sdpa_helpers_cacheable(_restore_cacheable, monkeypatch):+ # Every regional compile goes through _compile_repeated_blocks; that is where the helpers must become cacheable.+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", raising = False)+ torch._inductor.config.unsafe_marked_cacheable_functions = {}+ pipe = types.SimpleNamespace(transformer = _Dit())+ speed._compile_repeated_blocks(pipe, None)+ marked = torch._inductor.config.unsafe_marked_cacheable_functions+ for name in HELPERS:+ assert name in marked+ assert torch.__version__ in marked[name]+++def test_sdpa_aot_cache_kill_switch(_restore_cacheable, monkeypatch):+ from core.inference import diffusion_aot_cache++ monkeypatch.setenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", "0")+ torch._inductor.config.unsafe_marked_cacheable_functions = {}+ assert diffusion_aot_cache.install() is False+ assert torch._inductor.config.unsafe_marked_cacheable_functions == {}+++def test_sdpa_aot_cache_key_carries_the_priority_order(_restore_cacheable, monkeypatch):+ from core.inference import diffusion_aot_cache++ monkeypatch.delenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", raising = False)+ torch._inductor.config.unsafe_marked_cacheable_functions = {"user.fn": "1"}+ assert diffusion_aot_cache.install() is True+ marked = dict(torch._inductor.config.unsafe_marked_cacheable_functions)+ assert marked["user.fn"] == "1" # a user's own entries survive+ priority = tuple(int(b) for b in torch._C._get_sdp_priority_order())+ assert f"sdp_priority={priority}" in marked[HELPERS[0]]+++def test_sdpa_kernel_graph_hits_the_aot_cache_after_a_restart(+ _restore_cacheable, monkeypatch, tmp_path+):+ # A dynamo reset is a fresh process for dynamo; the second compile must be served by AOTAutogradCache, not bypass it.+ from torch._dynamo.utils import counters+ from core.inference import diffusion_aot_cache++ monkeypatch.setenv("TORCHINDUCTOR_CACHE_DIR", str(tmp_path))+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", raising = False)+ torch._inductor.config.unsafe_marked_cacheable_functions = {}+ assert diffusion_aot_cache.install() is True+ compile_config.apply()+ block = _Block().eval()+ x = torch.randn(1, 4, 16)+ with torch.no_grad():+ ref = block(x)+ counters.clear()+ first = torch.compile(block, backend = "inductor")(x)+ assert counters["aot_autograd"]["autograd_cache_bypass"] == 0+ assert counters["aot_autograd"]["autograd_cache_miss"] >= 1+ torch._dynamo.reset()+ counters.clear()+ second = torch.compile(block, backend = "inductor")(x)+ assert counters["aot_autograd"]["autograd_cache_hit"] >= 1+ assert counters["aot_autograd"]["autograd_cache_bypass"] == 0+ # The cache hit must serve the artifact the first compile built.+ assert torch.equal(second, first)+ torch.testing.assert_close(first, ref)