Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""AOTAutograd cache hit for sdpa_kernel denoiser graphs, via a real CPU inductor compile."""
from __future__ import annotations
import types
import pytest
torch = pytest.importorskip("torch")
from core.inference import diffusion_compile_config as compile_config # noqa: E402
from core.inference import diffusion_speed as speed # noqa: E402
HELPERS = ("torch.nn.attention._backend_from_string", "torch.nn.attention._sdpa_kernel")
@pytest.fixture(autouse = True)
def _reset_dynamo():
torch._dynamo.reset()
yield
torch._dynamo.reset()
@pytest.fixture
def _restore_cacheable():
cfg = torch._inductor.config
before = dict(cfg.unsafe_marked_cacheable_functions)
recorded = compile_config.recorded()
yield
cfg.unsafe_marked_cacheable_functions = before
compile_config._reset_for_tests()
for (module_name, attr), value in recorded.items():
compile_config.set_knob(module_name, attr, value)
class _Block(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.lin = torch.nn.Linear(16, 16)
def forward(self, x):
from torch.nn.attention import SDPBackend, sdpa_kernel
q = self.lin(x)
with sdpa_kernel(SDPBackend.MATH):
out = torch.nn.functional.scaled_dot_product_attention(q, q, q)
return out + x
class _Dit(torch.nn.Module):
_repeated_blocks = ["_Block"]
def __init__(self) -> None:
super().__init__()
self.blocks = torch.nn.ModuleList([_Block(), _Block()])
self.compiled_kwargs = None
def compile_repeated_blocks(self, **kwargs):
self.compiled_kwargs = kwargs
def test_regional_compile_marks_the_sdpa_helpers_cacheable(_restore_cacheable, monkeypatch):
# every regional compile goes through _compile_repeated_blocks, where helpers become cacheable
monkeypatch.delenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", raising = False)
torch._inductor.config.unsafe_marked_cacheable_functions = {}
pipe = types.SimpleNamespace(transformer = _Dit())
speed._compile_repeated_blocks(pipe, None)
marked = torch._inductor.config.unsafe_marked_cacheable_functions
for name in HELPERS:
assert name in marked
assert torch.__version__ in marked[name]
def test_sdpa_aot_cache_kill_switch(_restore_cacheable, monkeypatch):
from core.inference import diffusion_aot_cache
monkeypatch.setenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", "0")
torch._inductor.config.unsafe_marked_cacheable_functions = {}
assert diffusion_aot_cache.install() is False
assert torch._inductor.config.unsafe_marked_cacheable_functions == {}
def test_sdpa_aot_cache_key_carries_the_priority_order(_restore_cacheable, monkeypatch):
from core.inference import diffusion_aot_cache
monkeypatch.delenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", raising = False)
torch._inductor.config.unsafe_marked_cacheable_functions = {"user.fn": "1"}
assert diffusion_aot_cache.install() is True
marked = dict(torch._inductor.config.unsafe_marked_cacheable_functions)
assert marked["user.fn"] == "1"
priority = tuple(int(b) for b in torch._C._get_sdp_priority_order())
assert f"sdp_priority={priority}" in marked[HELPERS[0]]
def test_sdpa_kernel_graph_hits_the_aot_cache_after_a_restart(
_restore_cacheable, monkeypatch, tmp_path
):
# after a dynamo reset the second compile must be served by AOTAutogradCache
from torch._dynamo.utils import counters
from core.inference import diffusion_aot_cache
monkeypatch.setenv("TORCHINDUCTOR_CACHE_DIR", str(tmp_path))
monkeypatch.delenv("UNSLOTH_DIFFUSION_SDPA_AOT_CACHE", raising = False)
torch._inductor.config.unsafe_marked_cacheable_functions = {}
assert diffusion_aot_cache.install() is True
compile_config.apply()
block = _Block().eval()
x = torch.randn(1, 4, 16)
with torch.no_grad():
ref = block(x)
counters.clear()
first = torch.compile(block, backend = "inductor")(x)
assert counters["aot_autograd"]["autograd_cache_bypass"] == 0
assert counters["aot_autograd"]["autograd_cache_miss"] >= 1
torch._dynamo.reset()
counters.clear()
second = torch.compile(block, backend = "inductor")(x)
assert counters["aot_autograd"]["autograd_cache_hit"] >= 1
assert counters["aot_autograd"]["autograd_cache_bypass"] == 0
assert torch.equal(second, first)
torch.testing.assert_close(first, ref)