Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Query-chunked VAE attention where no fused SDPA takes the head dim; the score tile is bounded."""
import threading
import types
import pytest
torch = pytest.importorskip("torch")
from torch.torch_version import TorchVersion # noqa: E402
F = torch.nn.functional
attention = pytest.importorskip("torch.nn.attention")
from torch.utils._python_dispatch import TorchDispatchMode # noqa: E402
from core.inference import diffusion_vae_attn_chunked as C # noqa: E402
# CUDA version tag: Studio also reads ROCm from "+rocm" (AMD wheels leave torch.version.hip unset).
_CUDA_TORCH_VERSION = TorchVersion(str(torch.__version__).split("+")[0] + "+cu130")
def _math():
return attention.sdpa_kernel(attention.SDPBackend.MATH)
class _LargestTensor(TorchDispatchMode):
def __init__(self):
super().__init__()
self.max_bytes = 0
def __torch_dispatch__(
self,
func,
types_,
args = (),
kwargs = None,
):
out = func(*args, **(kwargs or {}))
for t in out if isinstance(out, (tuple, list)) else (out,):
if isinstance(t, torch.Tensor):
self.max_bytes = max(self.max_bytes, t.numel() * t.element_size())
return out
@pytest.fixture
def rocm(monkeypatch):
"""Pretend to be ROCm with a math-only SDPA: the fused-kernel query says no for every call."""
monkeypatch.setattr(torch.version, "hip", "7.2.0", raising = False)
monkeypatch.setattr(C, "fused_sdpa_available", lambda q, k, v: False)
monkeypatch.delenv(C.VAE_ATTN_CHUNK_ENV, raising = False)
return types.SimpleNamespace(backend = "rocm", device = "cuda")
@pytest.mark.parametrize("dtype,tol", [(torch.float32, 1e-5), (torch.bfloat16, 2e-2)])
@pytest.mark.parametrize("shape", [(1, 1, 1000, 64), (2, 300, 32), (1, 2, 517, 16)])
@pytest.mark.parametrize("budget_rows", [1, 64, 333, 10_000])
def test_chunked_equals_full_softmax(dtype, tol, shape, budget_rows):
g = torch.Generator().manual_seed(0)
q, k, v = (torch.randn(*shape, generator = g).to(dtype) for _ in range(3))
rows = 1
for s in shape[:-2]:
rows *= s
budget = budget_rows * rows * shape[-2] * 4
with _math():
ref = F.scaled_dot_product_attention(q, k, v)
out = C.chunked_sdpa(F.scaled_dot_product_attention, q, k, v, budget)
ref_s = F.scaled_dot_product_attention(q, k, v, scale = 0.3)
out_s = C.chunked_sdpa(F.scaled_dot_product_attention, q, k, v, budget, scale = 0.3)
assert out.dtype == ref.dtype and out.shape == ref.shape
torch.testing.assert_close(out, ref, atol = tol, rtol = tol)
torch.testing.assert_close(out_s, ref_s, atol = tol, rtol = tol)
def test_mode_bounds_the_score_tile(rocm):
q = torch.randn(1, 1, 2048, 64)
full = 2048 * 2048 * 4
budget = full // 16
with _math():
probe = _LargestTensor()
with probe:
ref = F.scaled_dot_product_attention(q, q, q)
assert probe.max_bytes >= full
mode = C.chunked_attention_mode(budget)
probe = _LargestTensor()
with mode, probe:
out = F.scaled_dot_product_attention(q, q, q)
assert mode.chunked == 1
assert probe.max_bytes <= budget
torch.testing.assert_close(out, ref, atol = 1e-5, rtol = 1e-5)
def test_mode_chunks_keyword_calls(rocm):
q = torch.randn(1, 1, 512, 32)
with _math():
ref = F.scaled_dot_product_attention(q, q, q, scale = 0.2)
mode = C.chunked_attention_mode(512 * 512 * 4 // 8)
with mode:
out = F.scaled_dot_product_attention(
query = q,
key = q,
value = q,
attn_mask = None,
dropout_p = 0.0,
is_causal = False,
scale = 0.2,
enable_gqa = False,
)
mixed = F.scaled_dot_product_attention(q, key = q, value = q, scale = 0.2)
assert mode.chunked == 2
torch.testing.assert_close(out, ref, atol = 1e-5, rtol = 1e-5)
torch.testing.assert_close(mixed, ref, atol = 1e-5, rtol = 1e-5)
def test_mode_leaves_fused_small_masked_and_causal_calls_alone(monkeypatch):
calls = []
real = C.chunked_sdpa
monkeypatch.setattr(C, "chunked_sdpa", lambda *a, **k: calls.append(1) or real(*a, **k))
q = torch.randn(1, 1, 256, 16)
big = 1024
mask = torch.ones(256, 256, dtype = torch.bool)
with _math():
ref = F.scaled_dot_product_attention(q, q, q)
# fused kernel available (CPU, and NVIDIA at head dim 512): the stock call, bit-identical
with C.chunked_attention_mode(big) as mode:
assert torch.equal(F.scaled_dot_product_attention(q, q, q), ref)
assert mode.chunked == 0
monkeypatch.setattr(C, "fused_sdpa_available", lambda q, k, v: False)
with C.chunked_attention_mode(big) as mode:
F.scaled_dot_product_attention(q, q, q, attn_mask = mask)
F.scaled_dot_product_attention(q, q, q, mask)
F.scaled_dot_product_attention(q, q, q, is_causal = True)
F.scaled_dot_product_attention(q, q, q, dropout_p = 0.1)
with C.chunked_attention_mode(256 * 256 * 4) as small:
assert torch.equal(F.scaled_dot_product_attention(q, q, q), ref)
assert mode.chunked == 0 and small.chunked == 0 and calls == []
def test_fused_query_is_stock_off_cuda():
q = torch.randn(1, 1, 8, 512)
assert C.fused_sdpa_available(q, q, q) is True
@pytest.mark.parametrize("head_dim, fused", [(512, False), (384, False), (128, True)])
def test_fused_query_on_mps_asks_the_prefill_kernel(monkeypatch, head_dim, fused):
# No MPS fused kernel takes the VAE's head dim 384 / 512, whatever the torch build.
monkeypatch.setattr(torch, "__version__", "2.14.1")
monkeypatch.delenv("PYTORCH_MPS_DISABLE_PREFILL_ATTENTION", raising = False)
q = types.SimpleNamespace(
device = types.SimpleNamespace(type = "mps"),
dtype = torch.bfloat16,
shape = (1, 1, 16384, head_dim),
requires_grad = False,
)
assert C.fused_sdpa_available(q, q, q) is fused
def _tiny_vae(name):
diffusers = pytest.importorskip("diffusers")
torch.manual_seed(0)
if name == "AutoencoderKL":
return diffusers.AutoencoderKL(
in_channels = 3,
out_channels = 3,
down_block_types = ("DownEncoderBlock2D", "DownEncoderBlock2D"),
up_block_types = ("UpDecoderBlock2D", "UpDecoderBlock2D"),
block_out_channels = (32, 64),
layers_per_block = 1,
latent_channels = 4,
norm_num_groups = 8,
).eval()
if name == "AutoencoderKLWan":
return diffusers.AutoencoderKLWan(
base_dim = 16, z_dim = 4, dim_mult = [1, 2], num_res_blocks = 1, temperal_downsample = [False]
).eval()
if name == "AutoencoderKLQwenImage":
return diffusers.AutoencoderKLQwenImage(
base_dim = 16, z_dim = 4, dim_mult = [1, 2], num_res_blocks = 1, temperal_downsample = [False]
).eval()
raise AssertionError(name)
@pytest.mark.parametrize("name", ["AutoencoderKL", "AutoencoderKLWan", "AutoencoderKLQwenImage"])
def test_vae_decode_peak_bounded_and_equal(rocm, monkeypatch, name):
vae = _tiny_vae(name)
if name == "AutoencoderKL":
z = torch.randn(1, 4, 48, 48)
else:
z = torch.randn(1, 4, 1, 48, 48)
tokens = 48 * 48
full = tokens * tokens * 4
with torch.no_grad(), _math():
probe = _LargestTensor()
with probe:
ref = vae.decode(z).sample
assert probe.max_bytes >= full
monkeypatch.setenv(
C.VAE_ATTN_CHUNK_MB_ENV, "1"
) # 1 MiB of scores per chunk, well under the 21 MiB matrix
assert C.install(vae, rocm) >= 1
probe = _LargestTensor()
with probe:
out = vae.decode(z).sample
assert probe.max_bytes < full // 4
torch.testing.assert_close(out, ref, atol = 1e-4, rtol = 1e-4)
@pytest.mark.parametrize(
"backend, env, budget_mb", [("rocm", None, 1024), ("mps", None, 256), ("mps", "64", 64)]
)
def test_install_budget_follows_the_device(rocm, monkeypatch, backend, env, budget_mb):
# MPSGraph holds several score-sized buffers, so a 1 GiB FLUX.1 tile is chunked there but not on ROCm.
if env is not None:
monkeypatch.setenv(C.VAE_ATTN_CHUNK_MB_ENV, env)
else:
monkeypatch.delenv(C.VAE_ATTN_CHUNK_MB_ENV, raising = False)
monkeypatch.setattr(torch.version, "hip", None if backend == "mps" else "7.2.0", raising = False)
vae = _tiny_vae("AutoencoderKL")
assert C.install(vae, types.SimpleNamespace(backend = backend, device = backend)) == 2
budgets = []
real = C.chunked_attention_mode
monkeypatch.setattr(
C, "chunked_attention_mode", lambda budget = None: budgets.append(budget) or real(budget)
)
with torch.no_grad():
vae.decode(torch.randn(1, 4, 8, 8))
assert budgets == [budget_mb * 2**20]
def test_install_gates(monkeypatch, rocm):
vae = _tiny_vae("AutoencoderKL")
assert C.install(vae, types.SimpleNamespace(backend = "cuda", device = "cuda")) == 0
monkeypatch.setenv(C.VAE_ATTN_CHUNK_ENV, "0")
assert C.install(vae, rocm) == 0
monkeypatch.delenv(C.VAE_ATTN_CHUNK_ENV)
monkeypatch.setattr(torch.version, "hip", None, raising = False)
monkeypatch.setattr(torch, "__version__", _CUDA_TORCH_VERSION, raising = False)
assert C.install(vae, rocm) == 0
assert not any(getattr(m, C._PATCHED, False) for m in vae.modules())
assert C.install(None, rocm) == 0
def test_install_patches_encoder_and_decoder_once(rocm):
vae = _tiny_vae("AutoencoderKL")
assert C.install(vae, rocm) == 2
assert C.install(vae, rocm) == 0
# a processor reset (fuse_qkv_projections) keeps the module-level patch
for m in vae.modules():
if type(m).__name__ == "Attention":
m.set_processor(type(m.processor)())
assert m.forward._unsloth_vae_attn_chunked
def test_kill_switch_at_call_time(rocm, monkeypatch):
vae = _tiny_vae("AutoencoderKL")
monkeypatch.setenv(C.VAE_ATTN_CHUNK_MB_ENV, "1")
C.install(vae, rocm)
monkeypatch.setenv(C.VAE_ATTN_CHUNK_ENV, "off")
z = torch.randn(1, 4, 48, 48)
with torch.no_grad(), _math():
probe = _LargestTensor()
with probe:
vae.decode(z)
assert probe.max_bytes >= 48 * 48 * 48 * 48 * 4
def test_speed_layer_installs_on_rocm_and_mps_even_when_off(rocm, monkeypatch):
from core.inference import diffusion_speed as S
vae = _tiny_vae("AutoencoderKL")
pipe = types.SimpleNamespace(vae = vae)
S.apply_speed_optims(
pipe,
types.SimpleNamespace(backend = "rocm", device = "cuda", dtype = torch.bfloat16),
is_gguf = False,
family = types.SimpleNamespace(),
speed_mode = S.SPEED_OFF,
)
assert sum(getattr(m, C._PATCHED, False) for m in vae.modules()) == 2
mac = _tiny_vae("AutoencoderKL")
S.apply_speed_optims(
types.SimpleNamespace(vae = mac),
types.SimpleNamespace(backend = "mps", device = "mps", dtype = torch.bfloat16),
is_gguf = False,
family = types.SimpleNamespace(),
speed_mode = S.SPEED_OFF,
)
assert sum(getattr(m, C._PATCHED, False) for m in mac.modules()) == 2
nvidia = _tiny_vae("AutoencoderKL")
S.apply_speed_optims(
types.SimpleNamespace(vae = nvidia),
types.SimpleNamespace(backend = "cuda", device = "cuda", dtype = torch.bfloat16),
is_gguf = False,
family = types.SimpleNamespace(),
speed_mode = S.SPEED_OFF,
)
assert not any(getattr(m, C._PATCHED, False) for m in nvidia.modules())
_HALF_FLAG = getattr(torch.backends.cuda, "fp16_bf16_reduction_math_sdp_allowed", None)
@pytest.mark.skipif(_HALF_FLAG is None, reason = "torch < 2.5 has no fp32 upcast to skip")
@pytest.mark.parametrize("chunk", [False, True])
def test_half_math_keeps_dtype_scoped_and_close(rocm, monkeypatch, chunk):
monkeypatch.setattr(C, "_half_math_call", lambda q: q.dtype == torch.bfloat16)
g = torch.Generator().manual_seed(0)
q, k, v = (torch.randn(1, 1, 1024, 512, generator = g).to(torch.bfloat16) for _ in range(3))
ref32 = F.scaled_dot_product_attention(q.float(), k.float(), v.float())
before = _HALF_FLAG()
with _math():
upcast = F.scaled_dot_product_attention(q, k, v)
torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(True)
try:
want = F.scaled_dot_product_attention(q, k, v)
finally:
torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(before)
mode = C.chunked_attention_mode(1024 * 1024 * 4 // 4 if chunk else 1 << 40)
with mode:
out = F.scaled_dot_product_attention(q, k, v)
assert _HALF_FLAG() == before
assert mode.half == 1 and mode.chunked == int(chunk)
if not chunk:
assert torch.equal(out, want)
assert not torch.equal(out, upcast)
torch.testing.assert_close(out.float(), ref32, atol = 2e-2, rtol = 2e-2)
@pytest.mark.skipif(_HALF_FLAG is None, reason = "torch < 2.5")
@pytest.mark.parametrize("scale", [None, 0.3, -0.3])
def test_half_math_sdpa_matches_the_flag(scale):
g = torch.Generator().manual_seed(1)
q, k, v = (torch.randn(2, 3, 200, 64, generator = g).to(torch.bfloat16) for _ in range(3))
before = _HALF_FLAG()
with _math():
torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(True)
try:
want = F.scaled_dot_product_attention(q, k, v, scale = scale)
finally:
torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(before)
assert torch.equal(C.half_math_sdpa(q, k, v, scale), want)
class _BlockFirstOp(TorchDispatchMode):
"""Parks this thread at its first aten op until released (dispatch modes are per thread)."""
def __init__(self):
super().__init__()
self.inside = threading.Event()
self.release = threading.Event()
def __torch_dispatch__(
self,
func,
types_,
args = (),
kwargs = None,
):
if not self.inside.is_set():
self.inside.set()
assert self.release.wait(10)
return func(*args, **(kwargs or {}))
@pytest.mark.skipif(_HALF_FLAG is None, reason = "torch < 2.5")
def test_half_math_leaves_other_threads_and_the_flag_alone(rocm, monkeypatch):
monkeypatch.setattr(C, "_half_math_call", lambda q: q.dtype == torch.bfloat16)
g = torch.Generator().manual_seed(0)
q, k, v = (torch.randn(1, 1, 64, 64, generator = g).to(torch.bfloat16) for _ in range(3))
other = torch.randn(1, 1, 32, 64, generator = g).to(torch.bfloat16)
def unrelated():
with _math():
return F.scaled_dot_product_attention(other, other, other)
before = _HALF_FLAG()
reference = unrelated()
modes, blocks, errors = [], [], []
def vae_call(mode, block):
try:
with mode, block:
F.scaled_dot_product_attention(q, k, v)
except BaseException as exc: # noqa: BLE001
errors.append(exc)
finally:
block.inside.set()
threads = []
try:
for _ in range(2): # two overlapping VAE calls, as image and video decodes can be
modes.append(C.chunked_attention_mode(1 << 40))
blocks.append(_BlockFirstOp())
threads.append(threading.Thread(target = vae_call, args = (modes[-1], blocks[-1])))
threads[-1].start()
assert blocks[-1].inside.wait(10)
assert _HALF_FLAG() == before
assert torch.equal(unrelated(), reference)
finally:
for block, thread in zip(blocks, threads):
block.release.set()
thread.join(10)
assert not errors and not any(t.is_alive() for t in threads)
assert [m.half for m in modes] == [1, 1]
assert _HALF_FLAG() == before
assert torch.equal(unrelated(), reference)
def test_half_math_runs_in_the_autocast_dtype(rocm, monkeypatch):
monkeypatch.setattr(C, "_half_math_call", lambda q: q.dtype in (torch.bfloat16, torch.float16))
g = torch.Generator().manual_seed(0)
q, k, v = (torch.randn(1, 1, 128, 64, generator = g).to(torch.bfloat16) for _ in range(3))
want = C.half_math_sdpa(q, k, v)
mode = C.chunked_attention_mode(1 << 40)
with torch.autocast("cpu", dtype = torch.bfloat16), mode:
out = F.scaled_dot_product_attention(q, k.float(), v.float())
assert mode.half == 1 and out.dtype == torch.bfloat16
assert torch.equal(out, want)
def test_half_math_gates(monkeypatch):
cuda_bf16 = types.SimpleNamespace(
device = types.SimpleNamespace(type = "cuda"), dtype = torch.bfloat16
)
cuda_fp32 = types.SimpleNamespace(
device = types.SimpleNamespace(type = "cuda"), dtype = torch.float32
)
cpu_bf16 = types.SimpleNamespace(device = types.SimpleNamespace(type = "cpu"), dtype = torch.bfloat16)
monkeypatch.setattr(torch.version, "hip", "7.2.0", raising = False)
monkeypatch.delenv(C.VAE_ATTN_HALF_MATH_ENV, raising = False)
assert C._half_math_call(cuda_bf16)
assert not C._half_math_call(cuda_fp32)
assert not C._half_math_call(cpu_bf16)
monkeypatch.setenv(C.VAE_ATTN_HALF_MATH_ENV, "0")
assert not C._half_math_call(cuda_bf16)
monkeypatch.delenv(C.VAE_ATTN_HALF_MATH_ENV)
monkeypatch.setattr(torch.version, "hip", None, raising = False)
monkeypatch.setattr(torch, "__version__", _CUDA_TORCH_VERSION)
assert not C._half_math_call(cuda_bf16)