Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Tests for ``diffusion_rocm_fused.py``: the switches (auto = RoPE on ROCm and on NVIDIA CUDA under Linux, AdaLN
opt-in), install / uninstall wiring, and on a GPU the kernels against diffusers (RoPE bit-identical, AdaLN within one
rounding of stock) plus tiny FLUX.1 / FLUX.2 transformers."""
from __future__ import annotations
import ast
from pathlib import Path
import pytest
torch = pytest.importorskip("torch")
from torch.torch_version import TorchVersion # noqa: E402
from core.inference import diffusion_eager_patches as ep # noqa: E402
from core.inference import diffusion_rocm_fused as rf # noqa: E402
fmod = pytest.importorskip("diffusers.models.transformers.transformer_flux")
from diffusers.models import normalization as nm # noqa: E402
from diffusers.models.embeddings import apply_rotary_emb as stock_rope # noqa: E402
# A CUDA build's version tag: Studio reads ROCm from the "+rocm" tag too (AMD SDK / Radeon wheels leave
# torch.version.hip unset), so a test that fakes NVIDIA clears both.
_CUDA_TORCH_VERSION = TorchVersion(str(torch.__version__).split("+")[0] + "+cu130")
# another test file may leave Studio's eager AdaLN patch on the classes: start from stock
ep.uninstall_patches()
_STOCK_FWD = {name: getattr(nm, name).forward for name in rf._ADALN_CLASSES}
needs_gpu = pytest.mark.skipif(
not torch.cuda.is_available() or rf._rope_kernel() is None or rf._adaln_kernel() is None,
reason = "needs a CUDA / ROCm GPU and Triton",
)
class _PipeLike:
def __init__(self, module = "diffusers.models.transformers.transformer_flux"):
self.transformer = type("FluxTransformer2DModel", (), {"__module__": module})()
@pytest.fixture(autouse = True)
def _restore(monkeypatch):
monkeypatch.delenv(rf.FUSED_ROPE_ENV, raising = False)
monkeypatch.delenv(rf.FUSED_ADALN_ENV, raising = False)
rf.uninstall()
ep.uninstall_patches()
yield
rf.uninstall()
assert fmod.apply_rotary_emb is stock_rope
for name, fwd in _STOCK_FWD.items():
assert getattr(nm, name).forward is fwd
@pytest.fixture
def fake_kernels(monkeypatch):
monkeypatch.setattr(rf, "_rope_kernel", lambda: (lambda x, cos, sin: None))
monkeypatch.setattr(rf, "_adaln_kernel", lambda: (lambda x, scale, shift, eps: None))
def _nvidia(monkeypatch, platform = "linux"):
monkeypatch.setattr(torch.version, "hip", None, raising = False)
monkeypatch.setattr(torch, "__version__", _CUDA_TORCH_VERSION, raising = False)
monkeypatch.setattr(torch.version, "cuda", "13.0", raising = False)
monkeypatch.setattr(rf.sys, "platform", platform)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_auto_engages_rope_and_adaln_on_nvidia_linux(fake_kernels, monkeypatch, dtype):
_nvidia(monkeypatch)
assert rf.install_for_pipe(_PipeLike(), dtype, "cuda") == {"rope": True, "adaln": 3}
assert fmod.apply_rotary_emb is rf._ROPE_FNS[rf._MODULES[0]]
assert nm.AdaLayerNormZero.forward is rf._adaln_zero_forward
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_the_exact_tier_keeps_stock_adaln_on_nvidia(fake_kernels, monkeypatch, dtype):
_nvidia(monkeypatch)
rf.install_for_pipe(_PipeLike(), dtype, "cuda")
assert rf.install_for_pipe(_PipeLike(), dtype, "cuda", adaln = False) == {
"rope": True,
"adaln": 0,
}
assert fmod.apply_rotary_emb is rf._ROPE_FNS[rf._MODULES[0]]
assert nm.AdaLayerNormZero.forward is _STOCK_FWD["AdaLayerNormZero"]
def test_the_adaln_kill_switch_on_nvidia(fake_kernels, monkeypatch):
_nvidia(monkeypatch)
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "0")
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": True, "adaln": 0}
assert nm.AdaLayerNormZero.forward is _STOCK_FWD["AdaLayerNormZero"]
@pytest.mark.parametrize("platform", ["win32", "darwin"])
def test_auto_is_inert_on_nvidia_outside_linux(fake_kernels, monkeypatch, platform):
# Windows keeps stock: a Triton JIT failure there would be retried on every call.
_nvidia(monkeypatch, platform)
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}
assert not rf.is_installed()
def test_auto_is_inert_without_a_gpu_torch(fake_kernels, monkeypatch):
monkeypatch.setattr(torch.version, "hip", None, raising = False)
monkeypatch.setattr(torch, "__version__", _CUDA_TORCH_VERSION, raising = False)
monkeypatch.setattr(torch.version, "cuda", None, raising = False)
monkeypatch.setattr(rf.sys, "platform", "linux")
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}
assert not rf.is_installed()
@pytest.mark.parametrize("value", ["0", "off", "false"])
def test_kill_switch_on_nvidia(fake_kernels, monkeypatch, value):
_nvidia(monkeypatch)
monkeypatch.setenv(rf.FUSED_ROPE_ENV, value)
monkeypatch.setenv(rf.FUSED_ADALN_ENV, value)
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}
assert fmod.apply_rotary_emb is stock_rope
def test_flux2_fp16_keeps_its_own_patch_on_nvidia(fake_kernels, monkeypatch):
from core.inference import diffusion_flux2_rope as fr
f2mod = pytest.importorskip("diffusers.models.transformers.transformer_flux2")
_nvidia(monkeypatch)
monkeypatch.setattr(fr, "_kernel", lambda: (lambda *a, **k: None))
pipe = _PipeLike("diffusers.models.transformers.transformer_flux2")
try:
assert fr.install_for_pipe(pipe, torch.float16, "cuda")
assert rf.install_for_pipe(pipe, torch.float16, "cuda") == {"rope": False, "adaln": 3}
assert f2mod.apply_rotary_emb is fr._fused_apply_rotary_emb
finally:
fr.uninstall()
def test_auto_engages_rope_only_on_rocm(fake_kernels, monkeypatch):
# ROCm auto = bit-identical RoPE kernel; fused AdaLN gave no speed on gfx1151, so it is opt-in
monkeypatch.setattr(torch.version, "hip", "7.2.0", raising = False)
got = rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")
assert got == {"rope": True, "adaln": 0}
assert fmod.apply_rotary_emb is rf._ROPE_FNS[rf._MODULES[0]]
assert nm.AdaLayerNormZero.forward is _STOCK_FWD["AdaLayerNormZero"]
def test_adaln_opt_in_on_rocm(fake_kernels, monkeypatch):
monkeypatch.setattr(torch.version, "hip", "7.2.0", raising = False)
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
got = rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")
assert got == {"rope": True, "adaln": 3}
assert nm.AdaLayerNormZero.forward is rf._adaln_zero_forward
@pytest.mark.parametrize("value", ["0", "off", "false"])
def test_kill_switches_on_rocm(fake_kernels, monkeypatch, value):
monkeypatch.setattr(torch.version, "hip", "7.2.0", raising = False)
monkeypatch.setenv(rf.FUSED_ROPE_ENV, value)
monkeypatch.setenv(rf.FUSED_ADALN_ENV, value)
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}
def test_force_on_engages_on_cuda(fake_kernels, monkeypatch):
monkeypatch.setattr(torch.version, "hip", None, raising = False)
monkeypatch.setattr(torch, "__version__", _CUDA_TORCH_VERSION, raising = False)
monkeypatch.setenv(rf.FUSED_ROPE_ENV, "1")
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
assert rf.install_for_pipe(_PipeLike(), torch.float16, "cuda") == {"rope": True, "adaln": 3}
def test_switches_are_independent(fake_kernels, monkeypatch):
monkeypatch.setenv(rf.FUSED_ROPE_ENV, "1")
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "0")
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": True, "adaln": 0}
def test_fp32_cpu_and_other_families_stay_stock(fake_kernels, monkeypatch):
monkeypatch.setenv(rf.FUSED_ROPE_ENV, "1")
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
assert rf.install_for_pipe(_PipeLike(), torch.float32, "cuda") == {"rope": False, "adaln": 0}
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cpu") == {"rope": False, "adaln": 0}
other = _PipeLike("diffusers.models.transformers.transformer_qwenimage")
assert rf.install_for_pipe(other, torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}
def test_no_triton_keeps_stock(monkeypatch):
monkeypatch.setenv(rf.FUSED_ROPE_ENV, "1")
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
monkeypatch.setattr(rf, "_rope_kernel", lambda: None)
monkeypatch.setattr(rf, "_adaln_kernel", lambda: None)
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda") == {"rope": False, "adaln": 0}
def test_reinstall_is_clean(fake_kernels, monkeypatch):
monkeypatch.setenv(rf.FUSED_ROPE_ENV, "1")
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
for _ in range(2):
rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")
assert all(prev not in rf._FORWARDS.values() for prev in rf._ADALN_PREV.values())
def test_cpu_tensors_fall_through(fake_kernels, monkeypatch):
monkeypatch.setenv(rf.FUSED_ROPE_ENV, "1")
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")
x = torch.randn(1, 6, 2, 8)
pos = torch.randn(6, 4, dtype = torch.float64)
freqs = (pos.cos().repeat_interleave(2, -1).float(), pos.sin().repeat_interleave(2, -1).float())
assert torch.equal(
fmod.apply_rotary_emb(x, freqs, sequence_dim = 1), stock_rope(x, freqs, sequence_dim = 1)
)
torch.manual_seed(0)
norm = nm.AdaLayerNormZero(16)
xs, emb = torch.randn(2, 5, 16), torch.randn(2, 16)
got = norm(xs, emb = emb)
want = _STOCK_FWD["AdaLayerNormZero"](norm, xs, emb = emb)
assert all(torch.equal(a, b) for a, b in zip(got, want))
def test_load_and_teardown_wiring():
src = (Path(__file__).resolve().parents[1] / "core" / "inference" / "diffusion.py").read_text(
encoding = "utf-8"
)
tree = ast.parse(src)
teardown = next(
n
for n in ast.walk(tree)
if isinstance(n, ast.FunctionDef) and n.name == "_uninstall_fused_dit_patches"
)
# The teardown restores the registry's fused-DiT group, which carries the ROCm fused spec.
assert "uninstall_group(GROUP_FUSED_DIT)" in ast.unparse(teardown)
from core.inference.diffusion_autoopt import patches as ap
spec = next(s for s in ap.specs(group = ap.GROUP_FUSED_DIT) if s.name == "rocm_fused")
assert spec.stage == ap.STAGE_LOAD
# Both unload paths unwind the fused AdaLN again after the eager layer (deferred profile installs it on top).
assert src.count("_uninstall_fused_dit_patches()") >= 4
load = next(
n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "load_pipeline"
)
assert "autoopt.STAGE_LOAD," in ast.unparse(load)
import types as _types
from core.inference import diffusion, diffusion_rocm_fused
seen = []
orig_install, orig_uninstall = (
diffusion_rocm_fused.install_for_pipe,
diffusion_rocm_fused.uninstall,
)
try:
diffusion_rocm_fused.install_for_pipe = lambda *a, **k: seen.append(("install", a, k)) or {
"rope": False,
"adaln": 0,
}
diffusion_rocm_fused.uninstall = lambda: seen.append(("uninstall",))
pipe = _types.SimpleNamespace(transformer = None)
ap.install_stage(
ap.STAGE_LOAD,
ap.PatchContext(pipe = pipe, dtype = "bf16", device = "cuda", logger = "log", speed_mode = "off"),
)
diffusion._uninstall_fused_dit_patches()
for mode in (None, "eager", "default"):
ap.install_stage(
ap.STAGE_LOAD,
ap.PatchContext(
pipe = pipe, dtype = "bf16", device = "cuda", logger = "log", speed_mode = mode
),
)
diffusion._uninstall_fused_dit_patches()
# An unset dense request resolves to "off" but is not an explicit off.
ap.install_stage(
ap.STAGE_LOAD,
ap.PatchContext(
pipe = pipe,
dtype = "bf16",
device = "cuda",
logger = "log",
speed_mode = "off",
explicit_off = False,
),
)
diffusion._uninstall_fused_dit_patches()
finally:
diffusion_rocm_fused.install_for_pipe = orig_install
diffusion_rocm_fused.uninstall = orig_uninstall
assert ("install", (pipe, "bf16", "cuda", "log"), {"adaln": False}) in seen
assert seen.count(("install", (pipe, "bf16", "cuda", "log"), {"adaln": True})) == 4
stage_load = next(
c
for c in ast.walk(load)
if isinstance(c, ast.Call)
and any(isinstance(a, ast.Attribute) and a.attr == "STAGE_LOAD" for a in c.args)
)
explicit = next(k.value for k in stage_load.keywords if k.arg == "explicit_off")
assert "speed_mode is not None" in ast.unparse(explicit)
assert ("uninstall",) in seen
def test_stock_lines_still_present_in_diffusers():
import inspect
for name in rf._ADALN_CLASSES:
src = inspect.getsource(_STOCK_FWD[name])
assert all(line in src for line in rf._STOCK_LINES[name]), name
assert "apply_rotary_emb(query, image_rotary_emb, sequence_dim=1)" in Path(
fmod.__file__
).read_text("utf-8")
def _freqs(S, D, device):
pos = torch.randn(S, D // 2, dtype = torch.float64, device = device) * 50
return (pos.cos().repeat_interleave(2, -1).float(), pos.sin().repeat_interleave(2, -1).float())
@needs_gpu
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("shape", [(1, 4608, 24, 128), (2, 333, 3, 64)])
def test_rope_bit_identical(monkeypatch, dtype, shape):
monkeypatch.setenv(rf.FUSED_ROPE_ENV, "1")
dev = "cuda"
assert rf.install_for_pipe(_PipeLike(), dtype, dev)["rope"]
B, S, H, D = shape
x = torch.randn(B, S, H, D, device = dev).to(dtype)
freqs = _freqs(S, D, dev)
before = rf.COUNTS["rope_fused"]
with torch.no_grad():
got = fmod.apply_rotary_emb(x, freqs, sequence_dim = 1)
want = stock_rope(x, freqs, sequence_dim = 1)
assert rf.COUNTS["rope_fused"] == before + 1
assert torch.equal(got, want)
def _bf16_ulp(t):
a = t.float().abs().clamp_min(2.0**-126)
return torch.exp2(torch.floor(torch.log2(a)) - (7 if t.dtype is torch.bfloat16 else 10))
@needs_gpu
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize(
"cls_name,D,S",
[
("AdaLayerNormZero", 3072, 4096),
("AdaLayerNormZeroSingle", 3072, 777),
("AdaLayerNormContinuous", 256, 300),
],
)
def test_adaln_matches_stock(monkeypatch, dtype, cls_name, D, S):
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
dev = "cuda"
torch.manual_seed(0)
cls = getattr(nm, cls_name)
if cls_name == "AdaLayerNormContinuous":
mod = cls(D, D, elementwise_affine = False, eps = 1e-6)
else:
mod = cls(D)
mod = mod.to(dev, dtype).eval()
assert rf.install_for_pipe(_PipeLike(), dtype, dev)["adaln"] == 3
x = (torch.randn(2, S, D, device = dev) * 3 + 0.5).to(dtype)
emb = torch.randn(2, D, device = dev).to(dtype)
before = rf.COUNTS["adaln_fused"]
with torch.no_grad():
got = mod(x, emb) if cls_name == "AdaLayerNormContinuous" else mod(x, emb = emb)
want = (
_STOCK_FWD[cls_name](mod, x, emb)
if cls_name == "AdaLayerNormContinuous"
else _STOCK_FWD[cls_name](mod, x, emb = emb)
)
assert rf.COUNTS["adaln_fused"] == before + 1
got0, want0 = (got, want) if torch.is_tensor(got) else (got[0], want[0])
if not torch.is_tensor(got):
assert all(torch.equal(a, b) for a, b in zip(got[1:], want[1:]))
diff = (got0.float() - want0.float()).abs()
# only the row reduction order differs: allow a one-ULP flip plus one rounding of product and sum
with torch.no_grad():
e = mod.linear(mod.silu(emb))
scale = (
e.chunk(2, dim = 1)[0]
if cls_name == "AdaLayerNormContinuous"
else e.chunk(6 if cls_name == "AdaLayerNormZero" else 3, dim = 1)[1]
)
n = mod.norm(x)
m = (1 + scale)[:, None, :]
p = n * m
bound = 2 * (_bf16_ulp(n) * m.float().abs() + _bf16_ulp(p) + _bf16_ulp(want0))
assert (diff <= bound).all(), float((diff / bound).max())
assert (diff > 0).float().mean().item() < 0.01, (diff > 0).float().mean().item()
@needs_gpu
def test_tiny_flux1_transformer_end_to_end(monkeypatch):
from diffusers import FluxTransformer2DModel
monkeypatch.setenv(rf.FUSED_ROPE_ENV, "1")
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
dev, dtype = "cuda", torch.bfloat16
torch.manual_seed(0)
model = (
FluxTransformer2DModel(
patch_size = 1,
in_channels = 16,
num_layers = 2,
num_single_layers = 2,
attention_head_dim = 32,
num_attention_heads = 4,
joint_attention_dim = 64,
pooled_projection_dim = 32,
axes_dims_rope = (8, 12, 12),
)
.to(dev, dtype)
.eval()
)
B, img, txt = 1, 256, 32
inputs = dict(
hidden_states = torch.randn(B, img, 16, device = dev, dtype = dtype),
encoder_hidden_states = torch.randn(B, txt, 64, device = dev, dtype = dtype),
pooled_projections = torch.randn(B, 32, device = dev, dtype = dtype),
timestep = torch.tensor([0.5], device = dev, dtype = dtype),
img_ids = torch.randint(0, 16, (img, 3), device = dev).float(),
txt_ids = torch.zeros(txt, 3, device = dev),
return_dict = False,
)
with torch.no_grad():
want = model(**inputs)[0]
rf.install_for_pipe(type("P", (), {"transformer": model})(), dtype, dev)
r0, a0 = rf.COUNTS["rope_fused"], rf.COUNTS["adaln_fused"]
got = model(**inputs)[0]
# 2 double blocks (img + txt AdaLN, q + k RoPE) + 2 single blocks + norm_out
assert rf.COUNTS["rope_fused"] - r0 == 2 * 2 + 2 * 2
assert rf.COUNTS["adaln_fused"] - a0 == 2 * 2 + 2 + 1
rf.uninstall()
rel = (got.float() - want.float()).norm() / want.float().norm()
assert rel < 1e-2, float(rel)
@needs_gpu
def test_tiny_flux2_transformer_default_on(monkeypatch):
f2 = pytest.importorskip("diffusers").Flux2Transformer2DModel
if not (rf._is_rocm() or rf._is_nvidia_linux()):
pytest.skip("auto engages on ROCm and NVIDIA Linux only")
dev, dtype = "cuda", torch.bfloat16
torch.manual_seed(0)
model = (
f2(
patch_size = 1,
in_channels = 16,
num_layers = 2,
num_single_layers = 2,
attention_head_dim = 32,
num_attention_heads = 4,
joint_attention_dim = 64,
timestep_guidance_channels = 32,
axes_dims_rope = (8, 8, 8, 8),
guidance_embeds = False,
)
.to(dev, dtype)
.eval()
)
img, txt = 256, 32
ids = torch.cartesian_prod(torch.arange(1), torch.arange(16), torch.arange(16), torch.arange(1))
inputs = dict(
hidden_states = torch.randn(1, img, 16, device = dev, dtype = dtype),
encoder_hidden_states = torch.randn(1, txt, 64, device = dev, dtype = dtype),
timestep = torch.tensor([0.5], device = dev, dtype = dtype),
img_ids = ids.to(dev)[None].float(),
txt_ids = torch.cartesian_prod(
torch.arange(1), torch.arange(1), torch.arange(1), torch.arange(txt)
)
.to(dev)[None]
.float(),
return_dict = False,
)
with torch.no_grad():
want = model(**inputs)[0]
assert rf.install_for_pipe(type("P", (), {"transformer": model})(), dtype, dev) == {
"rope": True,
"adaln": 0,
}
r0 = rf.COUNTS["rope_fused"]
got = model(**inputs)[0]
assert rf.COUNTS["rope_fused"] - r0 == 2 * 2 + 2 * 2
assert torch.equal(got, want)
def test_teardown_survives_a_later_eager_layer(fake_kernels, monkeypatch):
# unload in diffusion.py's order: fused teardown first, eager after
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
try:
assert rf.install_adaln(torch.bfloat16, "cuda") == len(rf._ADALN_CLASSES)
ep.install_compile_safe_patches()
assert nm.AdaLayerNormZero.forward is not rf._FORWARDS["AdaLayerNormZero"]
rf.uninstall()
ep.uninstall_patches()
rf.uninstall()
assert {n: getattr(nm, n).forward for n in rf._ADALN_CLASSES} == _STOCK_FWD
torch.manual_seed(0)
norm = nm.AdaLayerNormZero(16)
xs, emb = torch.randn(2, 5, 16), torch.randn(2, 16)
got = norm(xs, emb = emb)
want = _STOCK_FWD["AdaLayerNormZero"](norm, xs, emb = emb)
assert all(torch.equal(a, b) for a, b in zip(got, want))
finally:
ep.uninstall_patches()
rf.uninstall()
for n, fwd in _STOCK_FWD.items():
getattr(nm, n).forward = fwd
def test_stock_branch_never_loses_its_previous_forward(fake_kernels, monkeypatch):
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
rf.install_adaln(torch.bfloat16, "cuda")
cls = nm.AdaLayerNormZero
rf._ADALN_PREV.clear()
try:
assert rf._prev(cls) is _STOCK_FWD["AdaLayerNormZero"]
finally:
for n, fwd in _STOCK_FWD.items():
getattr(nm, n).forward = fwd
def test_a_failed_rope_launch_disables_the_fused_path_for_the_load(monkeypatch):
calls = []
def failing_launch(x, cos, sin):
calls.append(1)
raise RuntimeError("PTX JIT compilation failed")
monkeypatch.setattr(rf, "_rope_kernel", lambda: failing_launch)
monkeypatch.setattr(rf, "_rope_eligible", lambda *a, **k: True)
_nvidia(monkeypatch)
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")["rope"] is True
x = torch.randn(1, 6, 2, 8)
pos = torch.randn(6, 4, dtype = torch.float64)
freqs = (pos.cos().repeat_interleave(2, -1).float(), pos.sin().repeat_interleave(2, -1).float())
want = stock_rope(x, freqs, sequence_dim = 1)
for _ in range(4):
assert torch.equal(fmod.apply_rotary_emb(x, freqs, sequence_dim = 1), want)
assert len(calls) == 1
rf.uninstall()
rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")
fmod.apply_rotary_emb(x, freqs, sequence_dim = 1)
assert len(calls) == 2
def test_a_failed_adaln_launch_disables_the_fused_path_for_the_load(monkeypatch):
calls = []
def failing_launch(x, scale, shift, eps):
calls.append(1)
raise RuntimeError("PTX JIT compilation failed")
monkeypatch.setattr(rf, "_adaln_kernel", lambda: failing_launch)
monkeypatch.setattr(rf, "_rope_kernel", lambda: (lambda x, cos, sin: None))
monkeypatch.setattr(rf, "_adaln_eligible", lambda norm, x, scale, shift: float(norm.eps))
monkeypatch.setattr(torch.version, "hip", "7.2.0", raising = False)
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
assert rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")["adaln"] == 3
torch.manual_seed(0)
norm = nm.AdaLayerNormContinuous(8, 8, elementwise_affine = False, eps = 1e-6)
x = torch.randn(2, 5, 8)
cond = torch.randn(2, 8)
with torch.no_grad():
want = _STOCK_FWD["AdaLayerNormContinuous"](norm, x, cond)
for _ in range(4):
assert torch.equal(norm(x, cond), want)
assert len(calls) == 1
rf.uninstall()
rf.install_for_pipe(_PipeLike(), torch.bfloat16, "cuda")
with torch.no_grad():
norm(x, cond)
assert len(calls) == 2
@pytest.mark.parametrize(
"cls_name, call",
[
("AdaLayerNormZero", lambda norm, x, emb: norm(x, emb = emb)[0]),
("AdaLayerNormZeroSingle", lambda norm, x, emb: norm(x, emb = emb)[0]),
("AdaLayerNormContinuous", lambda norm, x, emb: norm(x, emb)),
],
)
def test_compiled_adaln_never_recompiles_per_call(fake_kernels, monkeypatch, cls_name, call):
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
assert rf.install_adaln(torch.bfloat16, "cuda") == 3
torch.manual_seed(0)
if cls_name == "AdaLayerNormContinuous":
norm = nm.AdaLayerNormContinuous(16, 16, elementwise_affine = False, eps = 1e-6)
else:
norm = getattr(nm, cls_name)(16)
x, emb = torch.randn(2, 5, 16), torch.randn(2, 16)
before = dict(rf.COUNTS)
with torch.no_grad():
want = call(norm, x, emb)
assert rf.COUNTS["adaln_stock"] == before["adaln_stock"] + 1
before = dict(rf.COUNTS)
torch._dynamo.reset()
compiled = torch.compile(call, backend = "eager", fullgraph = True, dynamic = False)
try:
with torch._dynamo.config.patch(error_on_recompile = True), torch.no_grad():
outs = [compiled(norm, x, emb) for _ in range(4)]
finally:
torch._dynamo.reset()
assert all(torch.equal(out, want) for out in outs)
assert rf.COUNTS == before
def test_tiny_flux1_regional_compile_holds_its_graphs_across_steps(fake_kernels, monkeypatch):
from diffusers import FluxTransformer2DModel
from torch._dynamo.utils import counters
monkeypatch.setenv(rf.FUSED_ADALN_ENV, "1")
rf.install_adaln(torch.bfloat16, "cuda")
torch.manual_seed(0)
model = FluxTransformer2DModel(
patch_size = 1,
in_channels = 16,
num_layers = 2,
num_single_layers = 3,
attention_head_dim = 32,
num_attention_heads = 4,
joint_attention_dim = 64,
pooled_projection_dim = 32,
axes_dims_rope = (8, 12, 12),
).eval()
B, img, txt = 1, 64, 8
inputs = dict(
hidden_states = torch.randn(B, img, 16),
encoder_hidden_states = torch.randn(B, txt, 64),
pooled_projections = torch.randn(B, 32),
img_ids = torch.randint(0, 8, (img, 3)).float(),
txt_ids = torch.zeros(txt, 3),
return_dict = False,
)
torch._dynamo.reset()
counters.clear()
try:
# Studio's FLUX.1 settings (diffusion_speed: fullgraph, static shapes for a stream-merging block).
model.compile_repeated_blocks(backend = "eager", fullgraph = True, dynamic = False)
graphs = []
with torch.no_grad():
for t in (0.9, 0.6, 0.3):
model(timestep = torch.tensor([t]), **inputs)
graphs.append(counters["stats"]["unique_graphs"])
finally:
torch._dynamo.reset()
assert graphs[0] > 0
assert graphs[1:] == [graphs[0]] * 2, graphs