Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: compile the diffusion denoiser on fp16 GPUs when a speed tier is picked (T4 and other pre-Ampere cards) (#11899)
Daniel Hanae6556334c14
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex dcb34ed9a06..120e882e041 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -130,6 +130,8 @@ from .diffusion_speed import (compile_dynamic,compile_eligible,compiled_shapes_are_static,+ fp16_compile_explicit_only,+ fp16_unet_offloaded,fresh_compile_count,normalize_speed_mode,resolve_speed_mode,@@ -5550,6 +5552,7 @@ class DiffusionBackend:and effective_speed == SPEED_OFFand transformer_quant_engaged is Noneand compile_eligible(target, is_gguf = False, family = fam)+ and not fp16_compile_explicit_only(target))# Speed optims run BEFORE placement, so snapshot the global backend flags first for unload restore.# The dense transformer quant above builds quiet configs, so it mutated none of these flags.@@ -5629,8 +5632,12 @@ class DiffusionBackend:self._raise_if_load_cancelled(_load_token)# Pre-warmed torch.compile cache: a per-fingerprint inductor dir plus a bundle loaded before the# first compiled forward pays the 25-58s compile once.- if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and compile_eligible(- target, is_gguf = gguf_transformer, family = fam+ if (+ effective_speed in (SPEED_DEFAULT, SPEED_MAX)+ and compile_eligible(target, is_gguf = gguf_transformer, family = fam)+ and not fp16_unet_offloaded(+ target, pipe, offload_active = plan.offload_policy != OFFLOAD_NONE+ )):compile_ctx = compile_cache.begin(family = fam.name,diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 06d36d4b4b8..f2a07df3582 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -24,7 +24,7 @@ this). On opt-in it applies the near-lossless speedups in the diffusers-recommen``default`` is the cheap always-amortising compile; ``max`` pays the larger regional tax for thebigger warm speedup. The compiled dequant is skipped under ``max`` (the regional compile subsumesit; a separate compiled dequant would break that graph). ``supports_torch_compile`` + bf16/CUDA-checks gate regional compile.+(or fp16 on Turing+ NVIDIA) checks gate regional compile.The flags this flips (TF32, cudnn.benchmark) are PROCESS-WIDE, so ``snapshot_backend_flags`` /``restore_backend_flags`` let the caller restore prior values at unload, keeping a later ``off``@@ -204,11 +204,8 @@ def torch_compile_runtime_available() -> bool:def compile_eligible(target: Any, *, is_gguf: bool, family: Any) -> bool:- """Whether the denoiser's repeated block should be regionally compiled.-- Only on CUDA (incl. ROCm), for a bf16 transformer, on a compile-friendly family, in a process- that can run inductor. ``is_gguf`` no longer disqualifies (GGUF compiles fine and ~2.3x- faster); the param is kept for compat."""+ """Whether the denoiser's repeated block should be regionally compiled: CUDA (incl. ROCm) bf16, or fp16 on NVIDIA+ sm_75+, on a compile-friendly family with inductor available. ``is_gguf`` is kept for compat only."""del is_ggufif not torch_compile_runtime_available():return False@@ -216,7 +213,18 @@ def compile_eligible(target: Any, *, is_gguf: bool, family: Any) -> bool:return Falseif not bool(getattr(family, "supports_torch_compile", True)):return False- return _is_bfloat16(getattr(target, "dtype", None))+ dtype = getattr(target, "dtype", None)+ if _is_bfloat16(dtype):+ return True+ # fp16-incompatible families run in fp32 though video passes the fp16 target; fp32 compile is unmeasured.+ if bool(getattr(family, "fp16_incompatible", False)):+ return False+ return _is_float16(dtype) and _fp16_compile_capable(target)+++def fp16_compile_explicit_only(target: Any) -> bool:+ """fp16 compiles only on an explicit tier, never the deferred profile (T4 SDXL-Turbo: ~245 s for ~0.1 s/image)."""+ return _is_float16(getattr(target, "dtype", None))def _is_bfloat16(dtype: Any) -> bool:@@ -227,6 +235,35 @@ def _is_bfloat16(dtype: Any) -> bool:return str(dtype).endswith("bfloat16")+def _is_float16(dtype: Any) -> bool:+ try:+ import torch+ return dtype is torch.float16+ except Exception:+ return str(dtype).endswith("float16") and not str(dtype).endswith("bfloat16")+++# Only sm_75 (T4) was measured; Volta and older stay eager.+_FP16_COMPILE_MIN_CAPABILITY = (7, 5)+++def _fp16_compile_capable(target: Any) -> bool:+ if getattr(target, "backend", "cuda") != "cuda":+ return False+ try:+ import torch++ ordinal = getattr(target, "ordinal", None)+ cap = (+ torch.cuda.get_device_capability()+ if ordinal is None+ else torch.cuda.get_device_capability(ordinal)+ )+ return tuple(cap)[:2] >= _FP16_COMPILE_MIN_CAPABILITY+ except Exception: # noqa: BLE001 - an unanswerable probe keeps today's eager path+ return False++def apply_speed_optims(pipe: Any,target: Any,@@ -289,7 +326,9 @@ def apply_speed_optims(# Asked directly: this arm never reaches compile_eligible(), unlike the dense arm below.if is_gguf and on_cuda and family_allows_compile and torch_compile_runtime_available():applied["compiled_dequant"] = gguf_compile.install_compiled_dequant(logger)- elif compile_eligible(target, is_gguf = is_gguf, family = family):+ elif compile_eligible(target, is_gguf = is_gguf, family = family) and not fp16_unet_offloaded(+ target, pipe, offload_active = offload_active+ ):# A U-Net (SDXL) fuses QKV BEFORE its whole-module compile: 36.3 vs 39.3 ms/step (LPIPS 0.033). DiTs were# neutral, so they keep the fuse on max only.if _denoiser_unet(pipe) is not None:@@ -301,7 +340,11 @@ def apply_speed_optims(cache_active = cache_active,offload_active = offload_active,)- elif mode == SPEED_MAX and compile_eligible(target, is_gguf = is_gguf, family = family):+ elif (+ mode == SPEED_MAX+ and compile_eligible(target, is_gguf = is_gguf, family = family)+ and not fp16_unet_offloaded(target, pipe, offload_active = offload_active)+ ):applied["compiled"] = _compile_repeated_blocks(pipe,logger,@@ -352,6 +395,15 @@ def apply_speed_optims(return applied+def fp16_unet_offloaded(target: Any, pipe: Any, *, offload_active: bool) -> bool:+ """An offloaded fp16 U-Net stays eager: fused QKV costs more transfer than compile saves (L4: 5.43 vs 4.86 s)."""+ return (+ bool(offload_active)+ and _is_float16(getattr(target, "dtype", None))+ and _denoiser_unet(pipe) is not None+ )++def _vae_channels_last(pipe: Any, logger: Any) -> bool:vae = getattr(pipe, "vae", None)if vae is None or not hasattr(vae, "to"):diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex 8e3869bd786..afe0ee09629 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -1197,6 +1197,36 @@ def test_dense_speed_auto_defers_compile_to_third_generation(fake_runtime, tmp_pbackend.unload()+def test_deferred_speed_stays_off_when_only_an_explicit_tier_may_compile(+ fake_runtime, tmp_path, monkeypatch+):+ from core.inference import diffusion as dmod++ seen = []+ monkeypatch.setattr(dmod, "compile_eligible", lambda *a, **k: True)+ monkeypatch.setattr(+ dmod, "fp16_compile_explicit_only", lambda target: seen.append(target) or True+ )+ engaged = []+ monkeypatch.setattr(+ DiffusionBackend, "_engage_deferred_speed", lambda self, state: engaged.append(1)+ )+ monkeypatch.setattr(dmod.compile_cache, "begin", lambda **k: None)++ (tmp_path / "model.safetensors").write_bytes(b"weights")+ backend = DiffusionBackend()+ status = _load_into(+ backend, tmp_path, gguf_filename = "model.safetensors", family_override = "qwen-image"+ )+ assert seen+ assert status["resolved"]["speed_mode"]["value"] == "off"+ for p in ("one", "two", "three"):+ backend.generate(prompt = p)+ assert engaged == []+ assert backend.status()["speed_mode"] == "off"+ backend.unload()++def test_deferred_speed_skips_when_lora_requested(fake_runtime, tmp_path, monkeypatch):# A compiled transformer rejects LoRA, so the deferral must skip while a LoRA is requested.from core.inference import diffusion as dmoddiff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex 260f974de07..5bbfcbbab04 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -126,7 +126,7 @@ def test_compile_eligible_requires_bf16_cuda_friendly(monkeypatch):assert compile_eligible(_target(), is_gguf = False, family = _family()) is True# GGUF is compile-eligible too (measured ~2.3x, PSNR ~37 dB vs eager).assert compile_eligible(_target(), is_gguf = True, family = _family()) is True- # fp16 (non-bf16) is excluded.+ # fp16 is excluded when the card's capability cannot be read (this stub has no probe).assert compile_eligible(_target(dtype = "float16"), is_gguf = False, family = _family()) is False# A family flagged not compile-friendly is excluded.assert compile_eligible(_target(), is_gguf = False, family = _family(compile_ok = False)) is False@@ -134,6 +134,168 @@ def test_compile_eligible_requires_bf16_cuda_friendly(monkeypatch):assert compile_eligible(_target(compile_ok = False), is_gguf = False, family = _family()) is False+def _stub_torch_capability(+ monkeypatch,+ cap,+ seen = None,+):+ torch = _stub_torch(monkeypatch)+ torch.float16 = "float16"++ def get_device_capability(*args):+ if seen is not None:+ seen.append(args)+ return cap++ torch.cuda = types.SimpleNamespace(+ is_available = lambda: True, get_device_capability = get_device_capability+ )+ return torch+++def _fp16_target(**kw):+ t = _target(dtype = "float16")+ for k, v in kw.items():+ setattr(t, k, v)+ return t++[email protected]("cap", [(7, 5), (8, 6), (8, 9), (12, 0)])+def test_compile_eligible_fp16_on_turing_or_newer(monkeypatch, cap):+ _stub_torch_capability(monkeypatch, cap)+ assert compile_eligible(_fp16_target(), is_gguf = False, family = _family()) is True+ assert compile_eligible(_fp16_target(backend = "cuda"), is_gguf = True, family = _family()) is True++[email protected]("cap", [(7, 0), (6, 1), (6, 0), (5, 2)])+def test_compile_eligible_fp16_stays_eager_below_turing(monkeypatch, cap):+ _stub_torch_capability(monkeypatch, cap)+ assert compile_eligible(_fp16_target(), is_gguf = False, family = _family()) is False+++def test_compile_eligible_fp16_asks_the_selected_card(monkeypatch):+ seen = []+ _stub_torch_capability(monkeypatch, (7, 5), seen)+ assert compile_eligible(_fp16_target(ordinal = 3), is_gguf = False, family = _family()) is True+ assert seen == [(3,)]+++def test_compile_eligible_fp16_refusals(monkeypatch):+ _stub_torch_capability(monkeypatch, (7, 5))+ fam = types.SimpleNamespace(supports_torch_compile = True, fp16_incompatible = True)+ assert compile_eligible(_fp16_target(), is_gguf = False, family = fam) is False+ assert compile_eligible(_fp16_target(backend = "rocm"), is_gguf = False, family = _family()) is False+ assert (+ compile_eligible(_fp16_target(), is_gguf = False, family = _family(compile_ok = False)) is False+ )+ assert (+ compile_eligible(+ _fp16_target(supports_default_torch_compile = False), is_gguf = False, family = _family()+ )+ is False+ )+ assert compile_eligible(_target(dtype = "float32"), is_gguf = False, family = _family()) is False+++def test_compile_eligible_fp16_probe_failure_stays_eager(monkeypatch):+ torch = _stub_torch_capability(monkeypatch, (7, 5))++ def boom(*args):+ raise RuntimeError("no device")++ torch.cuda.get_device_capability = boom+ assert compile_eligible(_fp16_target(), is_gguf = False, family = _family()) is False+++def test_fp16_compile_is_explicit_tier_only(monkeypatch):+ _stub_torch_capability(monkeypatch, (7, 5))+ assert ds_mod.fp16_compile_explicit_only(_fp16_target()) is True+ assert compile_eligible(_fp16_target(), is_gguf = False, family = _family()) is True+ assert ds_mod.fp16_compile_explicit_only(_target()) is False+ assert ds_mod.fp16_compile_explicit_only(_target(dtype = "float32")) is False+++def test_compile_eligible_bf16_never_probes_capability(monkeypatch):+ seen = []+ _stub_torch_capability(monkeypatch, (7, 0), seen)+ fam = types.SimpleNamespace(supports_torch_compile = True, fp16_incompatible = True)+ assert compile_eligible(_target(), is_gguf = False, family = fam) is True+ assert seen == []+++def test_apply_speed_optims_compiles_fp16_dit(monkeypatch):+ _stub_torch_capability(monkeypatch, (7, 5))+ monkeypatch.setattr(ds_mod, "_compile_repeated_blocks", lambda *a, **k: True)+ applied = apply_speed_optims(+ types.SimpleNamespace(),+ _fp16_target(),+ is_gguf = False,+ family = _family(),+ speed_mode = SPEED_DEFAULT,+ )+ assert applied["compiled"] is True+++def _unet_pipe():+ UNet2DConditionModel = type("UNet2DConditionModel", (), {})+ return types.SimpleNamespace(unet = UNet2DConditionModel())++[email protected]("mode", [SPEED_DEFAULT, SPEED_MAX])+def test_fp16_unet_stays_eager_under_offload(monkeypatch, mode):+ _stub_torch_capability(monkeypatch, (7, 5))+ calls = []+ monkeypatch.setattr(ds_mod, "_compile_repeated_blocks", lambda *a, **k: calls.append(1) or True)+ monkeypatch.setattr(ds_mod, "_fuse_qkv", lambda *a, **k: True)+ off = apply_speed_optims(+ _unet_pipe(),+ _fp16_target(),+ is_gguf = False,+ family = _family(),+ speed_mode = mode,+ offload_active = True,+ )+ assert off["compiled"] is False and calls == []+ if mode == SPEED_DEFAULT:+ assert off["fused_qkv"] is False+ resident = apply_speed_optims(+ _unet_pipe(),+ _fp16_target(),+ is_gguf = False,+ family = _family(),+ speed_mode = mode,+ offload_active = False,+ )+ assert resident["compiled"] is True+++def test_fp16_offloaded_dit_and_bf16_unet_still_compile(monkeypatch):+ _stub_torch_capability(monkeypatch, (7, 5))+ monkeypatch.setattr(ds_mod, "_compile_repeated_blocks", lambda *a, **k: True)+ monkeypatch.setattr(ds_mod, "_fuse_qkv", lambda *a, **k: True)+ dit = apply_speed_optims(+ types.SimpleNamespace(),+ _fp16_target(),+ is_gguf = False,+ family = _family(),+ speed_mode = SPEED_DEFAULT,+ offload_active = True,+ )+ assert dit["compiled"] is True+ bf16 = apply_speed_optims(+ _unet_pipe(),+ _target(),+ is_gguf = False,+ family = _family(),+ speed_mode = SPEED_DEFAULT,+ offload_active = True,+ )+ assert bf16["compiled"] is True+ assert ds_mod.fp16_unet_offloaded(_target(), _unet_pipe(), offload_active = True) is False+ assert ds_mod.fp16_unet_offloaded(_fp16_target(), _unet_pipe(), offload_active = True) is True+ assert ds_mod.fp16_unet_offloaded(_fp16_target(), _unet_pipe(), offload_active = False) is False++# ── backend-flag snapshot / restore (TF32 / cudnn.benchmark leak guard) ────────