Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: compile the diffusion denoiser on fp16 GPUs when a speed tier is picked (T4 and other pre-Ampere cards) (#11899)

Daniel Hanae6556334c14
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index dcb34ed9a06..120e882e041 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -130,6 +130,8 @@ from .diffusion_speed import (
compile_dynamic,
compile_eligible,
compiled_shapes_are_static,
+ fp16_compile_explicit_only,
+ fp16_unet_offloaded,
fresh_compile_count,
normalize_speed_mode,
resolve_speed_mode,
@@ -5550,6 +5552,7 @@ class DiffusionBackend:
and effective_speed == SPEED_OFF
and transformer_quant_engaged is None
and compile_eligible(target, is_gguf = False, family = fam)
+ and not fp16_compile_explicit_only(target)
)
# Speed optims run BEFORE placement, so snapshot the global backend flags first for unload restore.
# The dense transformer quant above builds quiet configs, so it mutated none of these flags.
@@ -5629,8 +5632,12 @@ class DiffusionBackend:
self._raise_if_load_cancelled(_load_token)
# Pre-warmed torch.compile cache: a per-fingerprint inductor dir plus a bundle loaded before the
# first compiled forward pays the 25-58s compile once.
- if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and compile_eligible(
- target, is_gguf = gguf_transformer, family = fam
+ if (
+ effective_speed in (SPEED_DEFAULT, SPEED_MAX)
+ and compile_eligible(target, is_gguf = gguf_transformer, family = fam)
+ and not fp16_unet_offloaded(
+ target, pipe, offload_active = plan.offload_policy != OFFLOAD_NONE
+ )
):
compile_ctx = compile_cache.begin(
family = fam.name,
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 06d36d4b4b8..f2a07df3582 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -24,7 +24,7 @@ this). On opt-in it applies the near-lossless speedups in the diffusers-recommen
``default`` is the cheap always-amortising compile; ``max`` pays the larger regional tax for the
bigger warm speedup. The compiled dequant is skipped under ``max`` (the regional compile subsumes
it; a separate compiled dequant would break that graph). ``supports_torch_compile`` + bf16/CUDA
-checks gate regional compile.
+(or fp16 on Turing+ NVIDIA) checks gate regional compile.
The flags this flips (TF32, cudnn.benchmark) are PROCESS-WIDE, so ``snapshot_backend_flags`` /
``restore_backend_flags`` let the caller restore prior values at unload, keeping a later ``off``
@@ -204,11 +204,8 @@ def torch_compile_runtime_available() -> bool:
def compile_eligible(target: Any, *, is_gguf: bool, family: Any) -> bool:
- """Whether the denoiser's repeated block should be regionally compiled.
-
- Only on CUDA (incl. ROCm), for a bf16 transformer, on a compile-friendly family, in a process
- that can run inductor. ``is_gguf`` no longer disqualifies (GGUF compiles fine and ~2.3x
- faster); the param is kept for compat."""
+ """Whether the denoiser's repeated block should be regionally compiled: CUDA (incl. ROCm) bf16, or fp16 on NVIDIA
+ sm_75+, on a compile-friendly family with inductor available. ``is_gguf`` is kept for compat only."""
del is_gguf
if not torch_compile_runtime_available():
return False
@@ -216,7 +213,18 @@ def compile_eligible(target: Any, *, is_gguf: bool, family: Any) -> bool:
return False
if not bool(getattr(family, "supports_torch_compile", True)):
return False
- return _is_bfloat16(getattr(target, "dtype", None))
+ dtype = getattr(target, "dtype", None)
+ if _is_bfloat16(dtype):
+ return True
+ # fp16-incompatible families run in fp32 though video passes the fp16 target; fp32 compile is unmeasured.
+ if bool(getattr(family, "fp16_incompatible", False)):
+ return False
+ return _is_float16(dtype) and _fp16_compile_capable(target)
+
+
+def fp16_compile_explicit_only(target: Any) -> bool:
+ """fp16 compiles only on an explicit tier, never the deferred profile (T4 SDXL-Turbo: ~245 s for ~0.1 s/image)."""
+ return _is_float16(getattr(target, "dtype", None))
def _is_bfloat16(dtype: Any) -> bool:
@@ -227,6 +235,35 @@ def _is_bfloat16(dtype: Any) -> bool:
return str(dtype).endswith("bfloat16")
+def _is_float16(dtype: Any) -> bool:
+ try:
+ import torch
+ return dtype is torch.float16
+ except Exception:
+ return str(dtype).endswith("float16") and not str(dtype).endswith("bfloat16")
+
+
+# Only sm_75 (T4) was measured; Volta and older stay eager.
+_FP16_COMPILE_MIN_CAPABILITY = (7, 5)
+
+
+def _fp16_compile_capable(target: Any) -> bool:
+ if getattr(target, "backend", "cuda") != "cuda":
+ return False
+ try:
+ import torch
+
+ ordinal = getattr(target, "ordinal", None)
+ cap = (
+ torch.cuda.get_device_capability()
+ if ordinal is None
+ else torch.cuda.get_device_capability(ordinal)
+ )
+ return tuple(cap)[:2] >= _FP16_COMPILE_MIN_CAPABILITY
+ except Exception: # noqa: BLE001 - an unanswerable probe keeps today's eager path
+ return False
+
+
def apply_speed_optims(
pipe: Any,
target: Any,
@@ -289,7 +326,9 @@ def apply_speed_optims(
# Asked directly: this arm never reaches compile_eligible(), unlike the dense arm below.
if is_gguf and on_cuda and family_allows_compile and torch_compile_runtime_available():
applied["compiled_dequant"] = gguf_compile.install_compiled_dequant(logger)
- elif compile_eligible(target, is_gguf = is_gguf, family = family):
+ elif compile_eligible(target, is_gguf = is_gguf, family = family) and not fp16_unet_offloaded(
+ target, pipe, offload_active = offload_active
+ ):
# A U-Net (SDXL) fuses QKV BEFORE its whole-module compile: 36.3 vs 39.3 ms/step (LPIPS 0.033). DiTs were
# neutral, so they keep the fuse on max only.
if _denoiser_unet(pipe) is not None:
@@ -301,7 +340,11 @@ def apply_speed_optims(
cache_active = cache_active,
offload_active = offload_active,
)
- elif mode == SPEED_MAX and compile_eligible(target, is_gguf = is_gguf, family = family):
+ elif (
+ mode == SPEED_MAX
+ and compile_eligible(target, is_gguf = is_gguf, family = family)
+ and not fp16_unet_offloaded(target, pipe, offload_active = offload_active)
+ ):
applied["compiled"] = _compile_repeated_blocks(
pipe,
logger,
@@ -352,6 +395,15 @@ def apply_speed_optims(
return applied
+def fp16_unet_offloaded(target: Any, pipe: Any, *, offload_active: bool) -> bool:
+ """An offloaded fp16 U-Net stays eager: fused QKV costs more transfer than compile saves (L4: 5.43 vs 4.86 s)."""
+ return (
+ bool(offload_active)
+ and _is_float16(getattr(target, "dtype", None))
+ and _denoiser_unet(pipe) is not None
+ )
+
+
def _vae_channels_last(pipe: Any, logger: Any) -> bool:
vae = getattr(pipe, "vae", None)
if vae is None or not hasattr(vae, "to"):
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 8e3869bd786..afe0ee09629 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -1197,6 +1197,36 @@ def test_dense_speed_auto_defers_compile_to_third_generation(fake_runtime, tmp_p
backend.unload()
+def test_deferred_speed_stays_off_when_only_an_explicit_tier_may_compile(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference import diffusion as dmod
+
+ seen = []
+ monkeypatch.setattr(dmod, "compile_eligible", lambda *a, **k: True)
+ monkeypatch.setattr(
+ dmod, "fp16_compile_explicit_only", lambda target: seen.append(target) or True
+ )
+ engaged = []
+ monkeypatch.setattr(
+ DiffusionBackend, "_engage_deferred_speed", lambda self, state: engaged.append(1)
+ )
+ monkeypatch.setattr(dmod.compile_cache, "begin", lambda **k: None)
+
+ (tmp_path / "model.safetensors").write_bytes(b"weights")
+ backend = DiffusionBackend()
+ status = _load_into(
+ backend, tmp_path, gguf_filename = "model.safetensors", family_override = "qwen-image"
+ )
+ assert seen
+ assert status["resolved"]["speed_mode"]["value"] == "off"
+ for p in ("one", "two", "three"):
+ backend.generate(prompt = p)
+ assert engaged == []
+ assert backend.status()["speed_mode"] == "off"
+ backend.unload()
+
+
def test_deferred_speed_skips_when_lora_requested(fake_runtime, tmp_path, monkeypatch):
# A compiled transformer rejects LoRA, so the deferral must skip while a LoRA is requested.
from core.inference import diffusion as dmod
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index 260f974de07..5bbfcbbab04 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -126,7 +126,7 @@ def test_compile_eligible_requires_bf16_cuda_friendly(monkeypatch):
assert compile_eligible(_target(), is_gguf = False, family = _family()) is True
# GGUF is compile-eligible too (measured ~2.3x, PSNR ~37 dB vs eager).
assert compile_eligible(_target(), is_gguf = True, family = _family()) is True
- # fp16 (non-bf16) is excluded.
+ # fp16 is excluded when the card's capability cannot be read (this stub has no probe).
assert compile_eligible(_target(dtype = "float16"), is_gguf = False, family = _family()) is False
# A family flagged not compile-friendly is excluded.
assert compile_eligible(_target(), is_gguf = False, family = _family(compile_ok = False)) is False
@@ -134,6 +134,168 @@ def test_compile_eligible_requires_bf16_cuda_friendly(monkeypatch):
assert compile_eligible(_target(compile_ok = False), is_gguf = False, family = _family()) is False
+def _stub_torch_capability(
+ monkeypatch,
+ cap,
+ seen = None,
+):
+ torch = _stub_torch(monkeypatch)
+ torch.float16 = "float16"
+
+ def get_device_capability(*args):
+ if seen is not None:
+ seen.append(args)
+ return cap
+
+ torch.cuda = types.SimpleNamespace(
+ is_available = lambda: True, get_device_capability = get_device_capability
+ )
+ return torch
+
+
+def _fp16_target(**kw):
+ t = _target(dtype = "float16")
+ for k, v in kw.items():
+ setattr(t, k, v)
+ return t
+
+
[email protected]("cap", [(7, 5), (8, 6), (8, 9), (12, 0)])
+def test_compile_eligible_fp16_on_turing_or_newer(monkeypatch, cap):
+ _stub_torch_capability(monkeypatch, cap)
+ assert compile_eligible(_fp16_target(), is_gguf = False, family = _family()) is True
+ assert compile_eligible(_fp16_target(backend = "cuda"), is_gguf = True, family = _family()) is True
+
+
[email protected]("cap", [(7, 0), (6, 1), (6, 0), (5, 2)])
+def test_compile_eligible_fp16_stays_eager_below_turing(monkeypatch, cap):
+ _stub_torch_capability(monkeypatch, cap)
+ assert compile_eligible(_fp16_target(), is_gguf = False, family = _family()) is False
+
+
+def test_compile_eligible_fp16_asks_the_selected_card(monkeypatch):
+ seen = []
+ _stub_torch_capability(monkeypatch, (7, 5), seen)
+ assert compile_eligible(_fp16_target(ordinal = 3), is_gguf = False, family = _family()) is True
+ assert seen == [(3,)]
+
+
+def test_compile_eligible_fp16_refusals(monkeypatch):
+ _stub_torch_capability(monkeypatch, (7, 5))
+ fam = types.SimpleNamespace(supports_torch_compile = True, fp16_incompatible = True)
+ assert compile_eligible(_fp16_target(), is_gguf = False, family = fam) is False
+ assert compile_eligible(_fp16_target(backend = "rocm"), is_gguf = False, family = _family()) is False
+ assert (
+ compile_eligible(_fp16_target(), is_gguf = False, family = _family(compile_ok = False)) is False
+ )
+ assert (
+ compile_eligible(
+ _fp16_target(supports_default_torch_compile = False), is_gguf = False, family = _family()
+ )
+ is False
+ )
+ assert compile_eligible(_target(dtype = "float32"), is_gguf = False, family = _family()) is False
+
+
+def test_compile_eligible_fp16_probe_failure_stays_eager(monkeypatch):
+ torch = _stub_torch_capability(monkeypatch, (7, 5))
+
+ def boom(*args):
+ raise RuntimeError("no device")
+
+ torch.cuda.get_device_capability = boom
+ assert compile_eligible(_fp16_target(), is_gguf = False, family = _family()) is False
+
+
+def test_fp16_compile_is_explicit_tier_only(monkeypatch):
+ _stub_torch_capability(monkeypatch, (7, 5))
+ assert ds_mod.fp16_compile_explicit_only(_fp16_target()) is True
+ assert compile_eligible(_fp16_target(), is_gguf = False, family = _family()) is True
+ assert ds_mod.fp16_compile_explicit_only(_target()) is False
+ assert ds_mod.fp16_compile_explicit_only(_target(dtype = "float32")) is False
+
+
+def test_compile_eligible_bf16_never_probes_capability(monkeypatch):
+ seen = []
+ _stub_torch_capability(monkeypatch, (7, 0), seen)
+ fam = types.SimpleNamespace(supports_torch_compile = True, fp16_incompatible = True)
+ assert compile_eligible(_target(), is_gguf = False, family = fam) is True
+ assert seen == []
+
+
+def test_apply_speed_optims_compiles_fp16_dit(monkeypatch):
+ _stub_torch_capability(monkeypatch, (7, 5))
+ monkeypatch.setattr(ds_mod, "_compile_repeated_blocks", lambda *a, **k: True)
+ applied = apply_speed_optims(
+ types.SimpleNamespace(),
+ _fp16_target(),
+ is_gguf = False,
+ family = _family(),
+ speed_mode = SPEED_DEFAULT,
+ )
+ assert applied["compiled"] is True
+
+
+def _unet_pipe():
+ UNet2DConditionModel = type("UNet2DConditionModel", (), {})
+ return types.SimpleNamespace(unet = UNet2DConditionModel())
+
+
[email protected]("mode", [SPEED_DEFAULT, SPEED_MAX])
+def test_fp16_unet_stays_eager_under_offload(monkeypatch, mode):
+ _stub_torch_capability(monkeypatch, (7, 5))
+ calls = []
+ monkeypatch.setattr(ds_mod, "_compile_repeated_blocks", lambda *a, **k: calls.append(1) or True)
+ monkeypatch.setattr(ds_mod, "_fuse_qkv", lambda *a, **k: True)
+ off = apply_speed_optims(
+ _unet_pipe(),
+ _fp16_target(),
+ is_gguf = False,
+ family = _family(),
+ speed_mode = mode,
+ offload_active = True,
+ )
+ assert off["compiled"] is False and calls == []
+ if mode == SPEED_DEFAULT:
+ assert off["fused_qkv"] is False
+ resident = apply_speed_optims(
+ _unet_pipe(),
+ _fp16_target(),
+ is_gguf = False,
+ family = _family(),
+ speed_mode = mode,
+ offload_active = False,
+ )
+ assert resident["compiled"] is True
+
+
+def test_fp16_offloaded_dit_and_bf16_unet_still_compile(monkeypatch):
+ _stub_torch_capability(monkeypatch, (7, 5))
+ monkeypatch.setattr(ds_mod, "_compile_repeated_blocks", lambda *a, **k: True)
+ monkeypatch.setattr(ds_mod, "_fuse_qkv", lambda *a, **k: True)
+ dit = apply_speed_optims(
+ types.SimpleNamespace(),
+ _fp16_target(),
+ is_gguf = False,
+ family = _family(),
+ speed_mode = SPEED_DEFAULT,
+ offload_active = True,
+ )
+ assert dit["compiled"] is True
+ bf16 = apply_speed_optims(
+ _unet_pipe(),
+ _target(),
+ is_gguf = False,
+ family = _family(),
+ speed_mode = SPEED_DEFAULT,
+ offload_active = True,
+ )
+ assert bf16["compiled"] is True
+ assert ds_mod.fp16_unet_offloaded(_target(), _unet_pipe(), offload_active = True) is False
+ assert ds_mod.fp16_unet_offloaded(_fp16_target(), _unet_pipe(), offload_active = True) is True
+ assert ds_mod.fp16_unet_offloaded(_fp16_target(), _unet_pipe(), offload_active = False) is False
+
+
# ── backend-flag snapshot / restore (TF32 / cudnn.benchmark leak guard) ────────