Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: decode the SDXL VAE in fp16 on fp16 GPUs (#12036)

Daniel Han83fbe4a34a3d
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index cdac7bf05b2..703e12ab6aa 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -154,6 +154,7 @@ from .diffusion_speed import (
snapshot_backend_flags,
vae_decode_compile_allowed,
)
+from .diffusion_vae_fp16 import enable_fp16_vae_decode
from .diffusion_attention import (
apply_attention_backend,
normalize_attention_backend,
@@ -6611,6 +6612,12 @@ class DiffusionBackend:
)
self._raise_if_load_cancelled(_load_token)
+ # Before the speed optims so their decode compile lands inside the non-finite check; `off` keeps fp32.
+ vae_fp16 = str(
+ speed_mode or ""
+ ).strip().lower() != SPEED_OFF and enable_fp16_vae_decode(
+ pipe, target, logger = logger
+ )
speed_applied = apply_speed_optims(
pipe,
target,
@@ -6622,6 +6629,8 @@ class DiffusionBackend:
offload_active = plan.offload_policy != OFFLOAD_NONE,
logger = logger,
)
+ if vae_fp16:
+ speed_applied["vae_fp16_decode"] = True
self._raise_if_load_cancelled(_load_token)
if (
transformer_quant_engaged is not None
@@ -8316,6 +8325,8 @@ class DiffusionBackend:
offload_active = state.offload_policy != OFFLOAD_NONE,
logger = logger,
)
+ if getattr(getattr(state.pipe, "vae", None), "_unsloth_fp16_decode", False):
+ speed_applied["vae_fp16_decode"] = True
object.__setattr__(state, "speed_mode", SPEED_DEFAULT)
object.__setattr__(state, "speed_optims", tuple(k for k, v in speed_applied.items() if v))
object.__setattr__(
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 80bf3a41a52..b8856c19712 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -1046,12 +1046,15 @@ def _guard_compiled_decode(
eager: Any,
logger: Any,
eager_when_tiled: bool = False,
+ owner: Any = None,
) -> Any:
"""``compiled`` behind an eager fallback: torch.compile is lazy, so lowering fails on the first call; OOMs still raise.
``eager_when_tiled``: a tiled decode unrolls its tile loop into one graph (minutes of compile on low-VRAM loads),
- and tiling is only settled by the memory plan after the compile, so it is read per call."""
- had_own = "decode" in getattr(vae, "__dict__", {})
+ and tiling is only settled by the memory plan after the compile, so it is read per call.
+ ``owner``: the slot the fallback restores."""
+ owner = vae if owner is None else owner
+ had_own = "decode" in getattr(owner, "__dict__", {})
failed: list = []
def guarded(*args: Any, **kwargs: Any) -> Any:
@@ -1071,9 +1074,9 @@ def _guard_compiled_decode(
vae._unsloth_compile_decode_error = error
vae._unsloth_compiled_decode = False
if had_own:
- vae.decode = eager
+ owner.decode = eager
else:
- del vae.decode
+ del owner.decode
except Exception: # noqa: BLE001 - `failed` still routes this wrapper to eager
pass
if logger is not None:
@@ -1096,7 +1099,10 @@ def _compile_vae_decode(
) -> bool:
"""torch.compile the VAE ``decode`` in place; no cudagraphs, whose capture would pin decode activations."""
vae = getattr(pipe, "vae", None)
- decode = getattr(vae, "decode", None) if vae is not None else None
+ # An outer wrapper that must stay eager (fp16 non-finite check) exposes the slot it calls through.
+ outer = getattr(vae, "__dict__", {}).get("decode") if vae is not None else None
+ owner = getattr(outer, "_unsloth_decode_slot", None) or vae
+ decode = getattr(owner, "decode", None) if vae is not None else None
if not callable(decode):
return False
# A dual-DiT family calls apply_speed_optims twice over the same pipe.
@@ -1111,8 +1117,8 @@ def _compile_vae_decode(
if max_autotune:
kwargs["mode"] = "max-autotune-no-cudagraphs"
compiled = torch.compile(decode, **kwargs)
- vae.decode = _guard_compiled_decode(
- vae, compiled, decode, logger, eager_when_tiled = eager_when_tiled
+ owner.decode = _guard_compiled_decode(
+ vae, compiled, decode, logger, eager_when_tiled = eager_when_tiled, owner = owner
)
vae._unsloth_compiled_decode = True
return True
diff --git a/studio/backend/core/inference/diffusion_vae_fp16.py b/studio/backend/core/inference/diffusion_vae_fp16.py
new file mode 100644
index 00000000000..0706915003e
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_vae_fp16.py
@@ -0,0 +1,159 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""fp16 VAE decode for U-Net (SDXL) pipelines whose VAE sets ``force_upcast``.
+
+The SDXL VAE overflows fp16 in its last decoder stages, so the pipelines upcast the whole VAE to fp32 on fp16 loads.
+Dividing the residual stream after the first upsampler by 2**8 (and the eps of each GroupNorm reading it by 2**16) is an
+identity, since GroupNorm is scale invariant, and keeps every activation in fp16 range. A non-finite decode still reruns
+in fp32 and hands later decodes back to the pipeline's upcast.
+"""
+
+from __future__ import annotations
+
+import functools
+import types
+from typing import Any, Optional
+
+_SCALE_LOG2 = 8
+
+
+def _scale_plan(decoder: Any) -> Optional[tuple]:
+ """(convs, norms) scaling the residual stream after ``up_blocks[0]``'s upsampler; None for other layouts.
+
+ ``(conv, divide_weight)``: unscaled input divides weight + bias, already-scaled input (shortcut, later upsamplers)
+ only bias. ``norms`` read the scaled stream, so their eps shrinks by scale**2."""
+ import torch
+
+ up_blocks = list(getattr(decoder, "up_blocks", None) or ())
+ if len(up_blocks) < 2 or not isinstance(
+ getattr(decoder, "conv_norm_out", None), torch.nn.GroupNorm
+ ):
+ return None
+ entry = list(getattr(up_blocks[0], "upsamplers", None) or ())
+ if len(entry) != 1 or not isinstance(getattr(entry[0], "conv", None), torch.nn.Conv2d):
+ return None
+ plan = [(entry[0].conv, True)]
+ norms = [decoder.conv_norm_out]
+ for block in up_blocks[1:]:
+ if getattr(block, "attentions", None):
+ return None
+ for resnet in getattr(block, "resnets", None) or ():
+ if not (
+ isinstance(getattr(resnet, "norm1", None), torch.nn.GroupNorm)
+ and isinstance(getattr(resnet, "norm2", None), torch.nn.GroupNorm)
+ and isinstance(getattr(resnet, "conv2", None), torch.nn.Conv2d)
+ and getattr(resnet, "time_emb_proj", None) is None
+ and getattr(resnet, "upsample", None) is None
+ and getattr(resnet, "downsample", None) is None
+ ):
+ return None
+ plan.append((resnet.conv2, True))
+ norms.append(resnet.norm1)
+ shortcut = getattr(resnet, "conv_shortcut", None)
+ if shortcut is not None:
+ if not isinstance(shortcut, torch.nn.Conv2d):
+ return None
+ plan.append((shortcut, False))
+ for upsampler in getattr(block, "upsamplers", None) or ():
+ if not isinstance(getattr(upsampler, "conv", None), torch.nn.Conv2d):
+ return None
+ plan.append((upsampler.conv, False))
+ return plan, norms
+
+
+def enable_fp16_vae_decode(
+ pipe: Any,
+ target: Any,
+ logger: Any = None,
+) -> bool:
+ """Decode a force_upcast fp16 AutoencoderKL in fp16 on a U-Net pipeline. True when engaged."""
+ vae = getattr(pipe, "vae", None)
+ if (
+ vae is None
+ or getattr(pipe, "unet", None) is None
+ or getattr(target, "device", None) != "cuda"
+ # Measured on NVIDIA only; ROCm keeps the fp32 upcast.
+ or getattr(target, "backend", "cuda") != "cuda"
+ ):
+ return False
+ if getattr(vae, "_unsloth_fp16_decode", False):
+ return True
+ config = getattr(vae, "config", None)
+ if type(vae).__name__ != "AutoencoderKL" or not getattr(config, "force_upcast", False):
+ return False
+ # A decode compiled before this would sit inside the check, and its eager fallback would unwrap it.
+ if getattr(vae, "_unsloth_compiled_decode", False):
+ return False
+ try:
+ import torch
+
+ if getattr(vae, "dtype", None) is not torch.float16:
+ return False
+ found = _scale_plan(getattr(vae, "decoder", None))
+ post_quant_conv = getattr(vae, "post_quant_conv", None)
+ encode, decode = getattr(vae, "encode", None), getattr(vae, "decode", None)
+ if found is None or not callable(encode) or not callable(decode):
+ return False
+ plan, norms = found
+ scale = float(2**-_SCALE_LOG2)
+ with torch.no_grad():
+ for conv, divide_weight in plan:
+ if divide_weight:
+ conv.weight.mul_(scale)
+ if conv.bias is not None:
+ conv.bias.mul_(scale)
+ for norm in norms:
+ norm.eps = norm.eps * scale * scale
+ decode_parts = [m for m in (post_quant_conv, vae.decoder) if m is not None]
+ encode_parts = [
+ m
+ for m in (getattr(vae, "encoder", None), getattr(vae, "quant_conv", None))
+ if m is not None
+ ]
+ fell_back: list = []
+ # Decode compile goes in this slot: the check stays eager (no graph break) and outlives a fallback.
+ slot = types.SimpleNamespace(decode = decode)
+
+ def _fp32_call(fn: Any, parts: list, x: Any, *args: Any, **kwargs: Any) -> Any:
+ for part in parts:
+ part.to(torch.float32)
+ try:
+ return fn(x.to(torch.float32), *args, **kwargs)
+ finally:
+ for part in parts:
+ part.to(torch.float16)
+
+ @functools.wraps(decode)
+ def fp16_decode(z: Any, *args: Any, **kwargs: Any) -> Any:
+ if fell_back or not torch.is_tensor(z) or z.dtype is not torch.float16:
+ return slot.decode(z, *args, **kwargs)
+ out = slot.decode(z, *args, **kwargs)
+ sample = out[0] if isinstance(out, tuple) else getattr(out, "sample", out)
+ if not torch.is_tensor(sample) or bool(torch.isfinite(sample).all()):
+ return out
+ if logger is not None:
+ logger.warning(
+ "diffusion.vae: fp16 decode was not finite; decoding in fp32 from now on"
+ )
+ fell_back.append(True)
+ vae.register_to_config(force_upcast = True)
+ return _fp32_call(slot.decode, decode_parts, z, *args, **kwargs)
+
+ # Pipelines upcast the VAE for encode too; keep fp32 encoder math now the flag is off.
+ @functools.wraps(encode)
+ def fp32_encode(x: Any, *args: Any, **kwargs: Any) -> Any:
+ if fell_back or not torch.is_tensor(x) or x.dtype is not torch.float16:
+ return encode(x, *args, **kwargs)
+ return _fp32_call(encode, encode_parts, x, *args, **kwargs)
+
+ fp16_decode._unsloth_decode_slot = slot
+ vae.decode = fp16_decode
+ vae.encode = fp32_encode
+ vae.register_to_config(force_upcast = False)
+ vae._unsloth_fp16_decode = True
+ return True
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ if logger is not None:
+ logger.warning("diffusion.vae: fp16 decode not enabled: %s", exc)
+ return False
diff --git a/studio/backend/tests/test_diffusion_vae_fp16.py b/studio/backend/tests/test_diffusion_vae_fp16.py
new file mode 100644
index 00000000000..8f0d5f5c39c
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_vae_fp16.py
@@ -0,0 +1,211 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""fp16 decode for a force_upcast SDXL VAE: the rescale is exact, the flag flips, the encode keeps fp32 math, and a
+non-finite decode falls back to fp32 for good."""
+
+from __future__ import annotations
+
+import copy
+import logging
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+
+from core.inference import diffusion_vae_fp16 as vf # noqa: E402
+
+CUDA = types.SimpleNamespace(device = "cuda")
+
+
+def _tiny_vae(force_upcast = True):
+ torch.manual_seed(0)
+ return diffusers.AutoencoderKL(
+ in_channels = 3,
+ out_channels = 3,
+ down_block_types = ("DownEncoderBlock2D",) * 3,
+ up_block_types = ("UpDecoderBlock2D",) * 3,
+ block_out_channels = (16, 32, 32),
+ layers_per_block = 1,
+ latent_channels = 4,
+ norm_num_groups = 8,
+ force_upcast = force_upcast,
+ ).eval()
+
+
+def _pipe(vae, unet = True):
+ return types.SimpleNamespace(vae = vae, unet = object() if unet else None)
+
+
+def test_rescaled_decoder_is_the_same_function_in_fp32():
+ vae = _tiny_vae()
+ ref = copy.deepcopy(vae)
+ z = torch.randn(1, 4, 8, 8)
+ vae.to(torch.float16)
+ assert vf.enable_fp16_vae_decode(_pipe(vae), CUDA) is True
+ assert vae.config.force_upcast is False and vae._unsloth_fp16_decode is True
+ # Rescaled convs really moved (else the equality below is vacuous).
+ entry = vae.decoder.up_blocks[0].upsamplers[0].conv
+ assert not torch.equal(
+ entry.weight.float(), ref.decoder.up_blocks[0].upsamplers[0].conv.weight.half().float()
+ )
+ vae.to(torch.float32)
+ ref.to(torch.float16).to(torch.float32)
+ with torch.no_grad():
+ got = vae.decode(z, return_dict = False)[0]
+ want = ref.decode(z, return_dict = False)[0]
+ # Only the fp16 rounding of the smallest rescaled weights remains; a wrong shortcut or eps scale is off by ~0.4.
+ torch.testing.assert_close(got, want, rtol = 0, atol = 1e-3)
+
+
+def test_encode_keeps_fp32_math_and_restores_fp16_weights():
+ vae = _tiny_vae()
+ ref = copy.deepcopy(vae).to(torch.float16).to(torch.float32)
+ vae.to(torch.float16)
+ vf.enable_fp16_vae_decode(_pipe(vae), CUDA)
+ x = torch.randn(1, 3, 32, 32).to(torch.float16)
+ with torch.no_grad():
+ got = vae.encode(x).latent_dist.mean
+ want = ref.encode(x.float()).latent_dist.mean
+ assert got.dtype is torch.float32
+ torch.testing.assert_close(got, want)
+ assert (
+ vae.encoder.conv_in.weight.dtype is torch.float16
+ and vae.quant_conv.weight.dtype is torch.float16
+ )
+
+
+def test_non_finite_decode_reruns_fp32_and_stays_there(monkeypatch, caplog):
+ vae = _tiny_vae().to(torch.float16)
+ calls = []
+
+ def fake_decode(z, return_dict = True):
+ calls.append(z.dtype)
+ dtype = vae.decoder.conv_in.weight.dtype
+ fill = float("nan") if dtype is torch.float16 else 0.5
+ return (torch.full((1, 3, 4, 4), fill, dtype = dtype),)
+
+ vae.decode = fake_decode
+ logger = logging.getLogger("vae_fp16_test")
+ assert vf.enable_fp16_vae_decode(_pipe(vae), CUDA, logger = logger) is True
+ with caplog.at_level(logging.WARNING, logger = "vae_fp16_test"):
+ out = vae.decode(torch.zeros(1, 4, 2, 2, dtype = torch.float16), return_dict = False)[0]
+ assert torch.isfinite(out).all() and out.dtype is torch.float32
+ assert calls == [torch.float16, torch.float32]
+ assert vae.config.force_upcast is True
+ assert vae.decoder.conv_in.weight.dtype is torch.float16
+ assert "not finite" in caplog.text
+ vae.decode(torch.zeros(1, 4, 2, 2, dtype = torch.float16), return_dict = False)
+ assert calls[-1] is torch.float16 and len(calls) == 3
+
+
+ "case",
+ [
+ "no_unet",
+ "not_cuda",
+ "rocm",
+ "bf16",
+ "fp32",
+ "no_force_upcast",
+ "other_class",
+ "attention_in_up_block",
+ ],
+)
+def test_declines_and_leaves_the_vae_untouched(case):
+ vae = _tiny_vae(force_upcast = case != "no_force_upcast")
+ dtype = {"bf16": torch.bfloat16, "fp32": torch.float32}.get(case, torch.float16)
+ vae.to(dtype)
+ if case == "attention_in_up_block":
+ vae.decoder.up_blocks[1].attentions = torch.nn.ModuleList([torch.nn.Identity()])
+ if case == "other_class":
+ vae.__class__ = type("AutoencoderKLOther", (type(vae),), {})
+ before = {k: v.clone() for k, v in vae.state_dict().items()}
+ target = {
+ "not_cuda": types.SimpleNamespace(device = "mps"),
+ "rocm": types.SimpleNamespace(device = "cuda", backend = "rocm"),
+ }.get(case, CUDA)
+ assert vf.enable_fp16_vae_decode(_pipe(vae, unet = case != "no_unet"), target) is False
+ assert all(torch.equal(before[k], v) for k, v in vae.state_dict().items())
+ assert vae.config.force_upcast is (case != "no_force_upcast")
+ assert "decode" not in vae.__dict__
+
+
+def test_second_call_is_a_no_op():
+ vae = _tiny_vae().to(torch.float16)
+ pipe = _pipe(vae)
+ assert vf.enable_fp16_vae_decode(pipe, CUDA) is True
+ after_first = {k: v.clone() for k, v in vae.state_dict().items()}
+ assert vf.enable_fp16_vae_decode(pipe, CUDA) is True
+ assert all(torch.equal(after_first[k], v) for k, v in vae.state_dict().items())
+
+
+def _unet_pipe(vae):
+ return types.SimpleNamespace(vae = vae, unet = type("UNet2DConditionModel", (), {})())
+
+
+def test_studio_decode_compile_lands_inside_the_non_finite_check(monkeypatch):
+ from core.inference import diffusion_speed as ds
+
+ vae = _tiny_vae().to(torch.float16)
+ original = vae.decode
+ assert vf.enable_fp16_vae_decode(_unet_pipe(vae), CUDA) is True
+ wrapper = vae.__dict__["decode"]
+ compiled_targets = []
+
+ def fake_compile(fn, **kwargs):
+ compiled_targets.append(fn)
+ return fn
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+ assert ds._compile_vae_decode(_unet_pipe(vae), None) is True
+ # The data-dependent check stays eager (outermost); only the real decode is compiled, so no graph break.
+ assert vae.__dict__["decode"] is wrapper
+ assert compiled_targets == [original]
+ assert wrapper._unsloth_decode_slot.decode is not original
+
+
+def test_decode_compile_failure_keeps_the_non_finite_check(monkeypatch):
+ from core.inference import diffusion_speed as ds
+
+ vae = _tiny_vae().to(torch.float16)
+ assert vf.enable_fp16_vae_decode(_unet_pipe(vae), CUDA) is True
+ wrapper = vae.__dict__["decode"]
+
+ def failing_compile(fn, **kwargs):
+ def lowering_fails(*args, **kw):
+ raise torch._dynamo.exc.Unsupported("lowering failed")
+
+ return lowering_fails
+
+ monkeypatch.setattr(torch, "compile", failing_compile)
+ assert ds._compile_vae_decode(_unet_pipe(vae), None) is True
+ fills = [0.5, float("nan"), 0.5]
+ dtypes = []
+
+ def decoder_forward(*args, **kwargs):
+ dtype = vae.post_quant_conv.weight.dtype
+ dtypes.append(dtype)
+ return torch.full((1, 3, 8, 8), fills[len(dtypes) - 1], dtype = dtype)
+
+ monkeypatch.setattr(vae.decoder, "forward", decoder_forward)
+ z = torch.zeros(1, 4, 2, 2, dtype = torch.float16)
+ assert torch.isfinite(vae.decode(z, return_dict = False)[0]).all()
+ assert vae.__dict__["decode"] is wrapper and vae._unsloth_compile_decode_error
+ out = vae.decode(z, return_dict = False)[0]
+ assert torch.isfinite(out).all() and out.dtype is torch.float32
+ assert dtypes == [torch.float16, torch.float16, torch.float32]
+ assert vae.config.force_upcast is True
+
+
+def test_declines_once_a_compiled_decode_owns_the_vae(monkeypatch):
+ from core.inference import diffusion_speed as ds
+
+ vae = _tiny_vae().to(torch.float16)
+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)
+ assert ds._compile_vae_decode(_unet_pipe(vae), None) is True
+ # Wrapping now would leave the check behind the compile guard, whose eager fallback unwraps it (NaN frames).
+ assert vf.enable_fp16_vae_decode(_unet_pipe(vae), CUDA) is False
+ assert vae.config.force_upcast is True