Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: decode the SDXL VAE in fp16 on fp16 GPUs (#12036)
Daniel Han83fbe4a34a3d
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex cdac7bf05b2..703e12ab6aa 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -154,6 +154,7 @@ from .diffusion_speed import (snapshot_backend_flags,vae_decode_compile_allowed,)+from .diffusion_vae_fp16 import enable_fp16_vae_decodefrom .diffusion_attention import (apply_attention_backend,normalize_attention_backend,@@ -6611,6 +6612,12 @@ class DiffusionBackend:)self._raise_if_load_cancelled(_load_token)+ # Before the speed optims so their decode compile lands inside the non-finite check; `off` keeps fp32.+ vae_fp16 = str(+ speed_mode or ""+ ).strip().lower() != SPEED_OFF and enable_fp16_vae_decode(+ pipe, target, logger = logger+ )speed_applied = apply_speed_optims(pipe,target,@@ -6622,6 +6629,8 @@ class DiffusionBackend:offload_active = plan.offload_policy != OFFLOAD_NONE,logger = logger,)+ if vae_fp16:+ speed_applied["vae_fp16_decode"] = Trueself._raise_if_load_cancelled(_load_token)if (transformer_quant_engaged is not None@@ -8316,6 +8325,8 @@ class DiffusionBackend:offload_active = state.offload_policy != OFFLOAD_NONE,logger = logger,)+ if getattr(getattr(state.pipe, "vae", None), "_unsloth_fp16_decode", False):+ speed_applied["vae_fp16_decode"] = Trueobject.__setattr__(state, "speed_mode", SPEED_DEFAULT)object.__setattr__(state, "speed_optims", tuple(k for k, v in speed_applied.items() if v))object.__setattr__(diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 80bf3a41a52..b8856c19712 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -1046,12 +1046,15 @@ def _guard_compiled_decode(eager: Any,logger: Any,eager_when_tiled: bool = False,+ owner: Any = None,) -> Any:"""``compiled`` behind an eager fallback: torch.compile is lazy, so lowering fails on the first call; OOMs still raise.``eager_when_tiled``: a tiled decode unrolls its tile loop into one graph (minutes of compile on low-VRAM loads),- and tiling is only settled by the memory plan after the compile, so it is read per call."""- had_own = "decode" in getattr(vae, "__dict__", {})+ and tiling is only settled by the memory plan after the compile, so it is read per call.+ ``owner``: the slot the fallback restores."""+ owner = vae if owner is None else owner+ had_own = "decode" in getattr(owner, "__dict__", {})failed: list = []def guarded(*args: Any, **kwargs: Any) -> Any:@@ -1071,9 +1074,9 @@ def _guard_compiled_decode(vae._unsloth_compile_decode_error = errorvae._unsloth_compiled_decode = Falseif had_own:- vae.decode = eager+ owner.decode = eagerelse:- del vae.decode+ del owner.decodeexcept Exception: # noqa: BLE001 - `failed` still routes this wrapper to eagerpassif logger is not None:@@ -1096,7 +1099,10 @@ def _compile_vae_decode() -> bool:"""torch.compile the VAE ``decode`` in place; no cudagraphs, whose capture would pin decode activations."""vae = getattr(pipe, "vae", None)- decode = getattr(vae, "decode", None) if vae is not None else None+ # An outer wrapper that must stay eager (fp16 non-finite check) exposes the slot it calls through.+ outer = getattr(vae, "__dict__", {}).get("decode") if vae is not None else None+ owner = getattr(outer, "_unsloth_decode_slot", None) or vae+ decode = getattr(owner, "decode", None) if vae is not None else Noneif not callable(decode):return False# A dual-DiT family calls apply_speed_optims twice over the same pipe.@@ -1111,8 +1117,8 @@ def _compile_vae_decode(if max_autotune:kwargs["mode"] = "max-autotune-no-cudagraphs"compiled = torch.compile(decode, **kwargs)- vae.decode = _guard_compiled_decode(- vae, compiled, decode, logger, eager_when_tiled = eager_when_tiled+ owner.decode = _guard_compiled_decode(+ vae, compiled, decode, logger, eager_when_tiled = eager_when_tiled, owner = owner)vae._unsloth_compiled_decode = Truereturn Truediff --git a/studio/backend/core/inference/diffusion_vae_fp16.py b/studio/backend/core/inference/diffusion_vae_fp16.pynew file mode 100644index 00000000000..0706915003e--- /dev/null+++ b/studio/backend/core/inference/diffusion_vae_fp16.py@@ -0,0 +1,159 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""fp16 VAE decode for U-Net (SDXL) pipelines whose VAE sets ``force_upcast``.++The SDXL VAE overflows fp16 in its last decoder stages, so the pipelines upcast the whole VAE to fp32 on fp16 loads.+Dividing the residual stream after the first upsampler by 2**8 (and the eps of each GroupNorm reading it by 2**16) is an+identity, since GroupNorm is scale invariant, and keeps every activation in fp16 range. A non-finite decode still reruns+in fp32 and hands later decodes back to the pipeline's upcast.+"""++from __future__ import annotations++import functools+import types+from typing import Any, Optional++_SCALE_LOG2 = 8+++def _scale_plan(decoder: Any) -> Optional[tuple]:+ """(convs, norms) scaling the residual stream after ``up_blocks[0]``'s upsampler; None for other layouts.++ ``(conv, divide_weight)``: unscaled input divides weight + bias, already-scaled input (shortcut, later upsamplers)+ only bias. ``norms`` read the scaled stream, so their eps shrinks by scale**2."""+ import torch++ up_blocks = list(getattr(decoder, "up_blocks", None) or ())+ if len(up_blocks) < 2 or not isinstance(+ getattr(decoder, "conv_norm_out", None), torch.nn.GroupNorm+ ):+ return None+ entry = list(getattr(up_blocks[0], "upsamplers", None) or ())+ if len(entry) != 1 or not isinstance(getattr(entry[0], "conv", None), torch.nn.Conv2d):+ return None+ plan = [(entry[0].conv, True)]+ norms = [decoder.conv_norm_out]+ for block in up_blocks[1:]:+ if getattr(block, "attentions", None):+ return None+ for resnet in getattr(block, "resnets", None) or ():+ if not (+ isinstance(getattr(resnet, "norm1", None), torch.nn.GroupNorm)+ and isinstance(getattr(resnet, "norm2", None), torch.nn.GroupNorm)+ and isinstance(getattr(resnet, "conv2", None), torch.nn.Conv2d)+ and getattr(resnet, "time_emb_proj", None) is None+ and getattr(resnet, "upsample", None) is None+ and getattr(resnet, "downsample", None) is None+ ):+ return None+ plan.append((resnet.conv2, True))+ norms.append(resnet.norm1)+ shortcut = getattr(resnet, "conv_shortcut", None)+ if shortcut is not None:+ if not isinstance(shortcut, torch.nn.Conv2d):+ return None+ plan.append((shortcut, False))+ for upsampler in getattr(block, "upsamplers", None) or ():+ if not isinstance(getattr(upsampler, "conv", None), torch.nn.Conv2d):+ return None+ plan.append((upsampler.conv, False))+ return plan, norms+++def enable_fp16_vae_decode(+ pipe: Any,+ target: Any,+ logger: Any = None,+) -> bool:+ """Decode a force_upcast fp16 AutoencoderKL in fp16 on a U-Net pipeline. True when engaged."""+ vae = getattr(pipe, "vae", None)+ if (+ vae is None+ or getattr(pipe, "unet", None) is None+ or getattr(target, "device", None) != "cuda"+ # Measured on NVIDIA only; ROCm keeps the fp32 upcast.+ or getattr(target, "backend", "cuda") != "cuda"+ ):+ return False+ if getattr(vae, "_unsloth_fp16_decode", False):+ return True+ config = getattr(vae, "config", None)+ if type(vae).__name__ != "AutoencoderKL" or not getattr(config, "force_upcast", False):+ return False+ # A decode compiled before this would sit inside the check, and its eager fallback would unwrap it.+ if getattr(vae, "_unsloth_compiled_decode", False):+ return False+ try:+ import torch++ if getattr(vae, "dtype", None) is not torch.float16:+ return False+ found = _scale_plan(getattr(vae, "decoder", None))+ post_quant_conv = getattr(vae, "post_quant_conv", None)+ encode, decode = getattr(vae, "encode", None), getattr(vae, "decode", None)+ if found is None or not callable(encode) or not callable(decode):+ return False+ plan, norms = found+ scale = float(2**-_SCALE_LOG2)+ with torch.no_grad():+ for conv, divide_weight in plan:+ if divide_weight:+ conv.weight.mul_(scale)+ if conv.bias is not None:+ conv.bias.mul_(scale)+ for norm in norms:+ norm.eps = norm.eps * scale * scale+ decode_parts = [m for m in (post_quant_conv, vae.decoder) if m is not None]+ encode_parts = [+ m+ for m in (getattr(vae, "encoder", None), getattr(vae, "quant_conv", None))+ if m is not None+ ]+ fell_back: list = []+ # Decode compile goes in this slot: the check stays eager (no graph break) and outlives a fallback.+ slot = types.SimpleNamespace(decode = decode)++ def _fp32_call(fn: Any, parts: list, x: Any, *args: Any, **kwargs: Any) -> Any:+ for part in parts:+ part.to(torch.float32)+ try:+ return fn(x.to(torch.float32), *args, **kwargs)+ finally:+ for part in parts:+ part.to(torch.float16)++ @functools.wraps(decode)+ def fp16_decode(z: Any, *args: Any, **kwargs: Any) -> Any:+ if fell_back or not torch.is_tensor(z) or z.dtype is not torch.float16:+ return slot.decode(z, *args, **kwargs)+ out = slot.decode(z, *args, **kwargs)+ sample = out[0] if isinstance(out, tuple) else getattr(out, "sample", out)+ if not torch.is_tensor(sample) or bool(torch.isfinite(sample).all()):+ return out+ if logger is not None:+ logger.warning(+ "diffusion.vae: fp16 decode was not finite; decoding in fp32 from now on"+ )+ fell_back.append(True)+ vae.register_to_config(force_upcast = True)+ return _fp32_call(slot.decode, decode_parts, z, *args, **kwargs)++ # Pipelines upcast the VAE for encode too; keep fp32 encoder math now the flag is off.+ @functools.wraps(encode)+ def fp32_encode(x: Any, *args: Any, **kwargs: Any) -> Any:+ if fell_back or not torch.is_tensor(x) or x.dtype is not torch.float16:+ return encode(x, *args, **kwargs)+ return _fp32_call(encode, encode_parts, x, *args, **kwargs)++ fp16_decode._unsloth_decode_slot = slot+ vae.decode = fp16_decode+ vae.encode = fp32_encode+ vae.register_to_config(force_upcast = False)+ vae._unsloth_fp16_decode = True+ return True+ except Exception as exc: # noqa: BLE001 - optimisation only+ if logger is not None:+ logger.warning("diffusion.vae: fp16 decode not enabled: %s", exc)+ return Falsediff --git a/studio/backend/tests/test_diffusion_vae_fp16.py b/studio/backend/tests/test_diffusion_vae_fp16.pynew file mode 100644index 00000000000..8f0d5f5c39c--- /dev/null+++ b/studio/backend/tests/test_diffusion_vae_fp16.py@@ -0,0 +1,211 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""fp16 decode for a force_upcast SDXL VAE: the rescale is exact, the flag flips, the encode keeps fp32 math, and a+non-finite decode falls back to fp32 for good."""++from __future__ import annotations++import copy+import logging+import types++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")++from core.inference import diffusion_vae_fp16 as vf # noqa: E402++CUDA = types.SimpleNamespace(device = "cuda")+++def _tiny_vae(force_upcast = True):+ torch.manual_seed(0)+ return diffusers.AutoencoderKL(+ in_channels = 3,+ out_channels = 3,+ down_block_types = ("DownEncoderBlock2D",) * 3,+ up_block_types = ("UpDecoderBlock2D",) * 3,+ block_out_channels = (16, 32, 32),+ layers_per_block = 1,+ latent_channels = 4,+ norm_num_groups = 8,+ force_upcast = force_upcast,+ ).eval()+++def _pipe(vae, unet = True):+ return types.SimpleNamespace(vae = vae, unet = object() if unet else None)+++def test_rescaled_decoder_is_the_same_function_in_fp32():+ vae = _tiny_vae()+ ref = copy.deepcopy(vae)+ z = torch.randn(1, 4, 8, 8)+ vae.to(torch.float16)+ assert vf.enable_fp16_vae_decode(_pipe(vae), CUDA) is True+ assert vae.config.force_upcast is False and vae._unsloth_fp16_decode is True+ # Rescaled convs really moved (else the equality below is vacuous).+ entry = vae.decoder.up_blocks[0].upsamplers[0].conv+ assert not torch.equal(+ entry.weight.float(), ref.decoder.up_blocks[0].upsamplers[0].conv.weight.half().float()+ )+ vae.to(torch.float32)+ ref.to(torch.float16).to(torch.float32)+ with torch.no_grad():+ got = vae.decode(z, return_dict = False)[0]+ want = ref.decode(z, return_dict = False)[0]+ # Only the fp16 rounding of the smallest rescaled weights remains; a wrong shortcut or eps scale is off by ~0.4.+ torch.testing.assert_close(got, want, rtol = 0, atol = 1e-3)+++def test_encode_keeps_fp32_math_and_restores_fp16_weights():+ vae = _tiny_vae()+ ref = copy.deepcopy(vae).to(torch.float16).to(torch.float32)+ vae.to(torch.float16)+ vf.enable_fp16_vae_decode(_pipe(vae), CUDA)+ x = torch.randn(1, 3, 32, 32).to(torch.float16)+ with torch.no_grad():+ got = vae.encode(x).latent_dist.mean+ want = ref.encode(x.float()).latent_dist.mean+ assert got.dtype is torch.float32+ torch.testing.assert_close(got, want)+ assert (+ vae.encoder.conv_in.weight.dtype is torch.float16+ and vae.quant_conv.weight.dtype is torch.float16+ )+++def test_non_finite_decode_reruns_fp32_and_stays_there(monkeypatch, caplog):+ vae = _tiny_vae().to(torch.float16)+ calls = []++ def fake_decode(z, return_dict = True):+ calls.append(z.dtype)+ dtype = vae.decoder.conv_in.weight.dtype+ fill = float("nan") if dtype is torch.float16 else 0.5+ return (torch.full((1, 3, 4, 4), fill, dtype = dtype),)++ vae.decode = fake_decode+ logger = logging.getLogger("vae_fp16_test")+ assert vf.enable_fp16_vae_decode(_pipe(vae), CUDA, logger = logger) is True+ with caplog.at_level(logging.WARNING, logger = "vae_fp16_test"):+ out = vae.decode(torch.zeros(1, 4, 2, 2, dtype = torch.float16), return_dict = False)[0]+ assert torch.isfinite(out).all() and out.dtype is torch.float32+ assert calls == [torch.float16, torch.float32]+ assert vae.config.force_upcast is True+ assert vae.decoder.conv_in.weight.dtype is torch.float16+ assert "not finite" in caplog.text+ vae.decode(torch.zeros(1, 4, 2, 2, dtype = torch.float16), return_dict = False)+ assert calls[-1] is torch.float16 and len(calls) == 3+++ "case",+ [+ "no_unet",+ "not_cuda",+ "rocm",+ "bf16",+ "fp32",+ "no_force_upcast",+ "other_class",+ "attention_in_up_block",+ ],+)+def test_declines_and_leaves_the_vae_untouched(case):+ vae = _tiny_vae(force_upcast = case != "no_force_upcast")+ dtype = {"bf16": torch.bfloat16, "fp32": torch.float32}.get(case, torch.float16)+ vae.to(dtype)+ if case == "attention_in_up_block":+ vae.decoder.up_blocks[1].attentions = torch.nn.ModuleList([torch.nn.Identity()])+ if case == "other_class":+ vae.__class__ = type("AutoencoderKLOther", (type(vae),), {})+ before = {k: v.clone() for k, v in vae.state_dict().items()}+ target = {+ "not_cuda": types.SimpleNamespace(device = "mps"),+ "rocm": types.SimpleNamespace(device = "cuda", backend = "rocm"),+ }.get(case, CUDA)+ assert vf.enable_fp16_vae_decode(_pipe(vae, unet = case != "no_unet"), target) is False+ assert all(torch.equal(before[k], v) for k, v in vae.state_dict().items())+ assert vae.config.force_upcast is (case != "no_force_upcast")+ assert "decode" not in vae.__dict__+++def test_second_call_is_a_no_op():+ vae = _tiny_vae().to(torch.float16)+ pipe = _pipe(vae)+ assert vf.enable_fp16_vae_decode(pipe, CUDA) is True+ after_first = {k: v.clone() for k, v in vae.state_dict().items()}+ assert vf.enable_fp16_vae_decode(pipe, CUDA) is True+ assert all(torch.equal(after_first[k], v) for k, v in vae.state_dict().items())+++def _unet_pipe(vae):+ return types.SimpleNamespace(vae = vae, unet = type("UNet2DConditionModel", (), {})())+++def test_studio_decode_compile_lands_inside_the_non_finite_check(monkeypatch):+ from core.inference import diffusion_speed as ds++ vae = _tiny_vae().to(torch.float16)+ original = vae.decode+ assert vf.enable_fp16_vae_decode(_unet_pipe(vae), CUDA) is True+ wrapper = vae.__dict__["decode"]+ compiled_targets = []++ def fake_compile(fn, **kwargs):+ compiled_targets.append(fn)+ return fn++ monkeypatch.setattr(torch, "compile", fake_compile)+ assert ds._compile_vae_decode(_unet_pipe(vae), None) is True+ # The data-dependent check stays eager (outermost); only the real decode is compiled, so no graph break.+ assert vae.__dict__["decode"] is wrapper+ assert compiled_targets == [original]+ assert wrapper._unsloth_decode_slot.decode is not original+++def test_decode_compile_failure_keeps_the_non_finite_check(monkeypatch):+ from core.inference import diffusion_speed as ds++ vae = _tiny_vae().to(torch.float16)+ assert vf.enable_fp16_vae_decode(_unet_pipe(vae), CUDA) is True+ wrapper = vae.__dict__["decode"]++ def failing_compile(fn, **kwargs):+ def lowering_fails(*args, **kw):+ raise torch._dynamo.exc.Unsupported("lowering failed")++ return lowering_fails++ monkeypatch.setattr(torch, "compile", failing_compile)+ assert ds._compile_vae_decode(_unet_pipe(vae), None) is True+ fills = [0.5, float("nan"), 0.5]+ dtypes = []++ def decoder_forward(*args, **kwargs):+ dtype = vae.post_quant_conv.weight.dtype+ dtypes.append(dtype)+ return torch.full((1, 3, 8, 8), fills[len(dtypes) - 1], dtype = dtype)++ monkeypatch.setattr(vae.decoder, "forward", decoder_forward)+ z = torch.zeros(1, 4, 2, 2, dtype = torch.float16)+ assert torch.isfinite(vae.decode(z, return_dict = False)[0]).all()+ assert vae.__dict__["decode"] is wrapper and vae._unsloth_compile_decode_error+ out = vae.decode(z, return_dict = False)[0]+ assert torch.isfinite(out).all() and out.dtype is torch.float32+ assert dtypes == [torch.float16, torch.float16, torch.float32]+ assert vae.config.force_upcast is True+++def test_declines_once_a_compiled_decode_owns_the_vae(monkeypatch):+ from core.inference import diffusion_speed as ds++ vae = _tiny_vae().to(torch.float16)+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)+ assert ds._compile_vae_decode(_unet_pipe(vae), None) is True+ # Wrapping now would leave the check behind the compile guard, whose eager fallback unwraps it (NaN frames).+ assert vf.enable_fp16_vae_decode(_unet_pipe(vae), CUDA) is False+ assert vae.config.force_upcast is True