Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: decode Wan video in fp16 with channels_last_3d convs (#11999)

Daniel Hanc8d146f7db99
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 39c1a2ee1f2..0dc3f987e93 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -372,6 +372,7 @@ def apply_speed_optims(
CUDA-graph arm refuses only on ``cache_engaged``: the caller bypasses per chunk if it toggles."""
applied = {
"channels_last": False,
+ "vae_fp16_decode": False,
"cudnn_benchmark": False,
"tf32": False,
"fp16_accum": False,
@@ -392,6 +393,8 @@ def apply_speed_optims(
# Lossless: a channels-last VAE speeds up its convs with no numeric change.
applied["channels_last"] = _vae_channels_last(pipe, logger)
+ # Near-lossless, not bit-identical, so never on "off" (returned above).
+ applied["vae_fp16_decode"] = _video_vae_half_decode(pipe, target, family, logger)
if on_cuda:
applied["cudnn_benchmark"] = _enable_cudnn_benchmark(logger)
@@ -503,6 +506,70 @@ def _vae_channels_last(pipe: Any, logger: Any) -> bool:
return False
+def _video_vae_half_decode(pipe: Any, target: Any, family: Any, logger: Any) -> bool:
+ """fp16 channels_last(_3d) decode for fp32-pinned video VAEs (Wan) on NVIDIA sm75+; non-finite output reruns fp32.
+
+ fp16, not bf16 (the pin exists because bf16 bands). channels_last_3d alone slows HV1.5 / LTX-2: keep it Wan-only.
+ """
+ if not getattr(family, "vae_force_fp32", False) or getattr(target, "device", None) != "cuda":
+ return False
+ vae = getattr(pipe, "vae", None)
+ decoder = getattr(vae, "decoder", None)
+ original = getattr(vae, "decode", None)
+ if decoder is None or not callable(original):
+ return False
+ # A dual-DiT family calls apply_speed_optims once per expert over the same VAE.
+ if getattr(vae, "_unsloth_half_decode", False):
+ return True
+ if getattr(target, "backend", None) != "cuda" or not _fp16_compile_capable(target):
+ return False
+ try:
+ import functools
+
+ import torch
+
+ parts = [m for m in (getattr(vae, "post_quant_conv", None), decoder) if m is not None]
+ for part in parts:
+ part.to(torch.float16)
+ for module in part.modules():
+ weight = getattr(module, "weight", None)
+ if isinstance(module, (torch.nn.Conv2d, torch.nn.Conv3d)) and weight is not None:
+ fmt = torch.channels_last if weight.dim() == 4 else torch.channels_last_3d
+ weight.data = weight.data.contiguous(memory_format = fmt)
+ fell_back: list = []
+
+ @functools.wraps(original)
+ def decode(z: Any, *args: Any, **kwargs: Any) -> Any:
+ if fell_back or not torch.is_tensor(z):
+ return original(z, *args, **kwargs)
+ out = original(z.to(torch.float16), *args, **kwargs)
+ sample = out[0] if isinstance(out, tuple) else getattr(out, "sample", out)
+ if not torch.is_tensor(sample) or bool(torch.isfinite(sample).all()):
+ return out
+ if logger is not None:
+ logger.warning(
+ "diffusion.speed: fp16 VAE decode was not finite; decoding in fp32 from now on"
+ )
+ for part in parts:
+ part.to(torch.float32)
+ fell_back.append(True)
+ return original(z, *args, **kwargs)
+
+ vae.decode = decode
+ vae._unsloth_half_decode = True
+ return True
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ try:
+ import torch
+ for part in (getattr(vae, "post_quant_conv", None), decoder):
+ if part is not None:
+ part.to(torch.float32)
+ except Exception: # noqa: BLE001, S110 - best-effort restore of the fp32 pin
+ pass
+ _warn(logger, "video vae fp16 decode", exc)
+ return False
+
+
# U-Net denoisers ship no ``_repeated_blocks``, so the regional compile cannot reach them; these classes get a
# WHOLE-module STATIC compile. SDXL (B200, 30 steps / 1024px): 26.9 vs 45.9 ms/step, 1.61x end-to-end at LPIPS 0.034.
# Static means a recompile per (height, width, batch).
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index c15e5a9ed56..3e3b074c4ed 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -416,6 +416,7 @@ def test_speed_off_applies_nothing(monkeypatch):
)
assert applied == {
"channels_last": False,
+ "vae_fp16_decode": False,
"cudnn_benchmark": False,
"tf32": False,
"fused_qkv": False,
diff --git a/studio/backend/tests/test_video_vae_fp16_decode.py b/studio/backend/tests/test_video_vae_fp16_decode.py
new file mode 100644
index 00000000000..741eed774da
--- /dev/null
+++ b/studio/backend/tests/test_video_vae_fp16_decode.py
@@ -0,0 +1,174 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""``_video_vae_half_decode`` on a tiny CPU stand-in for AutoencoderKLWan; capability probe monkeypatched."""
+
+from __future__ import annotations
+
+import logging
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_speed as ds_mod # noqa: E402
+from core.inference.diffusion_speed import SPEED_DEFAULT, SPEED_OFF, apply_speed_optims # noqa: E402
+
+
+class _FakeWanVAE(torch.nn.Module):
+ """Registration order matches AutoencoderKLWan, so ``dtype`` reads the (fp32) encoder like the real one."""
+
+ def __init__(self, blow_up = False) -> None:
+ super().__init__()
+ self.encoder = torch.nn.Conv3d(3, 4, 3)
+ self.quant_conv = torch.nn.Conv3d(4, 4, 1)
+ self.post_quant_conv = torch.nn.Conv3d(4, 4, 1)
+ self.decoder = torch.nn.Sequential(torch.nn.Conv3d(4, 4, 3), torch.nn.Conv2d(4, 4, 3))
+ self.blow_up = blow_up
+ self.seen: list = []
+
+ @property
+ def dtype(self):
+ return next(self.parameters()).dtype
+
+ def decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ weight = self.decoder[0].weight
+ self.seen.append((z.dtype, weight.dtype))
+ assert z.dtype == weight.dtype, "latents must reach the decoder in its own dtype"
+ out = z * 2
+ if self.blow_up and weight.dtype == torch.float16:
+ out = out * float("nan")
+ return (out,) if not return_dict else types.SimpleNamespace(sample = out)
+
+
+def _pipe(**kw):
+ return types.SimpleNamespace(vae = _FakeWanVAE(**kw))
+
+
+def _target(device = "cuda", backend = "cuda"):
+ return types.SimpleNamespace(device = device, backend = backend, dtype = torch.float16, ordinal = None)
+
+
+def _wan():
+ return types.SimpleNamespace(vae_force_fp32 = True, supports_torch_compile = False)
+
+
+def _snapshot(vae):
+ return {k: (v.dtype, v.stride(), v.detach().clone()) for k, v in vae.state_dict().items()}
+
+
+def _unchanged(vae, snap):
+ now = vae.state_dict()
+ return (
+ all(
+ now[k].dtype == d and now[k].stride() == s and torch.equal(now[k], t)
+ for k, (d, s, t) in snap.items()
+ )
+ and "decode" not in vae.__dict__
+ )
+
+
+def sm75(monkeypatch):
+ monkeypatch.setattr(ds_mod, "_fp16_compile_capable", lambda target: True)
+
+
+def test_engages_on_nvidia_for_a_pinned_family(sm75):
+ pipe = _pipe()
+ assert ds_mod._video_vae_half_decode(pipe, _target(), _wan(), None) is True
+ vae = pipe.vae
+ for part in (vae.post_quant_conv, vae.decoder):
+ assert all(p.dtype == torch.float16 for p in part.parameters())
+ assert vae.decoder[0].weight.is_contiguous(memory_format = torch.channels_last_3d)
+ assert not vae.decoder[0].weight.is_contiguous()
+ assert vae.decoder[1].weight.is_contiguous(memory_format = torch.channels_last)
+ assert vae.post_quant_conv.weight.is_contiguous(memory_format = torch.channels_last_3d)
+ # Encoder stays fp32 so the pipeline still reads an fp32 vae.dtype.
+ assert all(p.dtype == torch.float32 and p.is_contiguous() for p in vae.encoder.parameters())
+ assert vae.dtype == torch.float32
+ out = vae.decode(torch.randn(1, 4, 2, 5, 5), return_dict = False)[0]
+ assert out.dtype == torch.float16 and vae.seen[-1] == (torch.float16, torch.float16)
+ assert vae.decode(torch.randn(1, 4, 2, 5, 5)).sample.dtype == torch.float16
+
+
+def test_second_expert_call_is_idempotent(sm75):
+ pipe = _pipe()
+ assert ds_mod._video_vae_half_decode(pipe, _target(), _wan(), None) is True
+ wrapped = pipe.vae.decode
+ assert ds_mod._video_vae_half_decode(pipe, _target(), _wan(), None) is True
+ assert pipe.vae.decode is wrapped
+
+
+ "family, target",
+ [
+ (
+ types.SimpleNamespace(vae_force_fp32 = False),
+ _target(),
+ ), # LTX-2 / HV1.5 / H3 / image families
+ (types.SimpleNamespace(), _target()),
+ (_wan(), _target(backend = "rocm")),
+ (_wan(), _target(device = "mps", backend = "mps")),
+ (_wan(), _target(device = "cpu", backend = "cpu")),
+ (_wan(), _target(device = "xpu", backend = "xpu")),
+ (_wan(), types.SimpleNamespace(device = "cuda")), # no backend field: fail closed to fp32
+ ],
+)
+def test_everything_else_keeps_the_fp32_pin(sm75, family, target):
+ pipe = _pipe()
+ snap = _snapshot(pipe.vae)
+ assert ds_mod._video_vae_half_decode(pipe, target, family, None) is False
+ assert _unchanged(pipe.vae, snap)
+
+
+def test_below_sm75_keeps_the_fp32_pin(monkeypatch):
+ monkeypatch.setattr(ds_mod, "_fp16_compile_capable", lambda target: False)
+ pipe = _pipe()
+ snap = _snapshot(pipe.vae)
+ assert ds_mod._video_vae_half_decode(pipe, _target(), _wan(), None) is False
+ assert _unchanged(pipe.vae, snap)
+
+
+def test_non_finite_decode_reruns_and_stays_fp32(sm75, caplog):
+ pipe = _pipe(blow_up = True)
+ logger = logging.getLogger("test_vae_fp16")
+ assert ds_mod._video_vae_half_decode(pipe, _target(), _wan(), logger) is True
+ z = torch.randn(1, 4, 2, 5, 5)
+ with caplog.at_level(logging.WARNING, logger = "test_vae_fp16"):
+ out = pipe.vae.decode(z, return_dict = False)[0]
+ assert "not finite" in caplog.text
+ assert out.dtype == torch.float32 and torch.isfinite(out).all()
+ assert torch.equal(out, z * 2)
+ assert all(p.dtype == torch.float32 for p in pipe.vae.decoder.parameters())
+ pipe.vae.decode(z)
+ assert pipe.vae.seen[-1] == (torch.float32, torch.float32)
+
+
+def test_speed_layer_wires_it_and_off_does_not(monkeypatch):
+ calls = []
+ monkeypatch.setattr(
+ ds_mod,
+ "_video_vae_half_decode",
+ lambda pipe, target, family, logger: calls.append(family) or True,
+ )
+ for name in ("_vae_channels_last", "_enable_cudnn_benchmark", "_enable_fp16_accumulation"):
+ monkeypatch.setattr(ds_mod, name, lambda *a, **k: False)
+ pipe = types.SimpleNamespace(vae = None, transformer = types.SimpleNamespace())
+ fam = _wan()
+ target = types.SimpleNamespace(
+ device = "cpu", dtype = torch.float32, supports_default_torch_compile = False
+ )
+ assert (
+ apply_speed_optims(pipe, target, is_gguf = False, family = fam, speed_mode = SPEED_OFF)[
+ "vae_fp16_decode"
+ ]
+ is False
+ )
+ assert calls == []
+ applied = apply_speed_optims(pipe, target, is_gguf = False, family = fam, speed_mode = SPEED_DEFAULT)
+ assert applied["vae_fp16_decode"] is True and calls == [fam]