Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run Z-Image and Wan2.2-5B / HunyuanVideo-1.5 in fp16 on fp16-only GPUs instead of fp32 (#12476)

Daniel Han77c193477e9b
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index e05c6066fe5..3d8ca806021 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -1164,14 +1164,16 @@ def _estimate_eta(total_steps: int, step: int, first_step_at: float, now: float)
def _resolve_diffusion_compute_dtype(fam: Optional[DiffusionFamily], dtype: Any) -> Any:
- """Promote float16 -> float32 for fp16-incompatible families (e.g. Z-Image), whose activations
- overflow float16's finite range and render a black image. Every other dtype/family passes
- through unchanged."""
+ """Promote float16 -> float32 for fp16-incompatible families without a usable fp16 guard; else unchanged."""
+ from .diffusion_fp16_guard import fp16_promotes_to_fp32
+
if fam is None or not getattr(fam, "fp16_incompatible", False):
return dtype
import torch
- return torch.float32 if dtype == torch.float16 else dtype
+ if dtype != torch.float16 or not fp16_promotes_to_fp32(fam):
+ return dtype
+ return torch.float32
def _float_load_itemsize(dtype: Any) -> Optional[int]:
@@ -6866,6 +6868,15 @@ class DiffusionBackend:
from .diffusion_flux2_rope import install_for_pipe as install_flux2_rope
install_flux2_rope(pipe, dtype, device, logger)
+ # fp16-only cards: a guarded family stays float16; patch its overflow sites.
+ from .diffusion_fp16_guard import family_fp16_guard, install_fp16_guard
+
+ install_fp16_guard(
+ getattr(pipe, "transformer", None),
+ family_fp16_guard(fam),
+ getattr(target, "dtype", None),
+ logger = logger,
+ )
self._raise_if_load_cancelled(_load_token)
# Pre-warmed torch.compile cache: a per-fingerprint inductor dir plus a bundle loaded before the
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 22552512151..8889e1158e0 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -120,6 +120,8 @@ class DiffusionFamily:
# True for families whose activations overflow float16 (-> black image); the backend promotes a resolved float16
# to float32.
fp16_incompatible: bool = False
+ # diffusion_fp16_guard recipe keeping an fp16_incompatible family in float16 on fp16-only cards; None = promote.
+ fp16_guard: Optional[str] = None
# false only for a family whose denoiser block does not compile cleanly with regional torch.compile
supports_torch_compile: bool = True
# Optional pre-quantized transformer checkpoints as (scheme, repo_id): fetched instead of the dense bf16 (lower
@@ -513,6 +515,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
img2img_pipeline_class = "ZImageImg2ImgPipeline",
inpaint_pipeline_class = "ZImageInpaintPipeline",
fp16_incompatible = True,
+ # The attention / FFN branches overflow float16 before their post-norm.
+ fp16_guard = "rescale_post_norm",
# Byte-identical mirror of Comfy-Org/z_image_turbo (AE + Qwen3-4B).
sd_cpp_vae = ("unsloth/Z-Image-Turbo-ComfyUI", "split_files/vae/ae.safetensors"),
sd_cpp_text_encoders = (
diff --git a/studio/backend/core/inference/diffusion_fp16_guard.py b/studio/backend/core/inference/diffusion_fp16_guard.py
new file mode 100644
index 00000000000..e27cc9b3f02
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_fp16_guard.py
@@ -0,0 +1,251 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Keep bf16-native DiTs in float16 on fp16-only cards (T4 / sm75) instead of promoting them to float32.
+
+``rescale_post_norm``: a branch that overflows float16 but feeds an RMSNorm is divided by a power of two s before the
+overflow point and that norm's eps by s**2, since ``rms_norm(y / s, eps / s**2) == rms_norm(y, eps)`` (no clamping).
+Attention scales its input (q / k RMSNorm eps scaled too; v and the bias-free out projection are linear); the FFN scales
+the ``w3`` half of the gate. Hooks sit on the parents and call ``to_q`` / ``w1`` / ... at run time, so a later LoRA or
+quantized Linear sees the same rescale. Kill switch: ``UNSLOTH_STUDIO_FP16_GUARD=0`` restores the float32 promotion.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Optional
+
+FP16_GUARD_ENV = "UNSLOTH_STUDIO_FP16_GUARD"
+_MARK = "_unsloth_fp16_guard"
+_FP16_MAX = 65504.0
+
+# recipe -> (attention, ffn) scale, powers of two (exact in float16); None = finite in plain float16, no patch.
+RECIPES: dict[str, Optional[tuple[float, float]]] = {
+ "native": None,
+ "rescale_post_norm": (16.0, 128.0),
+}
+
+
+def fp16_guard_disabled() -> bool:
+ return (os.environ.get(FP16_GUARD_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def family_fp16_guard(fam: Any) -> Optional[str]:
+ if fam is None or fp16_guard_disabled():
+ return None
+ recipe = getattr(fam, "fp16_guard", None)
+ return recipe if isinstance(recipe, str) and recipe in RECIPES else None
+
+
+# Checked before the dtype is chosen: a diffusers release that restructures the block keeps the float32 promotion.
+_RECIPE_TOKENS: dict[str, tuple[str, ...]] = {
+ "rescale_post_norm": (
+ "self.attention_norm2(",
+ "self.ffn_norm2(",
+ "self.feed_forward(",
+ "self.w2(",
+ "self.w3(",
+ "norm_q",
+ ),
+}
+_SUPPORTED: dict[tuple[str, str], bool] = {}
+
+
+def _recipe_supported(fam: Any, recipe: str) -> bool:
+ tokens = _RECIPE_TOKENS.get(recipe)
+ if not tokens:
+ return True
+ cls_name = getattr(fam, "transformer_class", None)
+ if not isinstance(cls_name, str) or not cls_name:
+ return False
+ key = (cls_name, recipe)
+ if key not in _SUPPORTED:
+ # Runs from dtype resolution, ahead of load_pipeline's own guard: `import diffusers` imports torch._dynamo.
+ try:
+ from loggers import get_logger
+ from utils.torch_warmup import close_dynamo_import_window
+ close_dynamo_import_window(get_logger(__name__))
+ except Exception: # noqa: BLE001, S110 - optimisation only
+ pass
+ try:
+ import importlib
+ import inspect
+ import sys
+
+ cls = getattr(importlib.import_module("diffusers"), cls_name)
+ src = inspect.getsource(sys.modules[cls.__module__])
+ _SUPPORTED[key] = all(tok in src for tok in tokens)
+ except Exception: # noqa: BLE001 - unreadable source: keep the safe float32 promotion
+ _SUPPORTED[key] = False
+ return _SUPPORTED[key]
+
+
+def fp16_promotes_to_fp32(fam: Any) -> bool:
+ """True when a resolved float16 must be promoted to float32 for this family: fp16-incompatible with no usable
+ guard (none declared, the kill switch is set, or the installed diffusers block no longer matches the recipe)."""
+ if fam is None or not getattr(fam, "fp16_incompatible", False):
+ return False
+ recipe = family_fp16_guard(fam)
+ return recipe is None or not _recipe_supported(fam, recipe)
+
+
+def _is_fp16(dtype: Any) -> bool:
+ try:
+ import torch
+ return dtype is torch.float16 or dtype == torch.float16
+ except Exception: # noqa: BLE001 - no torch, nothing to guard
+ return False
+
+
+def _scale_eps(norm: Any, factor: float) -> bool:
+ eps = getattr(norm, "eps", None)
+ if not isinstance(eps, float) or isinstance(eps, bool):
+ return False
+ norm.eps = eps * factor
+ return True
+
+
+def _finite(x: Any) -> Any:
+ import torch
+ if x.dtype is torch.float16:
+ return torch.nan_to_num(x, nan = 0.0, posinf = _FP16_MAX, neginf = -_FP16_MAX)
+ return x
+
+
+def _attention_pre_hook(inv: float):
+ def hook(module, args, kwargs):
+ if args:
+ return (args[0] * inv, *args[1:]), kwargs
+ if "hidden_states" in kwargs:
+ kwargs = dict(kwargs)
+ kwargs["hidden_states"] = kwargs["hidden_states"] * inv
+ return args, kwargs
+ return None
+
+ return hook
+
+
+def _attention_post_hook(module, args, output):
+ import torch
+ if isinstance(output, torch.Tensor):
+ return _finite(output)
+ return None
+
+
+def _make_ffn_forward(inv: float):
+ def forward(self, x):
+ import torch.nn.functional as F
+ return self.w2(_finite(F.silu(self.w1(x)) * (self.w3(x) * inv)))
+
+ return forward
+
+
+def _block_sites(block: Any) -> Optional[tuple[Any, ...]]:
+ attn = getattr(block, "attention", None)
+ ffn = getattr(block, "feed_forward", None)
+ norms = tuple(getattr(block, n, None) for n in ("attention_norm2", "ffn_norm2"))
+ if attn is None or ffn is None or any(n is None for n in norms):
+ return None
+ if any(
+ getattr(attn, n, None) is None
+ for n in ("norm_q", "norm_k", "to_q", "to_k", "to_v", "to_out")
+ ):
+ return None
+ if any(getattr(ffn, n, None) is None for n in ("w1", "w2", "w3")):
+ return None
+ # The rescale is exact only through bias-free projections.
+ linears = [attn.to_q, attn.to_k, attn.to_v, attn.to_out[0], ffn.w2]
+ if any(getattr(lin, "bias", None) is not None for lin in linears):
+ return None
+ eps = [getattr(m, "eps", None) for m in (attn.norm_q, attn.norm_k, *norms)]
+ if not all(isinstance(e, float) for e in eps):
+ return None
+ return attn, ffn, norms[0], norms[1]
+
+
+def _install_rescale_post_norm(root: Any, attn_scale: float, ffn_scale: float) -> int:
+ import types
+
+ sites = 0
+ for _name, block in root.named_modules():
+ if block.__dict__.get(_MARK) is not None:
+ sites += 1
+ continue
+ found = _block_sites(block)
+ if found is None:
+ continue
+ attn, ffn, attn_norm2, ffn_norm2 = found
+ a_inv, f_inv = 1.0 / attn_scale, 1.0 / ffn_scale
+ _scale_eps(attn.norm_q, a_inv * a_inv)
+ _scale_eps(attn.norm_k, a_inv * a_inv)
+ _scale_eps(attn_norm2, a_inv * a_inv)
+ _scale_eps(ffn_norm2, f_inv * f_inv)
+ handles = (
+ attn.register_forward_pre_hook(_attention_pre_hook(a_inv), with_kwargs = True),
+ attn.register_forward_hook(_attention_post_hook),
+ )
+ ffn.forward = types.MethodType(_make_ffn_forward(f_inv), ffn)
+ block.__dict__[_MARK] = (attn_scale, ffn_scale, handles)
+ sites += 1
+ return sites
+
+
+def install_fp16_guard(
+ root: Any,
+ recipe: Optional[str],
+ dtype: Any,
+ logger: Any = None,
+) -> int:
+ """Install ``recipe`` on every matching block under ``root`` when ``dtype`` is float16. Returns the number of
+ guarded blocks (0 = not engaged). Idempotent."""
+ if root is None or recipe not in RECIPES or not _is_fp16(dtype) or fp16_guard_disabled():
+ return 0
+ scales = RECIPES[recipe]
+ if scales is None:
+ if logger is not None:
+ logger.info("diffusion.fp16_guard: recipe=%s (float16 kept, no patch needed)", recipe)
+ return 0
+ if not callable(getattr(root, "named_modules", None)):
+ return 0
+ attn_scale, ffn_scale = scales
+ sites = _install_rescale_post_norm(root, attn_scale, ffn_scale)
+ if logger is not None:
+ if sites:
+ logger.info(
+ "diffusion.fp16_guard: recipe=%s blocks=%d (float16 kept, no float32 promotion)",
+ recipe,
+ sites,
+ )
+ else:
+ logger.warning(
+ "diffusion.fp16_guard: recipe=%s matched no block; float16 output may overflow "
+ "(set %s=0 to promote to float32)",
+ recipe,
+ FP16_GUARD_ENV,
+ )
+ return sites
+
+
+def remove_fp16_guard(root: Any) -> int:
+ """Undo ``install_fp16_guard`` on ``root`` (eps restored, hooks and forward overrides dropped)."""
+ removed = 0
+ if root is None:
+ return 0
+ for _name, block in root.named_modules():
+ mark = block.__dict__.pop(_MARK, None)
+ if mark is None:
+ continue
+ attn_scale, ffn_scale, handles = mark
+ for handle in handles:
+ handle.remove()
+ attn, ffn = block.attention, block.feed_forward
+ for norm, scale in (
+ (attn.norm_q, attn_scale),
+ (attn.norm_k, attn_scale),
+ (block.attention_norm2, attn_scale),
+ (block.ffn_norm2, ffn_scale),
+ ):
+ _scale_eps(norm, scale * scale)
+ ffn.__dict__.pop("forward", None)
+ removed += 1
+ return removed
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 92572cc75f6..3781f5e014d 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -335,7 +335,8 @@ def compile_eligible(target: Any, *, is_gguf: bool, family: Any) -> bool:
dtype = getattr(target, "dtype", None)
if _is_bfloat16(dtype):
return True
- # fp16-incompatible families run in fp32 though video passes the fp16 target; fp32 compile is unmeasured.
+ # fp16-incompatible families run in fp32 (unmeasured under compile) or in fp16 behind diffusion_fp16_guard, which
+ # must stay eager: the int8 fused MLP installed for compile replaces the guard's FFN forward.
if bool(getattr(family, "fp16_incompatible", False)):
return False
return _is_float16(dtype) and _fp16_compile_capable(target)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index b25f7fab4f6..d9debeeac88 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -119,6 +119,7 @@ from .diffusion_memory import (
from .diffusion_torchao_patches import install_torchao_int_mm_patch
from .media_decode_phase import decode_phase as _decode_phase
from . import diffusion_render_thread as render_thread
+from .diffusion_fp16_guard import fp16_promotes_to_fp32
from .diffusion_speed import (
SPEED_DEFAULT,
SPEED_EAGER,
@@ -3919,7 +3920,7 @@ class VideoBackend:
else resolve_diffusion_device_target(ordinal = gpu_ordinal)
)
dtype = target.dtype
- if getattr(fam, "fp16_incompatible", False) and dtype is torch.float16:
+ if dtype is torch.float16 and fp16_promotes_to_fp32(fam):
dtype = torch.float32
return _h3_auto_denoiser_scheme(
fam,
@@ -3960,7 +3961,7 @@ class VideoBackend:
if gpu_ordinal is None
else resolve_diffusion_device_target(ordinal = gpu_ordinal)
)
- if getattr(fam, "fp16_incompatible", False) and target.dtype is torch.float16:
+ if target.dtype is torch.float16 and fp16_promotes_to_fp32(fam):
# The loader promotes fp16 to float32 here, so read the dtype the load will.
return None
scheme = _video_auto_denoiser_scheme(
@@ -5393,10 +5394,17 @@ class VideoBackend:
)
# Bound only at the commit past the token check: a superseded load may return from the install late.
_nvfp4_install_outcome: Optional[tuple[bool, str]] = None
- # Video DiTs are bf16-native; fp16 overflows, so a resolved fp16 promotes to float32.
+ # Video DiTs are bf16-native: fp16 promotes to float32 unless the family declares an fp16_guard.
dtype = target.dtype
- if fam.fp16_incompatible and dtype is torch.float16:
+ if dtype is torch.float16 and fp16_promotes_to_fp32(fam):
dtype = torch.float32
+ logger.warning(
+ "video.dtype_promoted: family=%s float16 -> float32 (fp16-incompatible)", fam.name
+ )
+ elif fam.fp16_incompatible and dtype is torch.float16:
+ logger.info(
+ "video.fp16_guard: family=%s recipe=%s (float16 kept)", fam.name, fam.fp16_guard
+ )
# Size tables below are bf16 (2-byte), so scale dense estimates when the promotion lands fp32 on an accelerator
dtype_scale = 2.0 if device != "cpu" and dtype is torch.float32 else 1.0
diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py
index 5a13d7f1480..0739f3c2e61 100644
--- a/studio/backend/core/inference/video_families.py
+++ b/studio/backend/core/inference/video_families.py
@@ -85,6 +85,8 @@ class VideoFamily:
supports_cuda_graph: bool = False
# Video DiTs are bf16-native, so fp16 promotes to float32; defaults True.
fp16_incompatible: bool = True
+ # "native" = measured accurate in plain float16 on fp16-only cards; None = promote (unmeasured).
+ fp16_guard: Optional[str] = None
# Wan VAE decodes in float32 (bf16 causes banding / black frames), so the loader pins it back. Its size term is
# already fp32.
vae_force_fp32: bool = False
@@ -286,6 +288,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (
# (11.4); VAE fp32 (2.8).
bf16_components_gb = (10.0, 11.4, 2.8),
vae_force_fp32 = True,
+ # UMT5 keeps its overflowing `wo` in fp32 itself; the VAE stays fp32 (vae_force_fp32).
+ fp16_guard = "native",
# Byte-identical mirror of QuantStack/Wan2.2-TI2V-5B-GGUF (13 quants + companion VAE).
gguf_repo = "unsloth/Wan2.2-TI2V-5B-GGUF",
),
@@ -354,6 +358,7 @@ _FAMILIES: tuple[VideoFamily, ...] = (
resolution_presets = ((832, 480), (480, 832), (640, 640)),
# DiT fp32 on disk (32.0 to 16.6 bf16); VAE 4.7 to 2.4; Qwen2.5-VL TE bf16 14.0 + ByT5 0.8
bf16_components_gb = (16.6, 14.8, 2.4),
+ fp16_guard = "native",
),
# The 720p t2v repack: same architecture and footprint as the 480p entry, only the trained resolution differs. Its
# own family so a 720p load defaults to 720p sizes; the full-path alias outranks the generic token.
@@ -378,6 +383,7 @@ _FAMILIES: tuple[VideoFamily, ...] = (
# 720p-class presets: landscape, vertical, square (all /16).
resolution_presets = ((1280, 720), (720, 1280), (960, 960)),
bf16_components_gb = (16.6, 14.8, 2.4),
+ fp16_guard = "native",
),
)
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index e84f9387477..f60f1eb8556 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -3866,11 +3866,16 @@ def test_zimage_is_fp16_incompatible():
assert detect_family("unsloth/FLUX.2-klein-4B-GGUF").fp16_incompatible is False
-def test_resolve_compute_dtype_promotes_fp16_for_zimage(fake_runtime):
+def test_resolve_compute_dtype_promotes_fp16_for_zimage(fake_runtime, monkeypatch):
+ from core.inference import diffusion_fp16_guard as guard
+
torch = sys.modules["torch"]
z = detect_family("unsloth/Z-Image-GGUF")
q = detect_family("unsloth/FLUX.1-schnell-GGUF")
- # Z-Image: fp16 promotes to fp32; bf16 / fp32 pass through unchanged.
+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: True)
+ assert _resolve_diffusion_compute_dtype(z, torch.float16) is torch.float16
+ # Kill switch: fp16 promotes to fp32; bf16 / fp32 unchanged.
+ monkeypatch.setenv(guard.FP16_GUARD_ENV, "0")
assert _resolve_diffusion_compute_dtype(z, torch.float16) is torch.float32
assert _resolve_diffusion_compute_dtype(z, torch.bfloat16) is torch.bfloat16
assert _resolve_diffusion_compute_dtype(z, torch.float32) is torch.float32
@@ -3880,13 +3885,22 @@ def test_resolve_compute_dtype_promotes_fp16_for_zimage(fake_runtime):
def test_load_promotes_fp16_to_fp32_for_zimage_only(fake_runtime, monkeypatch, tmp_path):
+ from core.inference import diffusion_fp16_guard as guard
+
torch = sys.modules["torch"]
- # Pre-Ampere CUDA resolves to fp16, so the guard must promote Z-Image (and only fp16-incompatible families) to fp32
- # or it renders black.
+ # Pre-Ampere resolves fp16: Z-Image stays fp16 behind its guard; the kill switch promotes it to fp32.
monkeypatch.setattr(torch.cuda, "is_available", lambda: True, raising = False)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda: (7, 5), raising = False)
(tmp_path / "m.gguf").write_bytes(b"x")
+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: True)
+ kept = DiffusionBackend().load_pipeline(
+ str(tmp_path), gguf_filename = "m.gguf", family_override = "z-image"
+ )
+ assert kept["dtype"] == "float16"
+ assert str(_FakeTransformer.last["torch_dtype"]) == "torch.float16"
+
+ monkeypatch.setenv(guard.FP16_GUARD_ENV, "0")
z = DiffusionBackend().load_pipeline(
str(tmp_path), gguf_filename = "m.gguf", family_override = "z-image"
)
diff --git a/studio/backend/tests/test_diffusion_fp16_guard.py b/studio/backend/tests/test_diffusion_fp16_guard.py
new file mode 100644
index 00000000000..03760bb5f78
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_fp16_guard.py
@@ -0,0 +1,358 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""fp16 guard for fp16-only cards: capability resolution, kill switch, bf16 untouched, and the post-norm rescale on a
+tiny block shaped like Z-Image's (CPU, real torch)."""
+
+from __future__ import annotations
+
+import pytest
+
+torch = pytest.importorskip("torch")
+nn = torch.nn
+F = torch.nn.functional
+
+from core.inference import diffusion_fp16_guard as guard # noqa: E402
+from core.inference.diffusion import _resolve_diffusion_compute_dtype # noqa: E402
+from core.inference.diffusion_families import _FAMILIES, detect_family # noqa: E402
+from core.inference.video_families import _FAMILIES as _VIDEO_FAMILIES # noqa: E402
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ monkeypatch.delenv(guard.FP16_GUARD_ENV, raising = False)
+ # The structural probe reads the installed diffusers; pin it so the resolution tests do not depend on it.
+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: True)
+
+
+def _video(name):
+ return next(f for f in _VIDEO_FAMILIES if f.name == name)
+
+
+def test_every_declared_recipe_is_known():
+ for fam in (*_FAMILIES, *_VIDEO_FAMILIES):
+ recipe = getattr(fam, "fp16_guard", None)
+ assert recipe is None or recipe in guard.RECIPES, (fam.name, recipe)
+ # a guard only means something on a family that would otherwise be promoted
+ assert recipe is None or fam.fp16_incompatible, fam.name
+
+
+def test_zimage_stays_fp16_with_guard():
+ z = detect_family("Tongyi-MAI/Z-Image-Turbo")
+ assert z.fp16_incompatible is True and z.fp16_guard == "rescale_post_norm"
+ assert guard.fp16_promotes_to_fp32(z) is False
+ assert _resolve_diffusion_compute_dtype(z, torch.float16) is torch.float16
+
+
+def test_unguarded_incompatible_family_still_promotes():
+ krea = detect_family("krea/Krea-2-Turbo")
+ assert krea.fp16_incompatible is True and krea.fp16_guard is None
+ assert _resolve_diffusion_compute_dtype(krea, torch.float16) is torch.float32
+
+
+def test_video_families_declare_only_native():
+ # The video loader resolves the dtype but installs no guard hooks, so a patching recipe there would render black.
+ for fam in _VIDEO_FAMILIES:
+ assert fam.fp16_guard in (None, "native"), fam.name
+
+
+def test_video_capability_flags():
+ for name in ("wan2.2-ti2v-5b", "hunyuanvideo-1.5", "hunyuanvideo-1.5-720p"):
+ assert guard.fp16_promotes_to_fp32(_video(name)) is False, name
+ # unmeasured families, and A14B (fp16 drifts from fp32 as far as bf16 does), keep the blanket promotion
+ for name in ("ltx-2", "minimax-h3", "wan2.2-t2v-a14b"):
+ assert guard.fp16_promotes_to_fp32(_video(name)) is True, name
+
+
+def test_kill_switch_restores_promotion(monkeypatch):
+ z = detect_family("Tongyi-MAI/Z-Image-Turbo")
+ monkeypatch.setenv(guard.FP16_GUARD_ENV, "0")
+ assert guard.fp16_promotes_to_fp32(z) is True
+ assert _resolve_diffusion_compute_dtype(z, torch.float16) is torch.float32
+ assert guard.fp16_promotes_to_fp32(_video("wan2.2-ti2v-5b")) is True
+
+
+def test_unsupported_diffusers_block_keeps_promotion(monkeypatch):
+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: False)
+ z = detect_family("Tongyi-MAI/Z-Image-Turbo")
+ assert _resolve_diffusion_compute_dtype(z, torch.float16) is torch.float32
+ # "native" needs no patch, so no structural probe either
+ monkeypatch.undo()
+ monkeypatch.setattr(guard, "_SUPPORTED", {})
+ assert guard._recipe_supported(_video("wan2.2-ti2v-5b"), "native") is True
+
+
+def test_bf16_and_fp32_never_change():
+ for fam in (*_FAMILIES, None):
+ for dt in (torch.bfloat16, torch.float32):
+ assert _resolve_diffusion_compute_dtype(fam, dt) is dt
+
+
+def test_non_fp16_resolution_never_probes_diffusers(monkeypatch):
+ def probe(fam, recipe):
+ raise AssertionError("bf16 / fp32 must not import diffusers")
+
+ monkeypatch.setattr(guard, "_recipe_supported", probe)
+ z = detect_family("Tongyi-MAI/Z-Image-Turbo")
+ assert _resolve_diffusion_compute_dtype(z, torch.bfloat16) is torch.bfloat16
+ assert _resolve_diffusion_compute_dtype(z, torch.float32) is torch.float32
+
+
+def test_probe_closes_dynamo_import_window_before_diffusers(monkeypatch):
+ import importlib
+
+ from utils import torch_warmup
+
+ monkeypatch.undo()
+ monkeypatch.setattr(guard, "_SUPPORTED", {})
+ order = []
+ monkeypatch.setattr(
+ torch_warmup, "close_dynamo_import_window", lambda log: order.append("close")
+ )
+ real = importlib.import_module
+
+ def tracking(name, *a, **k):
+ if name == "diffusers":
+ order.append("diffusers")
+ return real(name, *a, **k)
+
+ monkeypatch.setattr(importlib, "import_module", tracking)
+ guard._recipe_supported(detect_family("Tongyi-MAI/Z-Image-Turbo"), "rescale_post_norm")
+ assert order[:2] == ["close", "diffusers"], order
+
+
+class _RMS(nn.Module):
+ def __init__(
+ self,
+ dim,
+ eps = 1e-5,
+ ):
+ super().__init__()
+ self.eps = eps
+ self.weight = nn.Parameter(torch.ones(dim))
+
+ def forward(self, x):
+ var = x.float().pow(2).mean(-1, keepdim = True)
+ return (x.float() * torch.rsqrt(var + self.eps)).to(x.dtype) * self.weight.to(x.dtype)
+
+
+class _Attn(nn.Module):
+ def __init__(self, dim, heads):
+ super().__init__()
+ self.heads = heads
+ self.to_q = nn.Linear(dim, dim, bias = False)
+ self.to_k = nn.Linear(dim, dim, bias = False)
+ self.to_v = nn.Linear(dim, dim, bias = False)
+ self.to_out = nn.ModuleList([nn.Linear(dim, dim, bias = False), nn.Dropout(0.0)])
+ self.norm_q = _RMS(dim // heads)
+ self.norm_k = _RMS(dim // heads)
+
+ def forward(
+ self,
+ hidden_states,
+ attention_mask = None,
+ ):
+ b, s, d = hidden_states.shape
+ q, k, v = (
+ p(hidden_states).view(b, s, self.heads, -1) for p in (self.to_q, self.to_k, self.to_v)
+ )
+ q, k = self.norm_q(q), self.norm_k(k)
+ w = torch.softmax(
+ (q.float().transpose(1, 2) @ k.float().permute(0, 2, 3, 1)) / (d // self.heads) ** 0.5,
+ -1,
+ )
+ o = (w.to(v.dtype) @ v.transpose(1, 2)).transpose(1, 2).reshape(b, s, d)
+ return self.to_out[1](self.to_out[0](o))
+
+
+class _FFN(nn.Module):
+ def __init__(self, dim, hidden):
+ super().__init__()
+ self.w1 = nn.Linear(dim, hidden, bias = False)
+ self.w2 = nn.Linear(hidden, dim, bias = False)
+ self.w3 = nn.Linear(dim, hidden, bias = False)
+
+ def forward(self, x):
+ return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+
+class _Block(nn.Module):
+ def __init__(
+ self,
+ dim = 64,
+ heads = 4,
+ hidden = 128,
+ ):
+ super().__init__()
+ self.attention = _Attn(dim, heads)
+ self.feed_forward = _FFN(dim, hidden)
+ self.attention_norm1, self.attention_norm2 = _RMS(dim), _RMS(dim)
+ self.ffn_norm1, self.ffn_norm2 = _RMS(dim), _RMS(dim)
+
+ def forward(self, x):
+ x = x + 0.5 * self.attention_norm2(self.attention(self.attention_norm1(x)))
+ return x + 0.5 * self.ffn_norm2(self.feed_forward(self.ffn_norm1(x)))
+
+
+def _overflowing_model():
+ torch.manual_seed(0)
+ model = nn.Sequential(_Block(), _Block())
+ with torch.no_grad():
+ for blk in model:
+ # Z-Image-like: the FFN gate product and output, and the attention output, exceed float16's range.
+ blk.feed_forward.w1.weight.mul_(40.0)
+ blk.feed_forward.w3.weight.mul_(40.0)
+ blk.feed_forward.w2.weight.mul_(400.0)
+ blk.attention.to_v.weight.mul_(300.0)
+ blk.attention.to_out[0].weight.mul_(300.0)
+ return model
+
+
+def _fp16_ok():
+ try:
+ nn.Linear(4, 4).half()(torch.ones(1, 4, dtype = torch.float16))
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+needs_fp16 = pytest.mark.skipif(not _fp16_ok(), reason = "this torch has no CPU float16 matmul")
+
+
+@needs_fp16
+def test_guard_keeps_overflowing_block_finite_and_accurate():
+ model = _overflowing_model()
+ x = torch.randn(2, 16, 64)
+ with torch.no_grad():
+ ref = model.float()(x.float())
+ assert ref.abs().max() < 1e3 # the residual stream itself is in range
+ branch = model[0].feed_forward(model[0].ffn_norm1(x.float()))
+ assert branch.abs().max() > 65504 # ...the pre-norm branch is not
+
+ half = model.half()
+ stock = half(x.half())
+ assert not torch.isfinite(stock).all() # stock fp16 overflows (black image)
+
+ assert guard.install_fp16_guard(half, "rescale_post_norm", torch.float16) == 2
+ fixed = half(x.half())
+ assert torch.isfinite(fixed).all()
+ err = (fixed.float() - ref).abs().max() / ref.abs().max()
+ assert err < 2e-2, float(err)
+
+
+@needs_fp16
+def test_guard_is_idempotent_and_removable():
+ model = _overflowing_model().half()
+ eps_before = [m.eps for m in model.modules() if isinstance(m, _RMS)]
+ assert guard.install_fp16_guard(model, "rescale_post_norm", torch.float16) == 2
+ assert guard.install_fp16_guard(model, "rescale_post_norm", torch.float16) == 2
+ assert model[0].attention_norm2.eps == 1e-5 / 16**2
+ assert model[0].ffn_norm2.eps == 1e-5 / 128**2
+ assert guard.remove_fp16_guard(model) == 2
+ assert [m.eps for m in model.modules() if isinstance(m, _RMS)] == eps_before
+ assert "forward" not in model[0].feed_forward.__dict__
+ assert not model[0].attention._forward_pre_hooks and not model[0].attention._forward_hooks
+
+
[email protected]("dtype", ["bfloat16", "float32"])
+def test_guard_never_touches_non_fp16(dtype):
+ dt = getattr(torch, dtype)
+ torch.manual_seed(0)
+ model = nn.Sequential(_Block(), _Block()).to(dt)
+ x = torch.randn(2, 16, 64, dtype = dt)
+ with torch.no_grad():
+ before = model(x)
+ assert guard.install_fp16_guard(model, "rescale_post_norm", dt) == 0
+ after = model(x)
+ assert torch.equal(before, after)
+ assert all(m.eps == 1e-5 for m in model.modules() if isinstance(m, _RMS))
+ assert not model[0].attention._forward_pre_hooks
+
+
+@needs_fp16
+def test_guard_inert_under_kill_switch_and_native(monkeypatch):
+ model = _overflowing_model().half()
+ assert guard.install_fp16_guard(model, "native", torch.float16) == 0
+ assert guard.install_fp16_guard(model, None, torch.float16) == 0
+ monkeypatch.setenv(guard.FP16_GUARD_ENV, "0")
+ assert guard.install_fp16_guard(model, "rescale_post_norm", torch.float16) == 0
+ assert not model[0].attention._forward_pre_hooks
+
+
+def test_block_with_bias_is_not_guarded():
+ torch.manual_seed(0)
+ blk = _Block()
+ blk.feed_forward.w2 = nn.Linear(128, 64, bias = True)
+ assert guard.install_fp16_guard(nn.Sequential(blk), "rescale_post_norm", torch.float16) == 0
+
+
+@needs_fp16
+def test_guard_matches_real_zimage_block():
+ """The recipe still recognises the installed diffusers ZImageTransformerBlock and is a no-op on in-range input."""
+ zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")
+ z = detect_family("Tongyi-MAI/Z-Image-Turbo")
+ guard._SUPPORTED.clear()
+ torch.manual_seed(0)
+ blk = zmod.ZImageTransformerBlock(0, 64, 4, 4, 1e-5, True, modulation = False).half()
+ x = torch.randn(1, 8, 64, dtype = torch.float16)
+ freqs = torch.polar(torch.ones(1, 8, 8), torch.randn(1, 8, 8))
+ with torch.no_grad():
+ before = blk(x, None, freqs).float()
+ assert guard.install_fp16_guard(nn.Sequential(blk), z.fp16_guard, torch.float16) == 1
+ after = blk(x, None, freqs).float()
+ assert torch.isfinite(after).all()
+ assert (after - before).abs().max() < 2e-2 * before.abs().max()
+
+
+def test_real_diffusers_zimage_source_matches_recipe(monkeypatch):
+ pytest.importorskip("diffusers.models.transformers.transformer_z_image")
+ monkeypatch.undo()
+ monkeypatch.setattr(guard, "_SUPPORTED", {})
+ assert (
+ guard._recipe_supported(detect_family("Tongyi-MAI/Z-Image-Turbo"), "rescale_post_norm")
+ is True
+ )
+
+
+class _LoRALinear(nn.Module):
+ """base + B @ A; peft's own injection imports torchao symbols some versions lack."""
+
+ def __init__(self, base):
+ super().__init__()
+ self.base = base
+ self.lora_A = nn.Linear(base.in_features, 4, bias = False)
+ self.lora_B = nn.Linear(4, base.out_features, bias = False)
+
+ def forward(self, x):
+ return self.base(x) + self.lora_B(self.lora_A(x))
+
+
+@needs_fp16
+def test_guard_holds_through_a_lora_injected_after_install():
+ """LoRA wrappers replace the Linear children after the guard is installed; the rescale lives on the parents, so the
+ adapted branch is scaled as a whole."""
+ targets = ["w1", "w2", "w3", "to_q", "to_k", "to_v", "to_out.0"]
+
+ def adapted(model):
+ torch.manual_seed(1)
+ for name, module in list(model.named_modules()):
+ if isinstance(module, nn.Linear) and any(name.endswith(t) for t in targets):
+ parent, _, child = name.rpartition(".")
+ setattr(model.get_submodule(parent), child, _LoRALinear(module))
+ return model
+
+ x = torch.randn(2, 16, 64)
+ with torch.no_grad():
+ ref_model = adapted(_overflowing_model()).float()
+ ref = ref_model(x.float())
+ half = _overflowing_model().half()
+ assert guard.install_fp16_guard(half, "rescale_post_norm", torch.float16) == 2
+ half = adapted(half.float()).half()
+ assert isinstance(
+ half.get_submodule(next(n for n, m in half.named_modules() if n.endswith("w3"))),
+ _LoRALinear,
+ )
+ fixed = half(x.half())
+ assert torch.isfinite(fixed).all()
+ err = (fixed.float() - ref).abs().max() / ref.abs().max()
+ assert err < 2e-2, float(err)
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 868319aba09..fa21e258088 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -8806,8 +8806,11 @@ def test_the_fp32_promotion_does_not_double_an_already_fp32_wan_vae(fake_runtime
# ALREADY (the family comment says so) and assembly pins that VAE to fp32 whatever the
# promotion does, so doubling it counts 2.8 GB twice on TI2V-5B. Against a hard refusal that
# is a load rejected over bytes it never allocates.
+ from core.inference.diffusion_fp16_guard import FP16_GUARD_ENV
from core.inference.video_families import detect_video_family
+ # TI2V-5B keeps fp16 by default (fp16_guard "native"); the kill switch restores the promotion under test.
+ monkeypatch.setenv(FP16_GUARD_ENV, "0")
_fp32_promoted_cuda_target(monkeypatch)
transformer_gb, te_gb, vae_gb = detect_video_family(
"Wan-AI/Wan2.2-TI2V-5B-Diffusers"
@@ -8821,6 +8824,20 @@ def test_the_fp32_promotion_does_not_double_an_already_fp32_wan_vae(fake_runtime
)
+def test_fp16_native_wan_is_planned_at_fp16_size(fake_runtime, monkeypatch):
+ # On an fp16-only card TI2V-5B now stays fp16, so the plan prices the bf16-sized table as is (no doubling).
+ from core.inference.diffusion_fp16_guard import FP16_GUARD_ENV
+ from core.inference.video_families import detect_video_family
+
+ monkeypatch.delenv(FP16_GUARD_ENV, raising = False)
+ _fp32_promoted_cuda_target(monkeypatch)
+ components = detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers").bf16_components_gb
+
+ calls = _capture_plan(monkeypatch)
+ VideoBackend().load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")
+ assert calls[0]["model_dense_mib"] == int(sum(components) * _MIB_PER_GB)
+
+
def test_the_fp32_promotion_still_doubles_a_bf16_vae(fake_runtime, monkeypatch):
# LTX-2 does not force fp32, so its VAE term IS a bf16 one and must keep scaling with the rest.
from core.inference.video_families import detect_video_family