Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run Z-Image and Wan2.2-5B / HunyuanVideo-1.5 in fp16 on fp16-only GPUs instead of fp32 (#12476)
Daniel Han77c193477e9b
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex e05c6066fe5..3d8ca806021 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -1164,14 +1164,16 @@ def _estimate_eta(total_steps: int, step: int, first_step_at: float, now: float)def _resolve_diffusion_compute_dtype(fam: Optional[DiffusionFamily], dtype: Any) -> Any:- """Promote float16 -> float32 for fp16-incompatible families (e.g. Z-Image), whose activations- overflow float16's finite range and render a black image. Every other dtype/family passes- through unchanged."""+ """Promote float16 -> float32 for fp16-incompatible families without a usable fp16 guard; else unchanged."""+ from .diffusion_fp16_guard import fp16_promotes_to_fp32+if fam is None or not getattr(fam, "fp16_incompatible", False):return dtypeimport torch- return torch.float32 if dtype == torch.float16 else dtype+ if dtype != torch.float16 or not fp16_promotes_to_fp32(fam):+ return dtype+ return torch.float32def _float_load_itemsize(dtype: Any) -> Optional[int]:@@ -6866,6 +6868,15 @@ class DiffusionBackend:from .diffusion_flux2_rope import install_for_pipe as install_flux2_ropeinstall_flux2_rope(pipe, dtype, device, logger)+ # fp16-only cards: a guarded family stays float16; patch its overflow sites.+ from .diffusion_fp16_guard import family_fp16_guard, install_fp16_guard++ install_fp16_guard(+ getattr(pipe, "transformer", None),+ family_fp16_guard(fam),+ getattr(target, "dtype", None),+ logger = logger,+ )self._raise_if_load_cancelled(_load_token)# Pre-warmed torch.compile cache: a per-fingerprint inductor dir plus a bundle loaded before thediff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.pyindex 22552512151..8889e1158e0 100644--- a/studio/backend/core/inference/diffusion_families.py+++ b/studio/backend/core/inference/diffusion_families.py@@ -120,6 +120,8 @@ class DiffusionFamily:# True for families whose activations overflow float16 (-> black image); the backend promotes a resolved float16# to float32.fp16_incompatible: bool = False+ # diffusion_fp16_guard recipe keeping an fp16_incompatible family in float16 on fp16-only cards; None = promote.+ fp16_guard: Optional[str] = None# false only for a family whose denoiser block does not compile cleanly with regional torch.compilesupports_torch_compile: bool = True# Optional pre-quantized transformer checkpoints as (scheme, repo_id): fetched instead of the dense bf16 (lower@@ -513,6 +515,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (img2img_pipeline_class = "ZImageImg2ImgPipeline",inpaint_pipeline_class = "ZImageInpaintPipeline",fp16_incompatible = True,+ # The attention / FFN branches overflow float16 before their post-norm.+ fp16_guard = "rescale_post_norm",# Byte-identical mirror of Comfy-Org/z_image_turbo (AE + Qwen3-4B).sd_cpp_vae = ("unsloth/Z-Image-Turbo-ComfyUI", "split_files/vae/ae.safetensors"),sd_cpp_text_encoders = (diff --git a/studio/backend/core/inference/diffusion_fp16_guard.py b/studio/backend/core/inference/diffusion_fp16_guard.pynew file mode 100644index 00000000000..e27cc9b3f02--- /dev/null+++ b/studio/backend/core/inference/diffusion_fp16_guard.py@@ -0,0 +1,251 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Keep bf16-native DiTs in float16 on fp16-only cards (T4 / sm75) instead of promoting them to float32.++``rescale_post_norm``: a branch that overflows float16 but feeds an RMSNorm is divided by a power of two s before the+overflow point and that norm's eps by s**2, since ``rms_norm(y / s, eps / s**2) == rms_norm(y, eps)`` (no clamping).+Attention scales its input (q / k RMSNorm eps scaled too; v and the bias-free out projection are linear); the FFN scales+the ``w3`` half of the gate. Hooks sit on the parents and call ``to_q`` / ``w1`` / ... at run time, so a later LoRA or+quantized Linear sees the same rescale. Kill switch: ``UNSLOTH_STUDIO_FP16_GUARD=0`` restores the float32 promotion.+"""++from __future__ import annotations++import os+from typing import Any, Optional++FP16_GUARD_ENV = "UNSLOTH_STUDIO_FP16_GUARD"+_MARK = "_unsloth_fp16_guard"+_FP16_MAX = 65504.0++# recipe -> (attention, ffn) scale, powers of two (exact in float16); None = finite in plain float16, no patch.+RECIPES: dict[str, Optional[tuple[float, float]]] = {+ "native": None,+ "rescale_post_norm": (16.0, 128.0),+}+++def fp16_guard_disabled() -> bool:+ return (os.environ.get(FP16_GUARD_ENV) or "").strip().lower() in ("0", "off", "false", "no")+++def family_fp16_guard(fam: Any) -> Optional[str]:+ if fam is None or fp16_guard_disabled():+ return None+ recipe = getattr(fam, "fp16_guard", None)+ return recipe if isinstance(recipe, str) and recipe in RECIPES else None+++# Checked before the dtype is chosen: a diffusers release that restructures the block keeps the float32 promotion.+_RECIPE_TOKENS: dict[str, tuple[str, ...]] = {+ "rescale_post_norm": (+ "self.attention_norm2(",+ "self.ffn_norm2(",+ "self.feed_forward(",+ "self.w2(",+ "self.w3(",+ "norm_q",+ ),+}+_SUPPORTED: dict[tuple[str, str], bool] = {}+++def _recipe_supported(fam: Any, recipe: str) -> bool:+ tokens = _RECIPE_TOKENS.get(recipe)+ if not tokens:+ return True+ cls_name = getattr(fam, "transformer_class", None)+ if not isinstance(cls_name, str) or not cls_name:+ return False+ key = (cls_name, recipe)+ if key not in _SUPPORTED:+ # Runs from dtype resolution, ahead of load_pipeline's own guard: `import diffusers` imports torch._dynamo.+ try:+ from loggers import get_logger+ from utils.torch_warmup import close_dynamo_import_window+ close_dynamo_import_window(get_logger(__name__))+ except Exception: # noqa: BLE001, S110 - optimisation only+ pass+ try:+ import importlib+ import inspect+ import sys++ cls = getattr(importlib.import_module("diffusers"), cls_name)+ src = inspect.getsource(sys.modules[cls.__module__])+ _SUPPORTED[key] = all(tok in src for tok in tokens)+ except Exception: # noqa: BLE001 - unreadable source: keep the safe float32 promotion+ _SUPPORTED[key] = False+ return _SUPPORTED[key]+++def fp16_promotes_to_fp32(fam: Any) -> bool:+ """True when a resolved float16 must be promoted to float32 for this family: fp16-incompatible with no usable+ guard (none declared, the kill switch is set, or the installed diffusers block no longer matches the recipe)."""+ if fam is None or not getattr(fam, "fp16_incompatible", False):+ return False+ recipe = family_fp16_guard(fam)+ return recipe is None or not _recipe_supported(fam, recipe)+++def _is_fp16(dtype: Any) -> bool:+ try:+ import torch+ return dtype is torch.float16 or dtype == torch.float16+ except Exception: # noqa: BLE001 - no torch, nothing to guard+ return False+++def _scale_eps(norm: Any, factor: float) -> bool:+ eps = getattr(norm, "eps", None)+ if not isinstance(eps, float) or isinstance(eps, bool):+ return False+ norm.eps = eps * factor+ return True+++def _finite(x: Any) -> Any:+ import torch+ if x.dtype is torch.float16:+ return torch.nan_to_num(x, nan = 0.0, posinf = _FP16_MAX, neginf = -_FP16_MAX)+ return x+++def _attention_pre_hook(inv: float):+ def hook(module, args, kwargs):+ if args:+ return (args[0] * inv, *args[1:]), kwargs+ if "hidden_states" in kwargs:+ kwargs = dict(kwargs)+ kwargs["hidden_states"] = kwargs["hidden_states"] * inv+ return args, kwargs+ return None++ return hook+++def _attention_post_hook(module, args, output):+ import torch+ if isinstance(output, torch.Tensor):+ return _finite(output)+ return None+++def _make_ffn_forward(inv: float):+ def forward(self, x):+ import torch.nn.functional as F+ return self.w2(_finite(F.silu(self.w1(x)) * (self.w3(x) * inv)))++ return forward+++def _block_sites(block: Any) -> Optional[tuple[Any, ...]]:+ attn = getattr(block, "attention", None)+ ffn = getattr(block, "feed_forward", None)+ norms = tuple(getattr(block, n, None) for n in ("attention_norm2", "ffn_norm2"))+ if attn is None or ffn is None or any(n is None for n in norms):+ return None+ if any(+ getattr(attn, n, None) is None+ for n in ("norm_q", "norm_k", "to_q", "to_k", "to_v", "to_out")+ ):+ return None+ if any(getattr(ffn, n, None) is None for n in ("w1", "w2", "w3")):+ return None+ # The rescale is exact only through bias-free projections.+ linears = [attn.to_q, attn.to_k, attn.to_v, attn.to_out[0], ffn.w2]+ if any(getattr(lin, "bias", None) is not None for lin in linears):+ return None+ eps = [getattr(m, "eps", None) for m in (attn.norm_q, attn.norm_k, *norms)]+ if not all(isinstance(e, float) for e in eps):+ return None+ return attn, ffn, norms[0], norms[1]+++def _install_rescale_post_norm(root: Any, attn_scale: float, ffn_scale: float) -> int:+ import types++ sites = 0+ for _name, block in root.named_modules():+ if block.__dict__.get(_MARK) is not None:+ sites += 1+ continue+ found = _block_sites(block)+ if found is None:+ continue+ attn, ffn, attn_norm2, ffn_norm2 = found+ a_inv, f_inv = 1.0 / attn_scale, 1.0 / ffn_scale+ _scale_eps(attn.norm_q, a_inv * a_inv)+ _scale_eps(attn.norm_k, a_inv * a_inv)+ _scale_eps(attn_norm2, a_inv * a_inv)+ _scale_eps(ffn_norm2, f_inv * f_inv)+ handles = (+ attn.register_forward_pre_hook(_attention_pre_hook(a_inv), with_kwargs = True),+ attn.register_forward_hook(_attention_post_hook),+ )+ ffn.forward = types.MethodType(_make_ffn_forward(f_inv), ffn)+ block.__dict__[_MARK] = (attn_scale, ffn_scale, handles)+ sites += 1+ return sites+++def install_fp16_guard(+ root: Any,+ recipe: Optional[str],+ dtype: Any,+ logger: Any = None,+) -> int:+ """Install ``recipe`` on every matching block under ``root`` when ``dtype`` is float16. Returns the number of+ guarded blocks (0 = not engaged). Idempotent."""+ if root is None or recipe not in RECIPES or not _is_fp16(dtype) or fp16_guard_disabled():+ return 0+ scales = RECIPES[recipe]+ if scales is None:+ if logger is not None:+ logger.info("diffusion.fp16_guard: recipe=%s (float16 kept, no patch needed)", recipe)+ return 0+ if not callable(getattr(root, "named_modules", None)):+ return 0+ attn_scale, ffn_scale = scales+ sites = _install_rescale_post_norm(root, attn_scale, ffn_scale)+ if logger is not None:+ if sites:+ logger.info(+ "diffusion.fp16_guard: recipe=%s blocks=%d (float16 kept, no float32 promotion)",+ recipe,+ sites,+ )+ else:+ logger.warning(+ "diffusion.fp16_guard: recipe=%s matched no block; float16 output may overflow "+ "(set %s=0 to promote to float32)",+ recipe,+ FP16_GUARD_ENV,+ )+ return sites+++def remove_fp16_guard(root: Any) -> int:+ """Undo ``install_fp16_guard`` on ``root`` (eps restored, hooks and forward overrides dropped)."""+ removed = 0+ if root is None:+ return 0+ for _name, block in root.named_modules():+ mark = block.__dict__.pop(_MARK, None)+ if mark is None:+ continue+ attn_scale, ffn_scale, handles = mark+ for handle in handles:+ handle.remove()+ attn, ffn = block.attention, block.feed_forward+ for norm, scale in (+ (attn.norm_q, attn_scale),+ (attn.norm_k, attn_scale),+ (block.attention_norm2, attn_scale),+ (block.ffn_norm2, ffn_scale),+ ):+ _scale_eps(norm, scale * scale)+ ffn.__dict__.pop("forward", None)+ removed += 1+ return removeddiff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 92572cc75f6..3781f5e014d 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -335,7 +335,8 @@ def compile_eligible(target: Any, *, is_gguf: bool, family: Any) -> bool:dtype = getattr(target, "dtype", None)if _is_bfloat16(dtype):return True- # fp16-incompatible families run in fp32 though video passes the fp16 target; fp32 compile is unmeasured.+ # fp16-incompatible families run in fp32 (unmeasured under compile) or in fp16 behind diffusion_fp16_guard, which+ # must stay eager: the int8 fused MLP installed for compile replaces the guard's FFN forward.if bool(getattr(family, "fp16_incompatible", False)):return Falsereturn _is_float16(dtype) and _fp16_compile_capable(target)diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex b25f7fab4f6..d9debeeac88 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -119,6 +119,7 @@ from .diffusion_memory import (from .diffusion_torchao_patches import install_torchao_int_mm_patchfrom .media_decode_phase import decode_phase as _decode_phasefrom . import diffusion_render_thread as render_thread+from .diffusion_fp16_guard import fp16_promotes_to_fp32from .diffusion_speed import (SPEED_DEFAULT,SPEED_EAGER,@@ -3919,7 +3920,7 @@ class VideoBackend:else resolve_diffusion_device_target(ordinal = gpu_ordinal))dtype = target.dtype- if getattr(fam, "fp16_incompatible", False) and dtype is torch.float16:+ if dtype is torch.float16 and fp16_promotes_to_fp32(fam):dtype = torch.float32return _h3_auto_denoiser_scheme(fam,@@ -3960,7 +3961,7 @@ class VideoBackend:if gpu_ordinal is Noneelse resolve_diffusion_device_target(ordinal = gpu_ordinal))- if getattr(fam, "fp16_incompatible", False) and target.dtype is torch.float16:+ if target.dtype is torch.float16 and fp16_promotes_to_fp32(fam):# The loader promotes fp16 to float32 here, so read the dtype the load will.return Nonescheme = _video_auto_denoiser_scheme(@@ -5393,10 +5394,17 @@ class VideoBackend:)# Bound only at the commit past the token check: a superseded load may return from the install late._nvfp4_install_outcome: Optional[tuple[bool, str]] = None- # Video DiTs are bf16-native; fp16 overflows, so a resolved fp16 promotes to float32.+ # Video DiTs are bf16-native: fp16 promotes to float32 unless the family declares an fp16_guard.dtype = target.dtype- if fam.fp16_incompatible and dtype is torch.float16:+ if dtype is torch.float16 and fp16_promotes_to_fp32(fam):dtype = torch.float32+ logger.warning(+ "video.dtype_promoted: family=%s float16 -> float32 (fp16-incompatible)", fam.name+ )+ elif fam.fp16_incompatible and dtype is torch.float16:+ logger.info(+ "video.fp16_guard: family=%s recipe=%s (float16 kept)", fam.name, fam.fp16_guard+ )# Size tables below are bf16 (2-byte), so scale dense estimates when the promotion lands fp32 on an acceleratordtype_scale = 2.0 if device != "cpu" and dtype is torch.float32 else 1.0diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.pyindex 5a13d7f1480..0739f3c2e61 100644--- a/studio/backend/core/inference/video_families.py+++ b/studio/backend/core/inference/video_families.py@@ -85,6 +85,8 @@ class VideoFamily:supports_cuda_graph: bool = False# Video DiTs are bf16-native, so fp16 promotes to float32; defaults True.fp16_incompatible: bool = True+ # "native" = measured accurate in plain float16 on fp16-only cards; None = promote (unmeasured).+ fp16_guard: Optional[str] = None# Wan VAE decodes in float32 (bf16 causes banding / black frames), so the loader pins it back. Its size term is# already fp32.vae_force_fp32: bool = False@@ -286,6 +288,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (# (11.4); VAE fp32 (2.8).bf16_components_gb = (10.0, 11.4, 2.8),vae_force_fp32 = True,+ # UMT5 keeps its overflowing `wo` in fp32 itself; the VAE stays fp32 (vae_force_fp32).+ fp16_guard = "native",# Byte-identical mirror of QuantStack/Wan2.2-TI2V-5B-GGUF (13 quants + companion VAE).gguf_repo = "unsloth/Wan2.2-TI2V-5B-GGUF",),@@ -354,6 +358,7 @@ _FAMILIES: tuple[VideoFamily, ...] = (resolution_presets = ((832, 480), (480, 832), (640, 640)),# DiT fp32 on disk (32.0 to 16.6 bf16); VAE 4.7 to 2.4; Qwen2.5-VL TE bf16 14.0 + ByT5 0.8bf16_components_gb = (16.6, 14.8, 2.4),+ fp16_guard = "native",),# The 720p t2v repack: same architecture and footprint as the 480p entry, only the trained resolution differs. Its# own family so a 720p load defaults to 720p sizes; the full-path alias outranks the generic token.@@ -378,6 +383,7 @@ _FAMILIES: tuple[VideoFamily, ...] = (# 720p-class presets: landscape, vertical, square (all /16).resolution_presets = ((1280, 720), (720, 1280), (960, 960)),bf16_components_gb = (16.6, 14.8, 2.4),+ fp16_guard = "native",),)diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex e84f9387477..f60f1eb8556 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -3866,11 +3866,16 @@ def test_zimage_is_fp16_incompatible():assert detect_family("unsloth/FLUX.2-klein-4B-GGUF").fp16_incompatible is False-def test_resolve_compute_dtype_promotes_fp16_for_zimage(fake_runtime):+def test_resolve_compute_dtype_promotes_fp16_for_zimage(fake_runtime, monkeypatch):+ from core.inference import diffusion_fp16_guard as guard+torch = sys.modules["torch"]z = detect_family("unsloth/Z-Image-GGUF")q = detect_family("unsloth/FLUX.1-schnell-GGUF")- # Z-Image: fp16 promotes to fp32; bf16 / fp32 pass through unchanged.+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: True)+ assert _resolve_diffusion_compute_dtype(z, torch.float16) is torch.float16+ # Kill switch: fp16 promotes to fp32; bf16 / fp32 unchanged.+ monkeypatch.setenv(guard.FP16_GUARD_ENV, "0")assert _resolve_diffusion_compute_dtype(z, torch.float16) is torch.float32assert _resolve_diffusion_compute_dtype(z, torch.bfloat16) is torch.bfloat16assert _resolve_diffusion_compute_dtype(z, torch.float32) is torch.float32@@ -3880,13 +3885,22 @@ def test_resolve_compute_dtype_promotes_fp16_for_zimage(fake_runtime):def test_load_promotes_fp16_to_fp32_for_zimage_only(fake_runtime, monkeypatch, tmp_path):+ from core.inference import diffusion_fp16_guard as guard+torch = sys.modules["torch"]- # Pre-Ampere CUDA resolves to fp16, so the guard must promote Z-Image (and only fp16-incompatible families) to fp32- # or it renders black.+ # Pre-Ampere resolves fp16: Z-Image stays fp16 behind its guard; the kill switch promotes it to fp32.monkeypatch.setattr(torch.cuda, "is_available", lambda: True, raising = False)monkeypatch.setattr(torch.cuda, "get_device_capability", lambda: (7, 5), raising = False)(tmp_path / "m.gguf").write_bytes(b"x")+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: True)+ kept = DiffusionBackend().load_pipeline(+ str(tmp_path), gguf_filename = "m.gguf", family_override = "z-image"+ )+ assert kept["dtype"] == "float16"+ assert str(_FakeTransformer.last["torch_dtype"]) == "torch.float16"++ monkeypatch.setenv(guard.FP16_GUARD_ENV, "0")z = DiffusionBackend().load_pipeline(str(tmp_path), gguf_filename = "m.gguf", family_override = "z-image")diff --git a/studio/backend/tests/test_diffusion_fp16_guard.py b/studio/backend/tests/test_diffusion_fp16_guard.pynew file mode 100644index 00000000000..03760bb5f78--- /dev/null+++ b/studio/backend/tests/test_diffusion_fp16_guard.py@@ -0,0 +1,358 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""fp16 guard for fp16-only cards: capability resolution, kill switch, bf16 untouched, and the post-norm rescale on a+tiny block shaped like Z-Image's (CPU, real torch)."""++from __future__ import annotations++import pytest++torch = pytest.importorskip("torch")+nn = torch.nn+F = torch.nn.functional++from core.inference import diffusion_fp16_guard as guard # noqa: E402+from core.inference.diffusion import _resolve_diffusion_compute_dtype # noqa: E402+from core.inference.diffusion_families import _FAMILIES, detect_family # noqa: E402+from core.inference.video_families import _FAMILIES as _VIDEO_FAMILIES # noqa: E402++[email protected](autouse = True)+def _clean_env(monkeypatch):+ monkeypatch.delenv(guard.FP16_GUARD_ENV, raising = False)+ # The structural probe reads the installed diffusers; pin it so the resolution tests do not depend on it.+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: True)+++def _video(name):+ return next(f for f in _VIDEO_FAMILIES if f.name == name)+++def test_every_declared_recipe_is_known():+ for fam in (*_FAMILIES, *_VIDEO_FAMILIES):+ recipe = getattr(fam, "fp16_guard", None)+ assert recipe is None or recipe in guard.RECIPES, (fam.name, recipe)+ # a guard only means something on a family that would otherwise be promoted+ assert recipe is None or fam.fp16_incompatible, fam.name+++def test_zimage_stays_fp16_with_guard():+ z = detect_family("Tongyi-MAI/Z-Image-Turbo")+ assert z.fp16_incompatible is True and z.fp16_guard == "rescale_post_norm"+ assert guard.fp16_promotes_to_fp32(z) is False+ assert _resolve_diffusion_compute_dtype(z, torch.float16) is torch.float16+++def test_unguarded_incompatible_family_still_promotes():+ krea = detect_family("krea/Krea-2-Turbo")+ assert krea.fp16_incompatible is True and krea.fp16_guard is None+ assert _resolve_diffusion_compute_dtype(krea, torch.float16) is torch.float32+++def test_video_families_declare_only_native():+ # The video loader resolves the dtype but installs no guard hooks, so a patching recipe there would render black.+ for fam in _VIDEO_FAMILIES:+ assert fam.fp16_guard in (None, "native"), fam.name+++def test_video_capability_flags():+ for name in ("wan2.2-ti2v-5b", "hunyuanvideo-1.5", "hunyuanvideo-1.5-720p"):+ assert guard.fp16_promotes_to_fp32(_video(name)) is False, name+ # unmeasured families, and A14B (fp16 drifts from fp32 as far as bf16 does), keep the blanket promotion+ for name in ("ltx-2", "minimax-h3", "wan2.2-t2v-a14b"):+ assert guard.fp16_promotes_to_fp32(_video(name)) is True, name+++def test_kill_switch_restores_promotion(monkeypatch):+ z = detect_family("Tongyi-MAI/Z-Image-Turbo")+ monkeypatch.setenv(guard.FP16_GUARD_ENV, "0")+ assert guard.fp16_promotes_to_fp32(z) is True+ assert _resolve_diffusion_compute_dtype(z, torch.float16) is torch.float32+ assert guard.fp16_promotes_to_fp32(_video("wan2.2-ti2v-5b")) is True+++def test_unsupported_diffusers_block_keeps_promotion(monkeypatch):+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: False)+ z = detect_family("Tongyi-MAI/Z-Image-Turbo")+ assert _resolve_diffusion_compute_dtype(z, torch.float16) is torch.float32+ # "native" needs no patch, so no structural probe either+ monkeypatch.undo()+ monkeypatch.setattr(guard, "_SUPPORTED", {})+ assert guard._recipe_supported(_video("wan2.2-ti2v-5b"), "native") is True+++def test_bf16_and_fp32_never_change():+ for fam in (*_FAMILIES, None):+ for dt in (torch.bfloat16, torch.float32):+ assert _resolve_diffusion_compute_dtype(fam, dt) is dt+++def test_non_fp16_resolution_never_probes_diffusers(monkeypatch):+ def probe(fam, recipe):+ raise AssertionError("bf16 / fp32 must not import diffusers")++ monkeypatch.setattr(guard, "_recipe_supported", probe)+ z = detect_family("Tongyi-MAI/Z-Image-Turbo")+ assert _resolve_diffusion_compute_dtype(z, torch.bfloat16) is torch.bfloat16+ assert _resolve_diffusion_compute_dtype(z, torch.float32) is torch.float32+++def test_probe_closes_dynamo_import_window_before_diffusers(monkeypatch):+ import importlib++ from utils import torch_warmup++ monkeypatch.undo()+ monkeypatch.setattr(guard, "_SUPPORTED", {})+ order = []+ monkeypatch.setattr(+ torch_warmup, "close_dynamo_import_window", lambda log: order.append("close")+ )+ real = importlib.import_module++ def tracking(name, *a, **k):+ if name == "diffusers":+ order.append("diffusers")+ return real(name, *a, **k)++ monkeypatch.setattr(importlib, "import_module", tracking)+ guard._recipe_supported(detect_family("Tongyi-MAI/Z-Image-Turbo"), "rescale_post_norm")+ assert order[:2] == ["close", "diffusers"], order+++class _RMS(nn.Module):+ def __init__(+ self,+ dim,+ eps = 1e-5,+ ):+ super().__init__()+ self.eps = eps+ self.weight = nn.Parameter(torch.ones(dim))++ def forward(self, x):+ var = x.float().pow(2).mean(-1, keepdim = True)+ return (x.float() * torch.rsqrt(var + self.eps)).to(x.dtype) * self.weight.to(x.dtype)+++class _Attn(nn.Module):+ def __init__(self, dim, heads):+ super().__init__()+ self.heads = heads+ self.to_q = nn.Linear(dim, dim, bias = False)+ self.to_k = nn.Linear(dim, dim, bias = False)+ self.to_v = nn.Linear(dim, dim, bias = False)+ self.to_out = nn.ModuleList([nn.Linear(dim, dim, bias = False), nn.Dropout(0.0)])+ self.norm_q = _RMS(dim // heads)+ self.norm_k = _RMS(dim // heads)++ def forward(+ self,+ hidden_states,+ attention_mask = None,+ ):+ b, s, d = hidden_states.shape+ q, k, v = (+ p(hidden_states).view(b, s, self.heads, -1) for p in (self.to_q, self.to_k, self.to_v)+ )+ q, k = self.norm_q(q), self.norm_k(k)+ w = torch.softmax(+ (q.float().transpose(1, 2) @ k.float().permute(0, 2, 3, 1)) / (d // self.heads) ** 0.5,+ -1,+ )+ o = (w.to(v.dtype) @ v.transpose(1, 2)).transpose(1, 2).reshape(b, s, d)+ return self.to_out[1](self.to_out[0](o))+++class _FFN(nn.Module):+ def __init__(self, dim, hidden):+ super().__init__()+ self.w1 = nn.Linear(dim, hidden, bias = False)+ self.w2 = nn.Linear(hidden, dim, bias = False)+ self.w3 = nn.Linear(dim, hidden, bias = False)++ def forward(self, x):+ return self.w2(F.silu(self.w1(x)) * self.w3(x))+++class _Block(nn.Module):+ def __init__(+ self,+ dim = 64,+ heads = 4,+ hidden = 128,+ ):+ super().__init__()+ self.attention = _Attn(dim, heads)+ self.feed_forward = _FFN(dim, hidden)+ self.attention_norm1, self.attention_norm2 = _RMS(dim), _RMS(dim)+ self.ffn_norm1, self.ffn_norm2 = _RMS(dim), _RMS(dim)++ def forward(self, x):+ x = x + 0.5 * self.attention_norm2(self.attention(self.attention_norm1(x)))+ return x + 0.5 * self.ffn_norm2(self.feed_forward(self.ffn_norm1(x)))+++def _overflowing_model():+ torch.manual_seed(0)+ model = nn.Sequential(_Block(), _Block())+ with torch.no_grad():+ for blk in model:+ # Z-Image-like: the FFN gate product and output, and the attention output, exceed float16's range.+ blk.feed_forward.w1.weight.mul_(40.0)+ blk.feed_forward.w3.weight.mul_(40.0)+ blk.feed_forward.w2.weight.mul_(400.0)+ blk.attention.to_v.weight.mul_(300.0)+ blk.attention.to_out[0].weight.mul_(300.0)+ return model+++def _fp16_ok():+ try:+ nn.Linear(4, 4).half()(torch.ones(1, 4, dtype = torch.float16))+ return True+ except Exception: # noqa: BLE001+ return False+++needs_fp16 = pytest.mark.skipif(not _fp16_ok(), reason = "this torch has no CPU float16 matmul")+++@needs_fp16+def test_guard_keeps_overflowing_block_finite_and_accurate():+ model = _overflowing_model()+ x = torch.randn(2, 16, 64)+ with torch.no_grad():+ ref = model.float()(x.float())+ assert ref.abs().max() < 1e3 # the residual stream itself is in range+ branch = model[0].feed_forward(model[0].ffn_norm1(x.float()))+ assert branch.abs().max() > 65504 # ...the pre-norm branch is not++ half = model.half()+ stock = half(x.half())+ assert not torch.isfinite(stock).all() # stock fp16 overflows (black image)++ assert guard.install_fp16_guard(half, "rescale_post_norm", torch.float16) == 2+ fixed = half(x.half())+ assert torch.isfinite(fixed).all()+ err = (fixed.float() - ref).abs().max() / ref.abs().max()+ assert err < 2e-2, float(err)+++@needs_fp16+def test_guard_is_idempotent_and_removable():+ model = _overflowing_model().half()+ eps_before = [m.eps for m in model.modules() if isinstance(m, _RMS)]+ assert guard.install_fp16_guard(model, "rescale_post_norm", torch.float16) == 2+ assert guard.install_fp16_guard(model, "rescale_post_norm", torch.float16) == 2+ assert model[0].attention_norm2.eps == 1e-5 / 16**2+ assert model[0].ffn_norm2.eps == 1e-5 / 128**2+ assert guard.remove_fp16_guard(model) == 2+ assert [m.eps for m in model.modules() if isinstance(m, _RMS)] == eps_before+ assert "forward" not in model[0].feed_forward.__dict__+ assert not model[0].attention._forward_pre_hooks and not model[0].attention._forward_hooks++[email protected]("dtype", ["bfloat16", "float32"])+def test_guard_never_touches_non_fp16(dtype):+ dt = getattr(torch, dtype)+ torch.manual_seed(0)+ model = nn.Sequential(_Block(), _Block()).to(dt)+ x = torch.randn(2, 16, 64, dtype = dt)+ with torch.no_grad():+ before = model(x)+ assert guard.install_fp16_guard(model, "rescale_post_norm", dt) == 0+ after = model(x)+ assert torch.equal(before, after)+ assert all(m.eps == 1e-5 for m in model.modules() if isinstance(m, _RMS))+ assert not model[0].attention._forward_pre_hooks+++@needs_fp16+def test_guard_inert_under_kill_switch_and_native(monkeypatch):+ model = _overflowing_model().half()+ assert guard.install_fp16_guard(model, "native", torch.float16) == 0+ assert guard.install_fp16_guard(model, None, torch.float16) == 0+ monkeypatch.setenv(guard.FP16_GUARD_ENV, "0")+ assert guard.install_fp16_guard(model, "rescale_post_norm", torch.float16) == 0+ assert not model[0].attention._forward_pre_hooks+++def test_block_with_bias_is_not_guarded():+ torch.manual_seed(0)+ blk = _Block()+ blk.feed_forward.w2 = nn.Linear(128, 64, bias = True)+ assert guard.install_fp16_guard(nn.Sequential(blk), "rescale_post_norm", torch.float16) == 0+++@needs_fp16+def test_guard_matches_real_zimage_block():+ """The recipe still recognises the installed diffusers ZImageTransformerBlock and is a no-op on in-range input."""+ zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")+ z = detect_family("Tongyi-MAI/Z-Image-Turbo")+ guard._SUPPORTED.clear()+ torch.manual_seed(0)+ blk = zmod.ZImageTransformerBlock(0, 64, 4, 4, 1e-5, True, modulation = False).half()+ x = torch.randn(1, 8, 64, dtype = torch.float16)+ freqs = torch.polar(torch.ones(1, 8, 8), torch.randn(1, 8, 8))+ with torch.no_grad():+ before = blk(x, None, freqs).float()+ assert guard.install_fp16_guard(nn.Sequential(blk), z.fp16_guard, torch.float16) == 1+ after = blk(x, None, freqs).float()+ assert torch.isfinite(after).all()+ assert (after - before).abs().max() < 2e-2 * before.abs().max()+++def test_real_diffusers_zimage_source_matches_recipe(monkeypatch):+ pytest.importorskip("diffusers.models.transformers.transformer_z_image")+ monkeypatch.undo()+ monkeypatch.setattr(guard, "_SUPPORTED", {})+ assert (+ guard._recipe_supported(detect_family("Tongyi-MAI/Z-Image-Turbo"), "rescale_post_norm")+ is True+ )+++class _LoRALinear(nn.Module):+ """base + B @ A; peft's own injection imports torchao symbols some versions lack."""++ def __init__(self, base):+ super().__init__()+ self.base = base+ self.lora_A = nn.Linear(base.in_features, 4, bias = False)+ self.lora_B = nn.Linear(4, base.out_features, bias = False)++ def forward(self, x):+ return self.base(x) + self.lora_B(self.lora_A(x))+++@needs_fp16+def test_guard_holds_through_a_lora_injected_after_install():+ """LoRA wrappers replace the Linear children after the guard is installed; the rescale lives on the parents, so the+ adapted branch is scaled as a whole."""+ targets = ["w1", "w2", "w3", "to_q", "to_k", "to_v", "to_out.0"]++ def adapted(model):+ torch.manual_seed(1)+ for name, module in list(model.named_modules()):+ if isinstance(module, nn.Linear) and any(name.endswith(t) for t in targets):+ parent, _, child = name.rpartition(".")+ setattr(model.get_submodule(parent), child, _LoRALinear(module))+ return model++ x = torch.randn(2, 16, 64)+ with torch.no_grad():+ ref_model = adapted(_overflowing_model()).float()+ ref = ref_model(x.float())+ half = _overflowing_model().half()+ assert guard.install_fp16_guard(half, "rescale_post_norm", torch.float16) == 2+ half = adapted(half.float()).half()+ assert isinstance(+ half.get_submodule(next(n for n, m in half.named_modules() if n.endswith("w3"))),+ _LoRALinear,+ )+ fixed = half(x.half())+ assert torch.isfinite(fixed).all()+ err = (fixed.float() - ref).abs().max() / ref.abs().max()+ assert err < 2e-2, float(err)diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 868319aba09..fa21e258088 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -8806,8 +8806,11 @@ def test_the_fp32_promotion_does_not_double_an_already_fp32_wan_vae(fake_runtime# ALREADY (the family comment says so) and assembly pins that VAE to fp32 whatever the# promotion does, so doubling it counts 2.8 GB twice on TI2V-5B. Against a hard refusal that# is a load rejected over bytes it never allocates.+ from core.inference.diffusion_fp16_guard import FP16_GUARD_ENVfrom core.inference.video_families import detect_video_family+ # TI2V-5B keeps fp16 by default (fp16_guard "native"); the kill switch restores the promotion under test.+ monkeypatch.setenv(FP16_GUARD_ENV, "0")_fp32_promoted_cuda_target(monkeypatch)transformer_gb, te_gb, vae_gb = detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers"@@ -8821,6 +8824,20 @@ def test_the_fp32_promotion_does_not_double_an_already_fp32_wan_vae(fake_runtime)+def test_fp16_native_wan_is_planned_at_fp16_size(fake_runtime, monkeypatch):+ # On an fp16-only card TI2V-5B now stays fp16, so the plan prices the bf16-sized table as is (no doubling).+ from core.inference.diffusion_fp16_guard import FP16_GUARD_ENV+ from core.inference.video_families import detect_video_family++ monkeypatch.delenv(FP16_GUARD_ENV, raising = False)+ _fp32_promoted_cuda_target(monkeypatch)+ components = detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers").bf16_components_gb++ calls = _capture_plan(monkeypatch)+ VideoBackend().load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")+ assert calls[0]["model_dense_mib"] == int(sum(components) * _MIB_PER_GB)++def test_the_fp32_promotion_still_doubles_a_bf16_vae(fake_runtime, monkeypatch):# LTX-2 does not force fp32, so its VAE term IS a bf16 one and must keep scaling with the rest.from core.inference.video_families import detect_video_family