Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio FLUX.3: keep float16 on fp16-only GPUs behind a residual rescale (T4 INT8 -24% per image, GGUF Q8_0 -73%) (#17385)

Daniel Handd43f1028152
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index f0222b974b3..527e0048160 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -334,7 +334,7 @@ _QWENIMAGE_RDNA_AUTO_W8A8 = (
"compiled int8 W8A8 measured 133.1 vs 141.2 s/image against compiled bf16 on gfx1151 (20 steps CFG 4), "
"LPIPS 0.014"
)
-# FLUX.3 renders black in float16, so fp16-only cards run it in float32; UNSLOTH_STUDIO_FLUX3_FP16=1 keeps fp16.
+# FLUX.3 renders black in plain float16; UNSLOTH_STUDIO_FLUX3_FP16=1 keeps fp16.
_FLUX3_FP16_INCOMPATIBLE = (
os.environ.get("UNSLOTH_STUDIO_FLUX3_FP16") or ""
).strip().lower() not in (
@@ -343,6 +343,13 @@ _FLUX3_FP16_INCOMPATIBLE = (
"true",
"yes",
)
+# fp16 cards keep FLUX.3 in fp16 via the residual rescale; UNSLOTH_STUDIO_FLUX3_FP16_GUARD=0 promotes to fp32.
+_FLUX3_FP16_GUARD = (
+ None
+ if (os.environ.get("UNSLOTH_STUDIO_FLUX3_FP16_GUARD") or "").strip().lower()
+ in ("0", "off", "false", "no")
+ else "rescale_residual"
+)
# FLUX.3 Klein transformer-quant keep-list, every scheme (FamilyProfile.quant_exclude_tokens).
_FLUX3_QUANT_KEEP = (
"modulation",
@@ -652,6 +659,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
name = "flux.3-klein-base",
comfy_shift_node = "flux",
fp16_incompatible = _FLUX3_FP16_INCOMPATIBLE,
+ fp16_guard = _FLUX3_FP16_GUARD,
label = "FLUX.3 Klein Base",
profile = FamilyProfile(
auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,
@@ -711,6 +719,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
name = "flux.3-klein",
comfy_shift_node = "flux",
fp16_incompatible = _FLUX3_FP16_INCOMPATIBLE,
+ fp16_guard = _FLUX3_FP16_GUARD,
label = "FLUX.3 Klein",
profile = FamilyProfile(
auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,
diff --git a/studio/backend/core/inference/diffusion_fp16_guard.py b/studio/backend/core/inference/diffusion_fp16_guard.py
index c0f20863213..d77cacb11ea 100644
--- a/studio/backend/core/inference/diffusion_fp16_guard.py
+++ b/studio/backend/core/inference/diffusion_fp16_guard.py
@@ -17,6 +17,8 @@ _FP16_MAX = 65504.0
RECIPES: dict[str, Optional[tuple[float, float]]] = {
"native": None,
"rescale_post_norm": (16.0, 128.0),
+ # FLUX.3 residual reaches ~1e7: whole stream at 1/1024, LayerNorm eps by 1/1024^2.
+ "rescale_residual": (1024.0, 1024.0),
}
@@ -41,6 +43,16 @@ _RECIPE_TOKENS: dict[str, tuple[str, ...]] = {
"self.w3(",
"norm_q",
),
+ "rescale_residual": (
+ "self.context_embedder(encoder_hidden_states)",
+ 'self.x_embedders["image"](hidden_states)',
+ "return hidden_states + mod_gate * (attn_output + ff_output)",
+ "encoder_hidden_states = encoder_hidden_states + c_mod_gate * context_output",
+ "hidden_states = hidden_states + mod_gate * content_output",
+ "x = self.gate_fn(x[..., :half]) * x[..., half:]",
+ "return self.norm(hidden_states) * (1 + scale) + shift",
+ "attn.norm_q(query.float())",
+ ),
}
_SUPPORTED: dict[tuple[str, str], bool] = {}
@@ -157,6 +169,122 @@ def _block_sites(block: Any) -> Optional[tuple[Any, ...]]:
return attn, ffn, norms[0], norms[1]
+def _scale_output_hook(inv: float):
+ def hook(module, args, output):
+ return output * inv
+
+ return hook
+
+
+def _scale_input_hook(inv: float):
+ def hook(module, args):
+ return (args[0] * inv, *args[1:]) if args else None
+
+ return hook
+
+
+def _compiling() -> bool:
+ try:
+ import torch
+ return bool(torch.compiler.is_compiling())
+ except Exception: # noqa: BLE001 - no torch.compiler: eager
+ return False
+
+
+def _bias_free(*linears: Any) -> bool:
+ return all(lin is not None and getattr(lin, "bias", None) is None for lin in linears)
+
+
+def _residual_block_sites(block: Any) -> Optional[tuple[Any, ...]]:
+ """(norm, attn, act_fn) of a FLUX.3 mode / joint block, whose residual update is gate * (attn + ff)."""
+ norm, attn, ff = (getattr(block, n, None) for n in ("norm", "attn", "ff"))
+ if norm is None or attn is None or ff is None:
+ return None
+ to_out = getattr(attn, "to_out", None)
+ act = getattr(ff, "act_fn", None)
+ if to_out is None or act is None or not hasattr(ff, "linear_in"):
+ return None
+ try:
+ out_proj = to_out[0]
+ except (TypeError, IndexError):
+ return None
+ if not _bias_free(attn.to_q, attn.to_k, attn.to_v, out_proj, getattr(ff, "linear_out", None)):
+ return None
+ eps = [
+ getattr(m, "eps", None)
+ for m in (norm, getattr(attn, "norm_q", None), getattr(attn, "norm_k", None))
+ ]
+ if not all(isinstance(e, float) for e in eps):
+ return None
+ return norm, attn, act
+
+
+def _residual_entries(root: Any) -> Optional[list]:
+ """Modules that write the residual stream from the inputs (text and per-mode image projections)."""
+ ctx = getattr(root, "context_embedder", None)
+ xs = getattr(root, "x_embedders", None)
+ if ctx is None or xs is None or not callable(getattr(xs, "values", None)):
+ return None
+ return [ctx, *xs.values()]
+
+
+def _install_rescale_residual(root: Any, scale: float) -> int:
+ """Residual stream at 1/scale: exact via bias-free projections, scale-invariant norms, eps / scale^2."""
+ if root.__dict__.get(_MARK) is not None:
+ return root.__dict__[_MARK][1]
+ entries = _residual_entries(root)
+ final = getattr(getattr(root, "norm_out", None), "norm", None)
+ if (
+ entries is None
+ or not _bias_free(*entries)
+ or not isinstance(getattr(final, "eps", None), float)
+ ):
+ return 0
+ found = [(b, sites) for b in root.modules() if (sites := _residual_block_sites(b)) is not None]
+ if not found:
+ return 0
+ inv = 1.0 / scale
+ eps_scaled = [final]
+ handles = []
+ for _block, (norm, attn, act) in found:
+ eps_scaled += [norm, attn.norm_q, attn.norm_k]
+ handles.append(attn.register_forward_pre_hook(_attention_pre_hook(inv), with_kwargs = True))
+ handles.append(act.register_forward_hook(_scale_output_hook(inv)))
+ for norm in eps_scaled:
+ _scale_eps(norm, inv * inv)
+ # Re-hook whatever sits in the entry slot per call (e.g. a later LoRA wrapper) so the stream is scaled once.
+ entry_hooks: dict[int, Any] = {}
+
+ def sync_entries(
+ module,
+ args,
+ kwargs = None,
+ ):
+ if _compiling():
+ return None
+ current = _residual_entries(root) or []
+ ids = {id(m) for m in current}
+ for key in [k for k in entry_hooks if k not in ids]:
+ entry_hooks.pop(key).remove()
+ for m in current:
+ if id(m) not in entry_hooks:
+ entry_hooks[id(m)] = m.register_forward_pre_hook(_scale_input_hook(inv))
+ return None
+
+ sync_entries(root, ())
+ handles.append(root.register_forward_pre_hook(sync_entries))
+
+ def undo():
+ for handle in (*handles, *entry_hooks.values()):
+ handle.remove()
+ entry_hooks.clear()
+ for norm in eps_scaled:
+ _scale_eps(norm, scale * scale)
+
+ root.__dict__[_MARK] = (undo, len(found))
+ return len(found)
+
+
def _install_rescale_post_norm(root: Any, attn_scale: float, ffn_scale: float) -> int:
import types
@@ -202,7 +330,10 @@ def install_fp16_guard(
if not callable(getattr(root, "named_modules", None)):
return 0
attn_scale, ffn_scale = scales
- sites = _install_rescale_post_norm(root, attn_scale, ffn_scale)
+ if recipe == "rescale_residual":
+ sites = _install_rescale_residual(root, attn_scale)
+ else:
+ sites = _install_rescale_post_norm(root, attn_scale, ffn_scale)
if logger is not None:
if sites:
logger.info(
@@ -229,6 +360,10 @@ def remove_fp16_guard(root: Any) -> int:
mark = block.__dict__.pop(_MARK, None)
if mark is None:
continue
+ if callable(mark[0]):
+ mark[0]()
+ removed += mark[1]
+ continue
attn_scale, ffn_scale, handles = mark
for handle in handles:
handle.remove()
diff --git a/studio/backend/tests/test_diffusion_flux3.py b/studio/backend/tests/test_diffusion_flux3.py
index 9dd83fe4b69..9803e7323fe 100644
--- a/studio/backend/tests/test_diffusion_flux3.py
+++ b/studio/backend/tests/test_diffusion_flux3.py
@@ -98,12 +98,15 @@ def test_family_records():
assert base.uses_negative_prompt is True
-def test_fp16_only_cards_promote_flux3_to_fp32():
+def test_fp16_only_cards_promote_flux3_to_fp32_without_its_guard(monkeypatch):
import torch
from core.inference.diffusion import _resolve_diffusion_compute_dtype
+ from core.inference.diffusion_fp16_guard import FP16_GUARD_ENV
+
+ monkeypatch.setenv(FP16_GUARD_ENV, "0")
for fam in (detect_family(DISTILLED), detect_family(BASE)):
- assert fam.fp16_incompatible is True and fam.fp16_guard is None
+ assert fam.fp16_incompatible is True and fam.fp16_guard == "rescale_residual"
assert _resolve_diffusion_compute_dtype(fam, torch.float16) is torch.float32
assert _resolve_diffusion_compute_dtype(fam, torch.bfloat16) is torch.bfloat16
assert _resolve_diffusion_compute_dtype(fam, torch.float32) is torch.float32
diff --git a/studio/backend/tests/test_diffusion_flux3_fp16_guard.py b/studio/backend/tests/test_diffusion_flux3_fp16_guard.py
new file mode 100644
index 00000000000..e8fe51c8b2b
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_flux3_fp16_guard.py
@@ -0,0 +1,318 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.3 fp16 residual rescale: kill switch, and finite fp32-equal output where plain fp16 overflows. CPU only."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+nn = torch.nn
+F = torch.nn.functional
+
+from core.inference import diffusion_fp16_guard as guard # noqa: E402
+from core.inference.diffusion import _resolve_diffusion_compute_dtype # noqa: E402
+from core.inference.diffusion_families import detect_family # noqa: E402
+
+_REAL_RECIPE_SUPPORTED = guard._recipe_supported
+
+DISTILLED = "black-forest-labs/flux-3-image-klein"
+BASE = "black-forest-labs/flux-3-image-klein-base"
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ monkeypatch.delenv(guard.FP16_GUARD_ENV, raising = False)
+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: True)
+
+
+def test_flux3_stays_fp16_behind_residual_guard():
+ for repo in (DISTILLED, BASE):
+ fam = detect_family(repo)
+ assert fam.fp16_incompatible is True and fam.fp16_guard == "rescale_residual"
+ assert guard.fp16_promotes_to_fp32(fam) is False
+ assert _resolve_diffusion_compute_dtype(fam, torch.float16) is torch.float16
+ assert _resolve_diffusion_compute_dtype(fam, torch.bfloat16) is torch.bfloat16
+
+
+def test_global_kill_switch_promotes_flux3(monkeypatch):
+ monkeypatch.setenv(guard.FP16_GUARD_ENV, "0")
+ assert (
+ _resolve_diffusion_compute_dtype(detect_family(DISTILLED), torch.float16) is torch.float32
+ )
+
+
+def test_unmatched_diffusers_source_promotes_flux3(monkeypatch):
+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: False)
+ assert (
+ _resolve_diffusion_compute_dtype(detect_family(DISTILLED), torch.float16) is torch.float32
+ )
+
+
[email protected]("value, guarded", [("0", False), ("off", False), ("1", True), ("", True)])
+def test_flux3_guard_kill_switch(value, guarded):
+ backend = Path(__file__).resolve().parents[1]
+ env = {**os.environ, "UNSLOTH_STUDIO_FLUX3_FP16_GUARD": value, "PYTHONPATH": str(backend)}
+ code = (
+ "from core.inference.diffusion_families import detect_family as d;"
+ f"print(d({DISTILLED!r}).fp16_guard, d({BASE!r}).fp16_guard)"
+ )
+ out = subprocess.run(
+ [sys.executable, "-c", code], cwd = backend, env = env, capture_output = True, text = True
+ )
+ assert out.returncode == 0, out.stderr[-2000:]
+ want = "rescale_residual" if guarded else "None"
+ assert out.stdout.split()[-2:] == [want, want]
+
+
+# --- A FLUX.3-shaped model: the diffusers forward bodies, tiny widths, weights raised until float16 overflows. ---
+
+
+class _Attn(nn.Module):
+ def __init__(self, dim, heads):
+ super().__init__()
+ self.heads = heads
+ self.to_q = nn.Linear(dim, dim, bias = False)
+ self.to_k = nn.Linear(dim, dim, bias = False)
+ self.to_v = nn.Linear(dim, dim, bias = False)
+ self.norm_q = nn.RMSNorm(dim // heads, eps = 1e-6)
+ self.norm_k = nn.RMSNorm(dim // heads, eps = 1e-6)
+ self.to_out = nn.ModuleList([nn.Linear(dim, dim, bias = False), nn.Dropout(0.0)])
+
+ def forward(self, hidden_states):
+ q, k, v = (
+ p(hidden_states).unflatten(-1, (self.heads, -1))
+ for p in (self.to_q, self.to_k, self.to_v)
+ )
+ q = self.norm_q(q.float()).to(v.dtype).transpose(1, 2)
+ k = self.norm_k(k.float()).to(v.dtype).transpose(1, 2)
+ o = F.scaled_dot_product_attention(q, k, v.transpose(1, 2)).transpose(1, 2).flatten(2)
+ return self.to_out[1](self.to_out[0](o))
+
+
+class _SwiGLU(nn.Module):
+ def forward(self, x):
+ half = x.shape[-1] // 2
+ return F.silu(x[..., :half]) * x[..., half:]
+
+
+class _FF(nn.Module):
+ def __init__(self, dim):
+ super().__init__()
+ self.linear_in = nn.Linear(dim, 6 * dim, bias = False)
+ self.act_fn = _SwiGLU()
+ self.linear_out = nn.Linear(3 * dim, dim, bias = False)
+
+ def forward(self, x):
+ return self.linear_out(self.act_fn(self.linear_in(x)))
+
+
+class _Mode(nn.Module):
+ def __init__(self, dim, heads):
+ super().__init__()
+ self.norm = nn.LayerNorm(dim, elementwise_affine = False, eps = 1e-6)
+ self.attn = _Attn(dim, heads)
+ self.ff = _FF(dim)
+
+ def forward(self, hidden_states, temb_mod):
+ shift, scale, gate = torch.chunk(temb_mod, 3, dim = -1)
+ n = (1 + scale) * self.norm(hidden_states) + shift
+ return hidden_states + gate * (self.attn(hidden_states = n) + self.ff(n))
+
+
+class _Single(_Mode):
+ def forward(self, hidden_states, encoder_hidden_states, temb_mod):
+ shift, scale, gate = torch.chunk(temb_mod, 3, dim = -1)
+ n = (1 + scale) * self.norm(hidden_states) + shift
+ c = (1 + scale) * self.norm(encoder_hidden_states) + shift
+ combined = torch.cat([c, n], dim = 1)
+ out = self.attn(hidden_states = combined) + self.ff(combined)
+ ctx, img = torch.split(out, [encoder_hidden_states.shape[1], hidden_states.shape[1]], dim = 1)
+ return encoder_hidden_states + gate * ctx, hidden_states + gate * img
+
+
+class _OutNorm(nn.Module):
+ def __init__(self, dim):
+ super().__init__()
+ self.norm = nn.LayerNorm(dim, elementwise_affine = False, eps = 1e-6)
+
+ def forward(self, x):
+ return self.norm(x)
+
+
+class _Flux3Like(nn.Module):
+ def __init__(
+ self,
+ dim = 64,
+ heads = 4,
+ ctx_dim = 96,
+ ):
+ super().__init__()
+ self.context_embedder = nn.Linear(ctx_dim, dim, bias = False)
+ self.x_embedders = nn.ModuleDict({"image": nn.Linear(16, dim, bias = False)})
+ self.txt_mode_blocks = nn.ModuleList([_Mode(dim, heads) for _ in range(2)])
+ self.mode_blocks = nn.ModuleDict(
+ {"image": nn.ModuleList([_Mode(dim, heads) for _ in range(2)])}
+ )
+ self.single_transformer_blocks = nn.ModuleList([_Single(dim, heads) for _ in range(3)])
+ self.norm_out = _OutNorm(dim)
+ self.proj_out = nn.Linear(dim, 16, bias = False)
+ self.mod = nn.Parameter(torch.rand(3 * dim) * 0.5)
+
+ def forward(self, hidden_states, encoder_hidden_states):
+ mod = self.mod.to(hidden_states.dtype)
+ enc = self.context_embedder(encoder_hidden_states)
+ img = self.x_embedders["image"](hidden_states)
+ for b in self.txt_mode_blocks:
+ enc = b(enc, mod)
+ for b in self.mode_blocks["image"]:
+ img = b(img, mod)
+ for b in self.single_transformer_blocks:
+ enc, img = b(img, enc, mod)
+ return self.proj_out(self.norm_out(img))
+
+
+def _overflowing_model():
+ torch.manual_seed(0)
+ model = _Flux3Like()
+ with torch.no_grad():
+ model.context_embedder.weight.mul_(3000.0)
+ for b in model.single_transformer_blocks:
+ b.ff.linear_out.weight.mul_(2000.0)
+ return model
+
+
+def _inputs():
+ torch.manual_seed(1)
+ return torch.randn(1, 12, 16), torch.randn(1, 6, 96) * 30
+
+
+def _fp16_ok():
+ try:
+ nn.Linear(4, 4).half()(torch.ones(1, 4, dtype = torch.float16))
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+needs_fp16 = pytest.mark.skipif(not _fp16_ok(), reason = "this torch has no CPU float16 matmul")
+
+
+@needs_fp16
+def test_residual_guard_keeps_flux3_shape_finite_and_accurate():
+ model = _overflowing_model()
+ x, ctx = _inputs()
+ with torch.no_grad():
+ ref = model.float()(x, ctx)
+ assert model.context_embedder(ctx).abs().max() > 65504
+ half = model.half()
+ stock = half(x.half(), ctx.half())
+ assert not torch.isfinite(stock).all()
+ assert guard.install_fp16_guard(half, "rescale_residual", torch.float16) == 7
+ fixed = half(x.half(), ctx.half())
+ assert torch.isfinite(fixed).all()
+ err = (fixed.float() - ref).abs().max() / ref.abs().max()
+ assert err < 2e-2, float(err)
+
+
+@needs_fp16
+def test_residual_guard_is_idempotent_and_removable():
+ model = _overflowing_model().half()
+ eps = [m.eps for m in model.modules() if isinstance(m, (nn.LayerNorm, nn.RMSNorm))]
+ assert guard.install_fp16_guard(model, "rescale_residual", torch.float16) == 7
+ assert guard.install_fp16_guard(model, "rescale_residual", torch.float16) == 7
+ assert model.norm_out.norm.eps == 1e-6 / 1024**2
+ assert guard.remove_fp16_guard(model) == 7
+ assert [m.eps for m in model.modules() if isinstance(m, (nn.LayerNorm, nn.RMSNorm))] == eps
+ assert not any(m._forward_pre_hooks or m._forward_hooks for m in model.modules())
+
+
+@needs_fp16
+def test_residual_guard_follows_a_wrapped_entry_projection():
+ """A LoRA-style wrapper swapped into the entry slot after install scales the stream once."""
+
+ class Wrap(nn.Module):
+ def __init__(self, base):
+ super().__init__()
+ self.base_layer = base
+ self.delta = nn.Linear(base.in_features, base.out_features, bias = False)
+
+ def forward(self, x):
+ return self.base_layer(x) + self.delta(x)
+
+ x, ctx = _inputs()
+ model2 = _overflowing_model()
+ with torch.no_grad():
+ model2.context_embedder = Wrap(model2.context_embedder)
+ model2.context_embedder.delta.weight.normal_(0, 50.0)
+ ref = model2(x, ctx)
+ half2 = _overflowing_model().half()
+ assert guard.install_fp16_guard(half2, "rescale_residual", torch.float16) == 7
+ half2.context_embedder = Wrap(half2.context_embedder).half()
+ half2.context_embedder.delta.weight.copy_(model2.context_embedder.delta.weight.half())
+ out = half2(x.half(), ctx.half())
+ assert torch.isfinite(out).all()
+ err = (out.float() - ref).abs().max() / ref.abs().max()
+ assert err < 2e-2, float(err)
+ assert not half2.context_embedder.base_layer._forward_pre_hooks
+
+
+def test_residual_guard_needs_bias_free_projections():
+ model = _overflowing_model()
+ model.single_transformer_blocks[0].ff.linear_out = nn.Linear(192, 64, bias = True)
+ assert guard.install_fp16_guard(model.half(), "rescale_residual", torch.float16) == 6
+ model = _overflowing_model()
+ model.context_embedder = nn.Linear(96, 64, bias = True)
+ assert guard.install_fp16_guard(model.half(), "rescale_residual", torch.float16) == 0
+
+
[email protected]("dtype", ["bfloat16", "float32"])
+def test_residual_guard_never_touches_non_fp16(dtype):
+ model = _overflowing_model().to(getattr(torch, dtype))
+ assert guard.install_fp16_guard(model, "rescale_residual", getattr(torch, dtype)) == 0
+ assert not any(m._forward_pre_hooks or m._forward_hooks for m in model.modules())
+
+
+@needs_fp16
+def test_residual_guard_on_real_flux3_class():
+ """The installed diffusers FLUX.3 transformer matches the recipe's source tokens and every block takes it."""
+ mod = pytest.importorskip("diffusers.models.transformers.transformer_flux3")
+ guard._SUPPORTED.clear()
+ assert _REAL_RECIPE_SUPPORTED(detect_family(DISTILLED), "rescale_residual") is True
+ torch.manual_seed(0)
+ tr = mod.Flux3Transformer2DModel(
+ num_layers = 1,
+ num_single_layers = 2,
+ attention_head_dim = 16,
+ num_attention_heads = 2,
+ joint_attention_dim = 48,
+ axes_dims_rope = (4, 4, 4, 4),
+ flow_map = True,
+ )
+ x = torch.randn(1, 16, 128)
+ ctx = torch.randn(1, 8, 48)
+ img_ids = torch.zeros(16, 4)
+ txt_ids = torch.zeros(8, 4)
+ t = torch.full((1,), 0.7)
+ with torch.no_grad():
+ ref = tr(x, ctx, t, img_ids, txt_ids, target_timestep = t * 0.5, return_dict = False)[0]
+ half = tr.half()
+ assert guard.install_fp16_guard(half, "rescale_residual", torch.float16) == 5
+ out = half(
+ x.half(),
+ ctx.half(),
+ t.half(),
+ img_ids,
+ txt_ids,
+ target_timestep = (t * 0.5).half(),
+ return_dict = False,
+ )[0]
+ assert torch.isfinite(out).all()
+ err = (out.float() - ref).abs().max() / ref.abs().max()
+ assert err < 3e-2, float(err)