Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio FLUX.3: keep float16 on fp16-only GPUs behind a residual rescale (T4 INT8 -24% per image, GGUF Q8_0 -73%) (#17385)
Daniel Handd43f1028152
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.pyindex f0222b974b3..527e0048160 100644--- a/studio/backend/core/inference/diffusion_families.py+++ b/studio/backend/core/inference/diffusion_families.py@@ -334,7 +334,7 @@ _QWENIMAGE_RDNA_AUTO_W8A8 = ("compiled int8 W8A8 measured 133.1 vs 141.2 s/image against compiled bf16 on gfx1151 (20 steps CFG 4), ""LPIPS 0.014")-# FLUX.3 renders black in float16, so fp16-only cards run it in float32; UNSLOTH_STUDIO_FLUX3_FP16=1 keeps fp16.+# FLUX.3 renders black in plain float16; UNSLOTH_STUDIO_FLUX3_FP16=1 keeps fp16._FLUX3_FP16_INCOMPATIBLE = (os.environ.get("UNSLOTH_STUDIO_FLUX3_FP16") or "").strip().lower() not in (@@ -343,6 +343,13 @@ _FLUX3_FP16_INCOMPATIBLE = ("true","yes",)+# fp16 cards keep FLUX.3 in fp16 via the residual rescale; UNSLOTH_STUDIO_FLUX3_FP16_GUARD=0 promotes to fp32.+_FLUX3_FP16_GUARD = (+ None+ if (os.environ.get("UNSLOTH_STUDIO_FLUX3_FP16_GUARD") or "").strip().lower()+ in ("0", "off", "false", "no")+ else "rescale_residual"+)# FLUX.3 Klein transformer-quant keep-list, every scheme (FamilyProfile.quant_exclude_tokens)._FLUX3_QUANT_KEEP = ("modulation",@@ -652,6 +659,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (name = "flux.3-klein-base",comfy_shift_node = "flux",fp16_incompatible = _FLUX3_FP16_INCOMPATIBLE,+ fp16_guard = _FLUX3_FP16_GUARD,label = "FLUX.3 Klein Base",profile = FamilyProfile(auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,@@ -711,6 +719,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (name = "flux.3-klein",comfy_shift_node = "flux",fp16_incompatible = _FLUX3_FP16_INCOMPATIBLE,+ fp16_guard = _FLUX3_FP16_GUARD,label = "FLUX.3 Klein",profile = FamilyProfile(auto_bf16_when_resident = SMALL_DIT_BF16_WHEN_RESIDENT,diff --git a/studio/backend/core/inference/diffusion_fp16_guard.py b/studio/backend/core/inference/diffusion_fp16_guard.pyindex c0f20863213..d77cacb11ea 100644--- a/studio/backend/core/inference/diffusion_fp16_guard.py+++ b/studio/backend/core/inference/diffusion_fp16_guard.py@@ -17,6 +17,8 @@ _FP16_MAX = 65504.0RECIPES: dict[str, Optional[tuple[float, float]]] = {"native": None,"rescale_post_norm": (16.0, 128.0),+ # FLUX.3 residual reaches ~1e7: whole stream at 1/1024, LayerNorm eps by 1/1024^2.+ "rescale_residual": (1024.0, 1024.0),}@@ -41,6 +43,16 @@ _RECIPE_TOKENS: dict[str, tuple[str, ...]] = {"self.w3(","norm_q",),+ "rescale_residual": (+ "self.context_embedder(encoder_hidden_states)",+ 'self.x_embedders["image"](hidden_states)',+ "return hidden_states + mod_gate * (attn_output + ff_output)",+ "encoder_hidden_states = encoder_hidden_states + c_mod_gate * context_output",+ "hidden_states = hidden_states + mod_gate * content_output",+ "x = self.gate_fn(x[..., :half]) * x[..., half:]",+ "return self.norm(hidden_states) * (1 + scale) + shift",+ "attn.norm_q(query.float())",+ ),}_SUPPORTED: dict[tuple[str, str], bool] = {}@@ -157,6 +169,122 @@ def _block_sites(block: Any) -> Optional[tuple[Any, ...]]:return attn, ffn, norms[0], norms[1]+def _scale_output_hook(inv: float):+ def hook(module, args, output):+ return output * inv++ return hook+++def _scale_input_hook(inv: float):+ def hook(module, args):+ return (args[0] * inv, *args[1:]) if args else None++ return hook+++def _compiling() -> bool:+ try:+ import torch+ return bool(torch.compiler.is_compiling())+ except Exception: # noqa: BLE001 - no torch.compiler: eager+ return False+++def _bias_free(*linears: Any) -> bool:+ return all(lin is not None and getattr(lin, "bias", None) is None for lin in linears)+++def _residual_block_sites(block: Any) -> Optional[tuple[Any, ...]]:+ """(norm, attn, act_fn) of a FLUX.3 mode / joint block, whose residual update is gate * (attn + ff)."""+ norm, attn, ff = (getattr(block, n, None) for n in ("norm", "attn", "ff"))+ if norm is None or attn is None or ff is None:+ return None+ to_out = getattr(attn, "to_out", None)+ act = getattr(ff, "act_fn", None)+ if to_out is None or act is None or not hasattr(ff, "linear_in"):+ return None+ try:+ out_proj = to_out[0]+ except (TypeError, IndexError):+ return None+ if not _bias_free(attn.to_q, attn.to_k, attn.to_v, out_proj, getattr(ff, "linear_out", None)):+ return None+ eps = [+ getattr(m, "eps", None)+ for m in (norm, getattr(attn, "norm_q", None), getattr(attn, "norm_k", None))+ ]+ if not all(isinstance(e, float) for e in eps):+ return None+ return norm, attn, act+++def _residual_entries(root: Any) -> Optional[list]:+ """Modules that write the residual stream from the inputs (text and per-mode image projections)."""+ ctx = getattr(root, "context_embedder", None)+ xs = getattr(root, "x_embedders", None)+ if ctx is None or xs is None or not callable(getattr(xs, "values", None)):+ return None+ return [ctx, *xs.values()]+++def _install_rescale_residual(root: Any, scale: float) -> int:+ """Residual stream at 1/scale: exact via bias-free projections, scale-invariant norms, eps / scale^2."""+ if root.__dict__.get(_MARK) is not None:+ return root.__dict__[_MARK][1]+ entries = _residual_entries(root)+ final = getattr(getattr(root, "norm_out", None), "norm", None)+ if (+ entries is None+ or not _bias_free(*entries)+ or not isinstance(getattr(final, "eps", None), float)+ ):+ return 0+ found = [(b, sites) for b in root.modules() if (sites := _residual_block_sites(b)) is not None]+ if not found:+ return 0+ inv = 1.0 / scale+ eps_scaled = [final]+ handles = []+ for _block, (norm, attn, act) in found:+ eps_scaled += [norm, attn.norm_q, attn.norm_k]+ handles.append(attn.register_forward_pre_hook(_attention_pre_hook(inv), with_kwargs = True))+ handles.append(act.register_forward_hook(_scale_output_hook(inv)))+ for norm in eps_scaled:+ _scale_eps(norm, inv * inv)+ # Re-hook whatever sits in the entry slot per call (e.g. a later LoRA wrapper) so the stream is scaled once.+ entry_hooks: dict[int, Any] = {}++ def sync_entries(+ module,+ args,+ kwargs = None,+ ):+ if _compiling():+ return None+ current = _residual_entries(root) or []+ ids = {id(m) for m in current}+ for key in [k for k in entry_hooks if k not in ids]:+ entry_hooks.pop(key).remove()+ for m in current:+ if id(m) not in entry_hooks:+ entry_hooks[id(m)] = m.register_forward_pre_hook(_scale_input_hook(inv))+ return None++ sync_entries(root, ())+ handles.append(root.register_forward_pre_hook(sync_entries))++ def undo():+ for handle in (*handles, *entry_hooks.values()):+ handle.remove()+ entry_hooks.clear()+ for norm in eps_scaled:+ _scale_eps(norm, scale * scale)++ root.__dict__[_MARK] = (undo, len(found))+ return len(found)++def _install_rescale_post_norm(root: Any, attn_scale: float, ffn_scale: float) -> int:import types@@ -202,7 +330,10 @@ def install_fp16_guard(if not callable(getattr(root, "named_modules", None)):return 0attn_scale, ffn_scale = scales- sites = _install_rescale_post_norm(root, attn_scale, ffn_scale)+ if recipe == "rescale_residual":+ sites = _install_rescale_residual(root, attn_scale)+ else:+ sites = _install_rescale_post_norm(root, attn_scale, ffn_scale)if logger is not None:if sites:logger.info(@@ -229,6 +360,10 @@ def remove_fp16_guard(root: Any) -> int:mark = block.__dict__.pop(_MARK, None)if mark is None:continue+ if callable(mark[0]):+ mark[0]()+ removed += mark[1]+ continueattn_scale, ffn_scale, handles = markfor handle in handles:handle.remove()diff --git a/studio/backend/tests/test_diffusion_flux3.py b/studio/backend/tests/test_diffusion_flux3.pyindex 9dd83fe4b69..9803e7323fe 100644--- a/studio/backend/tests/test_diffusion_flux3.py+++ b/studio/backend/tests/test_diffusion_flux3.py@@ -98,12 +98,15 @@ def test_family_records():assert base.uses_negative_prompt is True-def test_fp16_only_cards_promote_flux3_to_fp32():+def test_fp16_only_cards_promote_flux3_to_fp32_without_its_guard(monkeypatch):import torchfrom core.inference.diffusion import _resolve_diffusion_compute_dtype+ from core.inference.diffusion_fp16_guard import FP16_GUARD_ENV++ monkeypatch.setenv(FP16_GUARD_ENV, "0")for fam in (detect_family(DISTILLED), detect_family(BASE)):- assert fam.fp16_incompatible is True and fam.fp16_guard is None+ assert fam.fp16_incompatible is True and fam.fp16_guard == "rescale_residual"assert _resolve_diffusion_compute_dtype(fam, torch.float16) is torch.float32assert _resolve_diffusion_compute_dtype(fam, torch.bfloat16) is torch.bfloat16assert _resolve_diffusion_compute_dtype(fam, torch.float32) is torch.float32diff --git a/studio/backend/tests/test_diffusion_flux3_fp16_guard.py b/studio/backend/tests/test_diffusion_flux3_fp16_guard.pynew file mode 100644index 00000000000..e8fe51c8b2b--- /dev/null+++ b/studio/backend/tests/test_diffusion_flux3_fp16_guard.py@@ -0,0 +1,318 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.3 fp16 residual rescale: kill switch, and finite fp32-equal output where plain fp16 overflows. CPU only."""++from __future__ import annotations++import os+import subprocess+import sys+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+nn = torch.nn+F = torch.nn.functional++from core.inference import diffusion_fp16_guard as guard # noqa: E402+from core.inference.diffusion import _resolve_diffusion_compute_dtype # noqa: E402+from core.inference.diffusion_families import detect_family # noqa: E402++_REAL_RECIPE_SUPPORTED = guard._recipe_supported++DISTILLED = "black-forest-labs/flux-3-image-klein"+BASE = "black-forest-labs/flux-3-image-klein-base"++[email protected](autouse = True)+def _clean_env(monkeypatch):+ monkeypatch.delenv(guard.FP16_GUARD_ENV, raising = False)+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: True)+++def test_flux3_stays_fp16_behind_residual_guard():+ for repo in (DISTILLED, BASE):+ fam = detect_family(repo)+ assert fam.fp16_incompatible is True and fam.fp16_guard == "rescale_residual"+ assert guard.fp16_promotes_to_fp32(fam) is False+ assert _resolve_diffusion_compute_dtype(fam, torch.float16) is torch.float16+ assert _resolve_diffusion_compute_dtype(fam, torch.bfloat16) is torch.bfloat16+++def test_global_kill_switch_promotes_flux3(monkeypatch):+ monkeypatch.setenv(guard.FP16_GUARD_ENV, "0")+ assert (+ _resolve_diffusion_compute_dtype(detect_family(DISTILLED), torch.float16) is torch.float32+ )+++def test_unmatched_diffusers_source_promotes_flux3(monkeypatch):+ monkeypatch.setattr(guard, "_recipe_supported", lambda fam, recipe: False)+ assert (+ _resolve_diffusion_compute_dtype(detect_family(DISTILLED), torch.float16) is torch.float32+ )++[email protected]("value, guarded", [("0", False), ("off", False), ("1", True), ("", True)])+def test_flux3_guard_kill_switch(value, guarded):+ backend = Path(__file__).resolve().parents[1]+ env = {**os.environ, "UNSLOTH_STUDIO_FLUX3_FP16_GUARD": value, "PYTHONPATH": str(backend)}+ code = (+ "from core.inference.diffusion_families import detect_family as d;"+ f"print(d({DISTILLED!r}).fp16_guard, d({BASE!r}).fp16_guard)"+ )+ out = subprocess.run(+ [sys.executable, "-c", code], cwd = backend, env = env, capture_output = True, text = True+ )+ assert out.returncode == 0, out.stderr[-2000:]+ want = "rescale_residual" if guarded else "None"+ assert out.stdout.split()[-2:] == [want, want]+++# --- A FLUX.3-shaped model: the diffusers forward bodies, tiny widths, weights raised until float16 overflows. ---+++class _Attn(nn.Module):+ def __init__(self, dim, heads):+ super().__init__()+ self.heads = heads+ self.to_q = nn.Linear(dim, dim, bias = False)+ self.to_k = nn.Linear(dim, dim, bias = False)+ self.to_v = nn.Linear(dim, dim, bias = False)+ self.norm_q = nn.RMSNorm(dim // heads, eps = 1e-6)+ self.norm_k = nn.RMSNorm(dim // heads, eps = 1e-6)+ self.to_out = nn.ModuleList([nn.Linear(dim, dim, bias = False), nn.Dropout(0.0)])++ def forward(self, hidden_states):+ q, k, v = (+ p(hidden_states).unflatten(-1, (self.heads, -1))+ for p in (self.to_q, self.to_k, self.to_v)+ )+ q = self.norm_q(q.float()).to(v.dtype).transpose(1, 2)+ k = self.norm_k(k.float()).to(v.dtype).transpose(1, 2)+ o = F.scaled_dot_product_attention(q, k, v.transpose(1, 2)).transpose(1, 2).flatten(2)+ return self.to_out[1](self.to_out[0](o))+++class _SwiGLU(nn.Module):+ def forward(self, x):+ half = x.shape[-1] // 2+ return F.silu(x[..., :half]) * x[..., half:]+++class _FF(nn.Module):+ def __init__(self, dim):+ super().__init__()+ self.linear_in = nn.Linear(dim, 6 * dim, bias = False)+ self.act_fn = _SwiGLU()+ self.linear_out = nn.Linear(3 * dim, dim, bias = False)++ def forward(self, x):+ return self.linear_out(self.act_fn(self.linear_in(x)))+++class _Mode(nn.Module):+ def __init__(self, dim, heads):+ super().__init__()+ self.norm = nn.LayerNorm(dim, elementwise_affine = False, eps = 1e-6)+ self.attn = _Attn(dim, heads)+ self.ff = _FF(dim)++ def forward(self, hidden_states, temb_mod):+ shift, scale, gate = torch.chunk(temb_mod, 3, dim = -1)+ n = (1 + scale) * self.norm(hidden_states) + shift+ return hidden_states + gate * (self.attn(hidden_states = n) + self.ff(n))+++class _Single(_Mode):+ def forward(self, hidden_states, encoder_hidden_states, temb_mod):+ shift, scale, gate = torch.chunk(temb_mod, 3, dim = -1)+ n = (1 + scale) * self.norm(hidden_states) + shift+ c = (1 + scale) * self.norm(encoder_hidden_states) + shift+ combined = torch.cat([c, n], dim = 1)+ out = self.attn(hidden_states = combined) + self.ff(combined)+ ctx, img = torch.split(out, [encoder_hidden_states.shape[1], hidden_states.shape[1]], dim = 1)+ return encoder_hidden_states + gate * ctx, hidden_states + gate * img+++class _OutNorm(nn.Module):+ def __init__(self, dim):+ super().__init__()+ self.norm = nn.LayerNorm(dim, elementwise_affine = False, eps = 1e-6)++ def forward(self, x):+ return self.norm(x)+++class _Flux3Like(nn.Module):+ def __init__(+ self,+ dim = 64,+ heads = 4,+ ctx_dim = 96,+ ):+ super().__init__()+ self.context_embedder = nn.Linear(ctx_dim, dim, bias = False)+ self.x_embedders = nn.ModuleDict({"image": nn.Linear(16, dim, bias = False)})+ self.txt_mode_blocks = nn.ModuleList([_Mode(dim, heads) for _ in range(2)])+ self.mode_blocks = nn.ModuleDict(+ {"image": nn.ModuleList([_Mode(dim, heads) for _ in range(2)])}+ )+ self.single_transformer_blocks = nn.ModuleList([_Single(dim, heads) for _ in range(3)])+ self.norm_out = _OutNorm(dim)+ self.proj_out = nn.Linear(dim, 16, bias = False)+ self.mod = nn.Parameter(torch.rand(3 * dim) * 0.5)++ def forward(self, hidden_states, encoder_hidden_states):+ mod = self.mod.to(hidden_states.dtype)+ enc = self.context_embedder(encoder_hidden_states)+ img = self.x_embedders["image"](hidden_states)+ for b in self.txt_mode_blocks:+ enc = b(enc, mod)+ for b in self.mode_blocks["image"]:+ img = b(img, mod)+ for b in self.single_transformer_blocks:+ enc, img = b(img, enc, mod)+ return self.proj_out(self.norm_out(img))+++def _overflowing_model():+ torch.manual_seed(0)+ model = _Flux3Like()+ with torch.no_grad():+ model.context_embedder.weight.mul_(3000.0)+ for b in model.single_transformer_blocks:+ b.ff.linear_out.weight.mul_(2000.0)+ return model+++def _inputs():+ torch.manual_seed(1)+ return torch.randn(1, 12, 16), torch.randn(1, 6, 96) * 30+++def _fp16_ok():+ try:+ nn.Linear(4, 4).half()(torch.ones(1, 4, dtype = torch.float16))+ return True+ except Exception: # noqa: BLE001+ return False+++needs_fp16 = pytest.mark.skipif(not _fp16_ok(), reason = "this torch has no CPU float16 matmul")+++@needs_fp16+def test_residual_guard_keeps_flux3_shape_finite_and_accurate():+ model = _overflowing_model()+ x, ctx = _inputs()+ with torch.no_grad():+ ref = model.float()(x, ctx)+ assert model.context_embedder(ctx).abs().max() > 65504+ half = model.half()+ stock = half(x.half(), ctx.half())+ assert not torch.isfinite(stock).all()+ assert guard.install_fp16_guard(half, "rescale_residual", torch.float16) == 7+ fixed = half(x.half(), ctx.half())+ assert torch.isfinite(fixed).all()+ err = (fixed.float() - ref).abs().max() / ref.abs().max()+ assert err < 2e-2, float(err)+++@needs_fp16+def test_residual_guard_is_idempotent_and_removable():+ model = _overflowing_model().half()+ eps = [m.eps for m in model.modules() if isinstance(m, (nn.LayerNorm, nn.RMSNorm))]+ assert guard.install_fp16_guard(model, "rescale_residual", torch.float16) == 7+ assert guard.install_fp16_guard(model, "rescale_residual", torch.float16) == 7+ assert model.norm_out.norm.eps == 1e-6 / 1024**2+ assert guard.remove_fp16_guard(model) == 7+ assert [m.eps for m in model.modules() if isinstance(m, (nn.LayerNorm, nn.RMSNorm))] == eps+ assert not any(m._forward_pre_hooks or m._forward_hooks for m in model.modules())+++@needs_fp16+def test_residual_guard_follows_a_wrapped_entry_projection():+ """A LoRA-style wrapper swapped into the entry slot after install scales the stream once."""++ class Wrap(nn.Module):+ def __init__(self, base):+ super().__init__()+ self.base_layer = base+ self.delta = nn.Linear(base.in_features, base.out_features, bias = False)++ def forward(self, x):+ return self.base_layer(x) + self.delta(x)++ x, ctx = _inputs()+ model2 = _overflowing_model()+ with torch.no_grad():+ model2.context_embedder = Wrap(model2.context_embedder)+ model2.context_embedder.delta.weight.normal_(0, 50.0)+ ref = model2(x, ctx)+ half2 = _overflowing_model().half()+ assert guard.install_fp16_guard(half2, "rescale_residual", torch.float16) == 7+ half2.context_embedder = Wrap(half2.context_embedder).half()+ half2.context_embedder.delta.weight.copy_(model2.context_embedder.delta.weight.half())+ out = half2(x.half(), ctx.half())+ assert torch.isfinite(out).all()+ err = (out.float() - ref).abs().max() / ref.abs().max()+ assert err < 2e-2, float(err)+ assert not half2.context_embedder.base_layer._forward_pre_hooks+++def test_residual_guard_needs_bias_free_projections():+ model = _overflowing_model()+ model.single_transformer_blocks[0].ff.linear_out = nn.Linear(192, 64, bias = True)+ assert guard.install_fp16_guard(model.half(), "rescale_residual", torch.float16) == 6+ model = _overflowing_model()+ model.context_embedder = nn.Linear(96, 64, bias = True)+ assert guard.install_fp16_guard(model.half(), "rescale_residual", torch.float16) == 0++[email protected]("dtype", ["bfloat16", "float32"])+def test_residual_guard_never_touches_non_fp16(dtype):+ model = _overflowing_model().to(getattr(torch, dtype))+ assert guard.install_fp16_guard(model, "rescale_residual", getattr(torch, dtype)) == 0+ assert not any(m._forward_pre_hooks or m._forward_hooks for m in model.modules())+++@needs_fp16+def test_residual_guard_on_real_flux3_class():+ """The installed diffusers FLUX.3 transformer matches the recipe's source tokens and every block takes it."""+ mod = pytest.importorskip("diffusers.models.transformers.transformer_flux3")+ guard._SUPPORTED.clear()+ assert _REAL_RECIPE_SUPPORTED(detect_family(DISTILLED), "rescale_residual") is True+ torch.manual_seed(0)+ tr = mod.Flux3Transformer2DModel(+ num_layers = 1,+ num_single_layers = 2,+ attention_head_dim = 16,+ num_attention_heads = 2,+ joint_attention_dim = 48,+ axes_dims_rope = (4, 4, 4, 4),+ flow_map = True,+ )+ x = torch.randn(1, 16, 128)+ ctx = torch.randn(1, 8, 48)+ img_ids = torch.zeros(16, 4)+ txt_ids = torch.zeros(8, 4)+ t = torch.full((1,), 0.7)+ with torch.no_grad():+ ref = tr(x, ctx, t, img_ids, txt_ids, target_timestep = t * 0.5, return_dict = False)[0]+ half = tr.half()+ assert guard.install_fp16_guard(half, "rescale_residual", torch.float16) == 5+ out = half(+ x.half(),+ ctx.half(),+ t.half(),+ img_ids,+ txt_ids,+ target_timestep = (t * 0.5).half(),+ return_dict = False,+ )[0]+ assert torch.isfinite(out).all()+ err = (out.float() - ref).abs().max() / ref.abs().max()+ assert err < 3e-2, float(err)