Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run FP8 on T4 and other Turing GPUs as weight-only fp8 instead of refusing it (#17218)

Daniel Hanbf9039df56c0
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 1ee71a0e262..5721bb93cce 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -532,7 +532,8 @@ from .diffusion_transformer_quant import (
explain_unusable_scheme,
family_denies_scheme,
fp16_card_int8_host,
- fp16_card_int8_scheme,
+ fp16_card_native_host,
+ fp16_card_native_scheme,
fp16_card_quant_storage_enabled,
mark_source_precision,
nvfp4_gated_base,
@@ -4520,8 +4521,11 @@ class DiffusionBackend:
):
if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
- elif model_kind == "pipeline" and pinned == TQ_INT8 and fp16_card_int8_host(target):
- if fp16_card_int8_scheme(target, pinned, family = getattr(fam, "name", None)) is None:
+ elif model_kind == "pipeline" and fp16_card_native_host(target, pinned):
+ if (
+ fp16_card_native_scheme(target, pinned, family = getattr(fam, "name", None))
+ is None
+ ):
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
elif (
model_kind == "pipeline"
@@ -8662,7 +8666,7 @@ class DiffusionBackend:
TQ_INT8 if rdna_auto_w8a8 is not None else transformer_quant_pinned,
family = getattr(fam, "name", None),
)
- or fp16_card_int8_scheme(
+ or fp16_card_native_scheme(
target, transformer_quant_pinned, family = getattr(fam, "name", None)
)
if kind == "pipeline"
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 4a158cc5bf1..6b661bbc6f4 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -1269,6 +1269,51 @@ def fp16_card_int8_scheme(
return TQ_INT8
+FP16_CARD_FP8_ENV = "UNSLOTH_DIFFUSION_FP16_CARD_FP8"
+
+
+def fp16_card_fp8_enabled() -> bool:
+ raw = (_os.environ.get(FP16_CARD_FP8_ENV) or "").strip().lower()
+ return raw not in ("0", "false", "no", "off")
+
+
+def fp16_card_fp8_scheme(
+ target: Any,
+ requested: Optional[str],
+ family: Optional[str] = None,
+) -> Optional[str]:
+ """``fp8`` for an explicit fp8 request on an ``fp16_card_int8_host``, else None. Weight-only (no fp8 matmul
+ below sm_89); ``UNSLOTH_DIFFUSION_FP16_CARD_FP8=0`` restores the refusal."""
+ if normalize_transformer_quant(requested) != TQ_FP8 or not fp16_card_fp8_enabled():
+ return None
+ if not fp16_card_int8_host(target) or _family_denied(family, TQ_FP8):
+ return None
+ try:
+ import torch
+ if not hasattr(torch, "float8_e4m3fn"):
+ return None
+ except Exception: # noqa: BLE001
+ return None
+ return TQ_FP8
+
+
+def fp16_card_native_host(target: Any, requested: Optional[str]) -> bool:
+ scheme = normalize_transformer_quant(requested)
+ if scheme == TQ_FP8 and not fp16_card_fp8_enabled():
+ return False
+ return scheme in (TQ_INT8, TQ_FP8) and fp16_card_int8_host(target)
+
+
+def fp16_card_native_scheme(
+ target: Any,
+ requested: Optional[str],
+ family: Optional[str] = None,
+) -> Optional[str]:
+ return fp16_card_int8_scheme(target, requested, family) or fp16_card_fp8_scheme(
+ target, requested, family
+ )
+
+
def fp16_card_int8_act(target: Any) -> bool:
"""W8A8 for the explicit-int8 Turing build; ``int8_storage_host`` (also true there) answers only for checkpoint
int8 storage, which stays weight-only."""
@@ -2172,11 +2217,12 @@ def quantize_transformer(
) -> Optional[str]:
"""Quantise ``pipe.transformer`` in place, returning the scheme or None (GGUF); ``base_repo`` picks the NVFP4 policy."""
native = native_quant_scheme(target, mode, family = family, offload = offload)
- fp16_card = native is None and fp16_card_int8_scheme(target, mode, family) is not None
+ fp16_card_scheme = fp16_card_native_scheme(target, mode, family) if native is None else None
+ fp16_card = fp16_card_scheme is not None
if fp16_card:
- native = TQ_INT8
- # native_int8_act is False on any fp16 CUDA target (int8_storage_host); this build is the W8A8 one.
- act_int8 = fp16_card_int8_act(target)
+ native = fp16_card_scheme
+ # native_int8_act is False on fp16 CUDA targets; int8 here is W8A8, fp8 weight-only.
+ act_int8 = fp16_card_int8_act(target) if native == TQ_INT8 else False
if native is not None:
mps = _mps_bf16(target)
if act_int8 is None or mps:
diff --git a/studio/backend/tests/test_diffusion_fp16_card_fp8.py b/studio/backend/tests/test_diffusion_fp16_card_fp8.py
new file mode 100644
index 00000000000..57de8439345
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_fp16_card_fp8.py
@@ -0,0 +1,209 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Weight-only fp8 on fp16-only NVIDIA cards (Turing); capability is monkeypatched, no GPU needed."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+if not hasattr(torch, "float8_e4m3fn"):
+ pytest.skip("torch without float8 dtypes", allow_module_level = True)
+
+import core.inference.diffusion_native_quant as nq # noqa: E402
+import core.inference.diffusion_transformer_quant as tq # noqa: E402
+
+# Literals, not tq.*_ENV: the behavioural tests must also run against a tree without the names.
+FP8_KILL_SWITCH = "UNSLOTH_DIFFUSION_FP16_CARD_FP8"
+INT8_KILL_SWITCH = "UNSLOTH_DIFFUSION_FP16_CARD_INT8"
+
+
+def _target(dtype, device = "cuda"):
+ return types.SimpleNamespace(device = device, dtype = dtype)
+
+
+def nvidia(monkeypatch):
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)
+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)
+ monkeypatch.delenv(FP8_KILL_SWITCH, raising = False)
+ monkeypatch.delenv(INT8_KILL_SWITCH, raising = False)
+
+
+def _cap(monkeypatch, cap):
+ monkeypatch.setattr(tq, "_capability", lambda ordinal = None: cap)
+
+
+class _Toy(torch.nn.Module):
+ def __init__(self, dim = 128):
+ super().__init__()
+ self.to_q = torch.nn.Linear(dim, dim)
+ self.ff = torch.nn.Linear(dim, 2 * dim)
+ self.out = torch.nn.Linear(2 * dim, dim)
+
+ def forward(self, x):
+ return self.out(torch.nn.functional.gelu(self.ff(self.to_q(x))))
+
+
[email protected]("dtype", [torch.float16, torch.float32])
+def test_turing_fp16_and_fp32_pipelines_take_fp8(nvidia, monkeypatch, dtype):
+ _cap(monkeypatch, (7, 5))
+ target = _target(dtype)
+ assert tq.fp16_card_fp8_scheme(target, "fp8") == "fp8"
+ assert tq.fp16_card_native_scheme(target, " FP8 ") == "fp8"
+ assert tq.fp16_card_native_scheme(target, "int8") == "int8"
+ assert tq.fp16_card_native_host(target, "fp8") and tq.fp16_card_native_host(target, "int8")
+
+
[email protected]("scheme", [None, "auto", "none", "int8", "mxfp8"])
+def test_only_an_explicit_fp8_request_is_taken(nvidia, monkeypatch, scheme):
+ _cap(monkeypatch, (7, 5))
+ assert tq.fp16_card_fp8_scheme(_target(torch.float16), scheme) is None
+ if scheme not in ("int8",):
+ assert tq.fp16_card_native_scheme(_target(torch.float16), scheme) is None
+
+
+ "cap", [None, (6, 1), (7, 0), (8, 0), (8, 6), (8, 9), (9, 0), (10, 0), (12, 0)]
+)
[email protected]("dtype", [torch.float16, torch.float32, torch.bfloat16])
+def test_every_other_nvidia_card_is_unchanged(nvidia, monkeypatch, cap, dtype):
+ _cap(monkeypatch, cap)
+ assert tq.fp16_card_fp8_scheme(_target(dtype), "fp8") is None
+ assert not tq.fp16_card_native_host(_target(dtype), "fp8")
+
+
[email protected]("device", ["cpu", "mps", "xpu"])
+def test_bf16_turing_and_non_cuda_are_unchanged(nvidia, monkeypatch, device):
+ _cap(monkeypatch, (7, 5))
+ assert tq.fp16_card_fp8_scheme(_target(torch.bfloat16), "fp8") is None
+ assert tq.fp16_card_fp8_scheme(_target(torch.float16, device), "fp8") is None
+
+
[email protected]("raw", ["0", "false", "off", "no"])
+def test_kill_switch_restores_the_refusal(nvidia, monkeypatch, raw):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.setenv(FP8_KILL_SWITCH, raw)
+ assert tq.fp16_card_fp8_scheme(_target(torch.float32), "fp8") is None
+ assert not tq.fp16_card_native_host(_target(torch.float32), "fp8")
+ assert tq.fp16_card_native_scheme(_target(torch.float32), "int8") == "int8"
+
+
+def test_the_int8_kill_switch_also_closes_fp8(nvidia, monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.setenv(INT8_KILL_SWITCH, "0")
+ assert tq.fp16_card_fp8_scheme(_target(torch.float32), "fp8") is None
+
+
+def test_the_family_deny_list_still_applies(nvidia, monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "toy-family", ("fp8",))
+ assert tq.fp16_card_fp8_scheme(_target(torch.float16), "fp8", family = "toy-family") is None
+
+
+def test_the_shared_native_route_is_not_widened(nvidia, monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ assert tq.native_quant_scheme(_target(torch.float16), "fp8") is None
+ assert tq.select_transformer_quant_scheme(_target(torch.float16), "fp8") is None
+
+
[email protected]("dtype", [torch.float16, torch.float32])
+def test_quantize_transformer_builds_weight_only_fp8_in_the_pipeline_dtype(
+ nvidia, monkeypatch, dtype
+):
+ _cap(monkeypatch, (7, 5))
+ torch.manual_seed(0)
+ model = _Toy().to(dtype)
+ x = torch.randn(4, 128, dtype = dtype)
+ with torch.no_grad():
+ ref = model(x).float()
+ got = tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model), _target(dtype), mode = "fp8", min_features = 64
+ )
+ assert got == "fp8"
+ for name in ("to_q", "ff", "out"):
+ layer = getattr(model, name)
+ assert nq.is_native_linear(layer), name
+ assert layer.compute_dtype == dtype
+ assert layer.scheme == "fp8" and not layer.act_int8
+ assert layer.weight_q.element_size() == 1
+ with torch.no_grad():
+ y = model(x)
+ assert y.dtype == dtype and torch.isfinite(y).all()
+ rel = ((y.float() - ref).norm() / ref.norm()).item()
+ assert rel < 0.05, rel
+
+
+def test_quantize_transformer_on_ampere_fp16_still_declines(nvidia, monkeypatch):
+ _cap(monkeypatch, (8, 6))
+ model = _Toy().to(torch.float16)
+ got = tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model),
+ _target(torch.float16),
+ mode = "fp8",
+ min_features = 64,
+ )
+ assert got is None
+ assert not nq.is_native_linear(model.to_q)
+
+
+def _image_family(name):
+ from core.inference.diffusion_families import _FAMILIES
+ return next(f for f in _FAMILIES if f.name == name)
+
+
+def _image_gate(
+ monkeypatch,
+ fam,
+ pinned,
+ dtype,
+ *,
+ memory_mode = None,
+):
+ import core.inference.diffusion as d
+ monkeypatch.setattr(d, "effective_te_quant", lambda *a, **k: None)
+ d.DiffusionBackend._assert_precision_for_target(
+ None,
+ fam,
+ _target(dtype),
+ model_kind = "pipeline",
+ pinned = pinned,
+ te_mode = None,
+ memory_mode = memory_mode,
+ cpu_offload = False,
+ )
+
+
+ "family,dtype",
+ [
+ ("qwen-image-2.1", torch.float16),
+ ("flux.3-klein", torch.float32),
+ ("flux.3-klein", torch.float16),
+ ],
+)
[email protected]("memory_mode", [None, "low_vram"])
+def test_image_gate_admits_fp8_on_turing(nvidia, monkeypatch, family, dtype, memory_mode):
+ _cap(monkeypatch, (7, 5))
+ _image_gate(monkeypatch, _image_family(family), "fp8", dtype, memory_mode = memory_mode)
+
+
+def test_image_gate_kill_switch_refuses_fp8_on_turing(nvidia, monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.setenv(FP8_KILL_SWITCH, "0")
+ with pytest.raises(RuntimeError, match = "needs a CUDA GPU in bf16"):
+ _image_gate(monkeypatch, _image_family("flux.3-klein"), "fp8", torch.float32)
+
+
[email protected]("cap", [(7, 0), (8, 6)])
+def test_image_gate_on_other_fp16_cards_is_unchanged(nvidia, monkeypatch, cap):
+ _cap(monkeypatch, cap)
+ with pytest.raises(RuntimeError, match = "needs a CUDA GPU in bf16"):
+ _image_gate(monkeypatch, _image_family("flux.3-klein"), "fp8", torch.float16)
+
+
+def test_the_kill_switch_name_matches_the_module():
+ assert tq.FP16_CARD_FP8_ENV == FP8_KILL_SWITCH
diff --git a/studio/backend/tests/test_diffusion_fp16_card_int8.py b/studio/backend/tests/test_diffusion_fp16_card_int8.py
index 841cf08695b..e31fb0056bb 100644
--- a/studio/backend/tests/test_diffusion_fp16_card_int8.py
+++ b/studio/backend/tests/test_diffusion_fp16_card_int8.py
@@ -211,7 +211,7 @@ def test_image_gate_admits_int8_on_turing(nvidia, monkeypatch, family, dtype, me
_image_gate(monkeypatch, _image_family(family), "int8", dtype, memory_mode = memory_mode)
[email protected]("pinned", ["fp8", "mxfp8"])
[email protected]("pinned", ["mxfp8"])
def test_image_gate_still_refuses_other_schemes_on_turing(nvidia, monkeypatch, pinned):
_cap(monkeypatch, (7, 5))
with pytest.raises(RuntimeError, match = "transformer_quant"):