Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run FP8 on T4 and other Turing GPUs as weight-only fp8 instead of refusing it (#17218)
Daniel Hanbf9039df56c0
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 1ee71a0e262..5721bb93cce 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -532,7 +532,8 @@ from .diffusion_transformer_quant import (explain_unusable_scheme,family_denies_scheme,fp16_card_int8_host,- fp16_card_int8_scheme,+ fp16_card_native_host,+ fp16_card_native_scheme,fp16_card_quant_storage_enabled,mark_source_precision,nvfp4_gated_base,@@ -4520,8 +4521,11 @@ class DiffusionBackend:):if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)- elif model_kind == "pipeline" and pinned == TQ_INT8 and fp16_card_int8_host(target):- if fp16_card_int8_scheme(target, pinned, family = getattr(fam, "name", None)) is None:+ elif model_kind == "pipeline" and fp16_card_native_host(target, pinned):+ if (+ fp16_card_native_scheme(target, pinned, family = getattr(fam, "name", None))+ is None+ ):reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)elif (model_kind == "pipeline"@@ -8662,7 +8666,7 @@ class DiffusionBackend:TQ_INT8 if rdna_auto_w8a8 is not None else transformer_quant_pinned,family = getattr(fam, "name", None),)- or fp16_card_int8_scheme(+ or fp16_card_native_scheme(target, transformer_quant_pinned, family = getattr(fam, "name", None))if kind == "pipeline"diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 4a158cc5bf1..6b661bbc6f4 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -1269,6 +1269,51 @@ def fp16_card_int8_scheme(return TQ_INT8+FP16_CARD_FP8_ENV = "UNSLOTH_DIFFUSION_FP16_CARD_FP8"+++def fp16_card_fp8_enabled() -> bool:+ raw = (_os.environ.get(FP16_CARD_FP8_ENV) or "").strip().lower()+ return raw not in ("0", "false", "no", "off")+++def fp16_card_fp8_scheme(+ target: Any,+ requested: Optional[str],+ family: Optional[str] = None,+) -> Optional[str]:+ """``fp8`` for an explicit fp8 request on an ``fp16_card_int8_host``, else None. Weight-only (no fp8 matmul+ below sm_89); ``UNSLOTH_DIFFUSION_FP16_CARD_FP8=0`` restores the refusal."""+ if normalize_transformer_quant(requested) != TQ_FP8 or not fp16_card_fp8_enabled():+ return None+ if not fp16_card_int8_host(target) or _family_denied(family, TQ_FP8):+ return None+ try:+ import torch+ if not hasattr(torch, "float8_e4m3fn"):+ return None+ except Exception: # noqa: BLE001+ return None+ return TQ_FP8+++def fp16_card_native_host(target: Any, requested: Optional[str]) -> bool:+ scheme = normalize_transformer_quant(requested)+ if scheme == TQ_FP8 and not fp16_card_fp8_enabled():+ return False+ return scheme in (TQ_INT8, TQ_FP8) and fp16_card_int8_host(target)+++def fp16_card_native_scheme(+ target: Any,+ requested: Optional[str],+ family: Optional[str] = None,+) -> Optional[str]:+ return fp16_card_int8_scheme(target, requested, family) or fp16_card_fp8_scheme(+ target, requested, family+ )++def fp16_card_int8_act(target: Any) -> bool:"""W8A8 for the explicit-int8 Turing build; ``int8_storage_host`` (also true there) answers only for checkpointint8 storage, which stays weight-only."""@@ -2172,11 +2217,12 @@ def quantize_transformer() -> Optional[str]:"""Quantise ``pipe.transformer`` in place, returning the scheme or None (GGUF); ``base_repo`` picks the NVFP4 policy."""native = native_quant_scheme(target, mode, family = family, offload = offload)- fp16_card = native is None and fp16_card_int8_scheme(target, mode, family) is not None+ fp16_card_scheme = fp16_card_native_scheme(target, mode, family) if native is None else None+ fp16_card = fp16_card_scheme is not Noneif fp16_card:- native = TQ_INT8- # native_int8_act is False on any fp16 CUDA target (int8_storage_host); this build is the W8A8 one.- act_int8 = fp16_card_int8_act(target)+ native = fp16_card_scheme+ # native_int8_act is False on fp16 CUDA targets; int8 here is W8A8, fp8 weight-only.+ act_int8 = fp16_card_int8_act(target) if native == TQ_INT8 else Falseif native is not None:mps = _mps_bf16(target)if act_int8 is None or mps:diff --git a/studio/backend/tests/test_diffusion_fp16_card_fp8.py b/studio/backend/tests/test_diffusion_fp16_card_fp8.pynew file mode 100644index 00000000000..57de8439345--- /dev/null+++ b/studio/backend/tests/test_diffusion_fp16_card_fp8.py@@ -0,0 +1,209 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Weight-only fp8 on fp16-only NVIDIA cards (Turing); capability is monkeypatched, no GPU needed."""++from __future__ import annotations++import types++import pytest++torch = pytest.importorskip("torch")+if not hasattr(torch, "float8_e4m3fn"):+ pytest.skip("torch without float8 dtypes", allow_module_level = True)++import core.inference.diffusion_native_quant as nq # noqa: E402+import core.inference.diffusion_transformer_quant as tq # noqa: E402++# Literals, not tq.*_ENV: the behavioural tests must also run against a tree without the names.+FP8_KILL_SWITCH = "UNSLOTH_DIFFUSION_FP16_CARD_FP8"+INT8_KILL_SWITCH = "UNSLOTH_DIFFUSION_FP16_CARD_INT8"+++def _target(dtype, device = "cuda"):+ return types.SimpleNamespace(device = device, dtype = dtype)+++def nvidia(monkeypatch):+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)+ monkeypatch.delenv(FP8_KILL_SWITCH, raising = False)+ monkeypatch.delenv(INT8_KILL_SWITCH, raising = False)+++def _cap(monkeypatch, cap):+ monkeypatch.setattr(tq, "_capability", lambda ordinal = None: cap)+++class _Toy(torch.nn.Module):+ def __init__(self, dim = 128):+ super().__init__()+ self.to_q = torch.nn.Linear(dim, dim)+ self.ff = torch.nn.Linear(dim, 2 * dim)+ self.out = torch.nn.Linear(2 * dim, dim)++ def forward(self, x):+ return self.out(torch.nn.functional.gelu(self.ff(self.to_q(x))))++[email protected]("dtype", [torch.float16, torch.float32])+def test_turing_fp16_and_fp32_pipelines_take_fp8(nvidia, monkeypatch, dtype):+ _cap(monkeypatch, (7, 5))+ target = _target(dtype)+ assert tq.fp16_card_fp8_scheme(target, "fp8") == "fp8"+ assert tq.fp16_card_native_scheme(target, " FP8 ") == "fp8"+ assert tq.fp16_card_native_scheme(target, "int8") == "int8"+ assert tq.fp16_card_native_host(target, "fp8") and tq.fp16_card_native_host(target, "int8")++[email protected]("scheme", [None, "auto", "none", "int8", "mxfp8"])+def test_only_an_explicit_fp8_request_is_taken(nvidia, monkeypatch, scheme):+ _cap(monkeypatch, (7, 5))+ assert tq.fp16_card_fp8_scheme(_target(torch.float16), scheme) is None+ if scheme not in ("int8",):+ assert tq.fp16_card_native_scheme(_target(torch.float16), scheme) is None+++ "cap", [None, (6, 1), (7, 0), (8, 0), (8, 6), (8, 9), (9, 0), (10, 0), (12, 0)]+)[email protected]("dtype", [torch.float16, torch.float32, torch.bfloat16])+def test_every_other_nvidia_card_is_unchanged(nvidia, monkeypatch, cap, dtype):+ _cap(monkeypatch, cap)+ assert tq.fp16_card_fp8_scheme(_target(dtype), "fp8") is None+ assert not tq.fp16_card_native_host(_target(dtype), "fp8")++[email protected]("device", ["cpu", "mps", "xpu"])+def test_bf16_turing_and_non_cuda_are_unchanged(nvidia, monkeypatch, device):+ _cap(monkeypatch, (7, 5))+ assert tq.fp16_card_fp8_scheme(_target(torch.bfloat16), "fp8") is None+ assert tq.fp16_card_fp8_scheme(_target(torch.float16, device), "fp8") is None++[email protected]("raw", ["0", "false", "off", "no"])+def test_kill_switch_restores_the_refusal(nvidia, monkeypatch, raw):+ _cap(monkeypatch, (7, 5))+ monkeypatch.setenv(FP8_KILL_SWITCH, raw)+ assert tq.fp16_card_fp8_scheme(_target(torch.float32), "fp8") is None+ assert not tq.fp16_card_native_host(_target(torch.float32), "fp8")+ assert tq.fp16_card_native_scheme(_target(torch.float32), "int8") == "int8"+++def test_the_int8_kill_switch_also_closes_fp8(nvidia, monkeypatch):+ _cap(monkeypatch, (7, 5))+ monkeypatch.setenv(INT8_KILL_SWITCH, "0")+ assert tq.fp16_card_fp8_scheme(_target(torch.float32), "fp8") is None+++def test_the_family_deny_list_still_applies(nvidia, monkeypatch):+ _cap(monkeypatch, (7, 5))+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "toy-family", ("fp8",))+ assert tq.fp16_card_fp8_scheme(_target(torch.float16), "fp8", family = "toy-family") is None+++def test_the_shared_native_route_is_not_widened(nvidia, monkeypatch):+ _cap(monkeypatch, (7, 5))+ assert tq.native_quant_scheme(_target(torch.float16), "fp8") is None+ assert tq.select_transformer_quant_scheme(_target(torch.float16), "fp8") is None++[email protected]("dtype", [torch.float16, torch.float32])+def test_quantize_transformer_builds_weight_only_fp8_in_the_pipeline_dtype(+ nvidia, monkeypatch, dtype+):+ _cap(monkeypatch, (7, 5))+ torch.manual_seed(0)+ model = _Toy().to(dtype)+ x = torch.randn(4, 128, dtype = dtype)+ with torch.no_grad():+ ref = model(x).float()+ got = tq.quantize_transformer(+ types.SimpleNamespace(transformer = model), _target(dtype), mode = "fp8", min_features = 64+ )+ assert got == "fp8"+ for name in ("to_q", "ff", "out"):+ layer = getattr(model, name)+ assert nq.is_native_linear(layer), name+ assert layer.compute_dtype == dtype+ assert layer.scheme == "fp8" and not layer.act_int8+ assert layer.weight_q.element_size() == 1+ with torch.no_grad():+ y = model(x)+ assert y.dtype == dtype and torch.isfinite(y).all()+ rel = ((y.float() - ref).norm() / ref.norm()).item()+ assert rel < 0.05, rel+++def test_quantize_transformer_on_ampere_fp16_still_declines(nvidia, monkeypatch):+ _cap(monkeypatch, (8, 6))+ model = _Toy().to(torch.float16)+ got = tq.quantize_transformer(+ types.SimpleNamespace(transformer = model),+ _target(torch.float16),+ mode = "fp8",+ min_features = 64,+ )+ assert got is None+ assert not nq.is_native_linear(model.to_q)+++def _image_family(name):+ from core.inference.diffusion_families import _FAMILIES+ return next(f for f in _FAMILIES if f.name == name)+++def _image_gate(+ monkeypatch,+ fam,+ pinned,+ dtype,+ *,+ memory_mode = None,+):+ import core.inference.diffusion as d+ monkeypatch.setattr(d, "effective_te_quant", lambda *a, **k: None)+ d.DiffusionBackend._assert_precision_for_target(+ None,+ fam,+ _target(dtype),+ model_kind = "pipeline",+ pinned = pinned,+ te_mode = None,+ memory_mode = memory_mode,+ cpu_offload = False,+ )+++ "family,dtype",+ [+ ("qwen-image-2.1", torch.float16),+ ("flux.3-klein", torch.float32),+ ("flux.3-klein", torch.float16),+ ],+)[email protected]("memory_mode", [None, "low_vram"])+def test_image_gate_admits_fp8_on_turing(nvidia, monkeypatch, family, dtype, memory_mode):+ _cap(monkeypatch, (7, 5))+ _image_gate(monkeypatch, _image_family(family), "fp8", dtype, memory_mode = memory_mode)+++def test_image_gate_kill_switch_refuses_fp8_on_turing(nvidia, monkeypatch):+ _cap(monkeypatch, (7, 5))+ monkeypatch.setenv(FP8_KILL_SWITCH, "0")+ with pytest.raises(RuntimeError, match = "needs a CUDA GPU in bf16"):+ _image_gate(monkeypatch, _image_family("flux.3-klein"), "fp8", torch.float32)++[email protected]("cap", [(7, 0), (8, 6)])+def test_image_gate_on_other_fp16_cards_is_unchanged(nvidia, monkeypatch, cap):+ _cap(monkeypatch, cap)+ with pytest.raises(RuntimeError, match = "needs a CUDA GPU in bf16"):+ _image_gate(monkeypatch, _image_family("flux.3-klein"), "fp8", torch.float16)+++def test_the_kill_switch_name_matches_the_module():+ assert tq.FP16_CARD_FP8_ENV == FP8_KILL_SWITCHdiff --git a/studio/backend/tests/test_diffusion_fp16_card_int8.py b/studio/backend/tests/test_diffusion_fp16_card_int8.pyindex 841cf08695b..e31fb0056bb 100644--- a/studio/backend/tests/test_diffusion_fp16_card_int8.py+++ b/studio/backend/tests/test_diffusion_fp16_card_int8.py@@ -211,7 +211,7 @@ def test_image_gate_admits_int8_on_turing(nvidia, monkeypatch, family, dtype, me_image_gate(monkeypatch, _image_family(family), "int8", dtype, memory_mode = memory_mode)[email protected]("pinned", ["fp8", "mxfp8"])[email protected]("pinned", ["mxfp8"])def test_image_gate_still_refuses_other_schemes_on_turing(nvidia, monkeypatch, pinned):_cap(monkeypatch, (7, 5))with pytest.raises(RuntimeError, match = "transformer_quant"):