Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio / Desktop: allow text-encoder nvfp4 on pre-Blackwell GPUs and explain TE precision refusals (#11413)

Maheswar Kumar443dfdabddce
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 622af0b4b7b..64234db3461 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -208,6 +208,7 @@ from .diffusion_precision import (
resolve_te_quant_request,
te_quant_needs_resident_weights,
te_quant_supported,
+ te_quant_unsupported_reason,
torchao_quantize_importable,
)
from .diffusion_te_prequant import te_prequant_pipe_kwargs
@@ -2128,10 +2129,7 @@ class DiffusionBackend:
"quantisations"
)
elif te_effective is not None and not te_quant_supported(target, te_effective):
- te_reason = (
- "this device does not have the tensor cores that backend needs (a CUDA GPU in "
- "bf16, plus fp8 / int8 / NVFP4 support depending on the mode)"
- )
+ te_reason = te_quant_unsupported_reason(te_effective)
elif te_quant_needs_resident_weights(te_effective) and _memory_request_forces_offload(
memory_mode, cpu_offload
):
diff --git a/studio/backend/core/inference/diffusion_precision.py b/studio/backend/core/inference/diffusion_precision.py
index 2c5f9094418..876cdfb2609 100644
--- a/studio/backend/core/inference/diffusion_precision.py
+++ b/studio/backend/core/inference/diffusion_precision.py
@@ -15,8 +15,9 @@ This shrinks it in place, with four backends:
with per-layer keep-bf16 selection. Degrades on large encoders unless the
sensitive decoder blocks stay bf16, so applied only for families with a
measured schedule (else falls back to fp8). ~2x smaller; cc >= 8.0.
- nvfp4 - torchao NVFP4 weight-only: 4-bit float, two-level microscaling, Blackwell
- sm_100+ FP4 cores. ~4x smaller (lowest VRAM) but a steeper quality cost.
+ nvfp4 - torchao NVFP4 weight-only: 4-bit float storage, two-level microscaling, each
+ weight dequantised to bf16 per forward (no FP4 GEMM). ~3.5x smaller (lowest
+ VRAM) but a steeper quality cost and a slower prompt encode; cc >= 8.0.
All keep norms / embeddings full precision, are a memory-vs-quality tradeoff (off by default),
and pair well with streamed (group) offload where the text encoder stays resident. Quantify
@@ -164,10 +165,38 @@ def torchao_quantize_importable() -> bool:
return not is_stubbed("torchao")
+_TE_QUANT_REQUIREMENTS = {
+ TE_QUANT_FP8: "an NVIDIA or AMD GPU that runs bf16 (NVIDIA Ampere sm_80 or newer)",
+ TE_QUANT_FP8_DYNAMIC: "an NVIDIA GPU with fp8 tensor cores (Ada sm_89 or newer)",
+ TE_QUANT_INT8: "an NVIDIA GPU that runs bf16 (Ampere sm_80 or newer)",
+ TE_QUANT_NVFP4: (
+ "an NVIDIA GPU that runs bf16 (Ampere sm_80 or newer) and torchao 0.15 or newer on "
+ "torch 2.8 or newer"
+ ),
+}
+
+
+def te_quant_unsupported_reason(mode: str) -> str:
+ """Why ``te_quant_supported`` declined ``mode``, naming what that mode needs."""
+ return f"text-encoder '{mode}' needs {_TE_QUANT_REQUIREMENTS[mode]}, which this host does not provide"
+
+
+@lru_cache(maxsize = 1)
+def nvfp4_weight_only_importable() -> bool:
+ """Whether torchao ships a usable ``NVFP4WeightOnlyConfig`` (0.15+; Studio pins 0.14 on torch <= 2.9)."""
+ try:
+ from torchao.prototype.mx_formats import NVFP4WeightOnlyConfig
+ from .diffusion_transformer_quant import _quiet_config
+ _quiet_config(NVFP4WeightOnlyConfig)
+ except Exception: # noqa: BLE001
+ return False
+ return True
+
+
def te_quant_supported(target: Any, mode: str) -> bool:
- """Whether ``mode`` is usable for ``target``: a CUDA bf16 device plus the tensor-core class
- each backend needs -- fp8 dtype (fp8), fp8 GEMM sm_89+ (fp8_dynamic), int8 sm_80+ (int8),
- Blackwell sm_100+ (nvfp4)."""
+ """Whether ``mode`` is usable for ``target``: a CUDA bf16 device plus what each backend
+ needs -- fp8 dtype (fp8, nvfp4 block scales), fp8 GEMM sm_89+ (fp8_dynamic), int8 sm_80+
+ (int8). nvfp4 is weight-only, so it has no FP4 tensor-core requirement."""
if getattr(target, "device", None) != "cuda":
return False
if nvfp4_blocked(mode):
@@ -189,7 +218,8 @@ def te_quant_supported(target: Any, mode: str) -> bool:
if mode == TE_QUANT_INT8:
return torch.cuda.get_device_capability()[0] >= 8 # int8 cores: Ampere sm_80+
if mode == TE_QUANT_NVFP4:
- return torch.cuda.get_device_capability()[0] >= 10 # NVFP4 cores: Blackwell sm_100+
+ # weight-only: dequantised to bf16 for a plain gemm, so only the e4m3 block-scale dtype is needed
+ return hasattr(torch, "float8_e4m3fn") and nvfp4_weight_only_importable()
except Exception:
return False
return False
@@ -263,8 +293,7 @@ def quantize_text_encoders(
_note(logger, f"text-encoder '{mode}' is not supported on this device; left dense")
return TEQuantOutcome(
None,
- f"this device cannot run text-encoder '{mode}' (it needs a CUDA GPU in bf16 with the "
- "tensor cores that backend requires), so the dense bf16 encoder was kept",
+ f"{te_quant_unsupported_reason(mode)}, so the dense bf16 encoder was kept",
RESOLVED_UNSUPPORTED,
)
if mode == TE_QUANT_INT8:
@@ -494,8 +523,7 @@ def _has_layerwise_hooks(encoder: Any) -> bool:
def _cast_nvfp4(encoder: Any, target: Any) -> None:
- # Weight-only NVFP4: linear weights become 4-bit NVFP4 on Blackwell FP4 cores, norms / embeddings untouched. Same
- # exclusions as the int8/fp8 TE modes; require_bf16 skips non-bf16 Linears so the cast engages instead of aborting.
+ # weight-only nvfp4: weights stored 4-bit, dequantised to bf16 per forward; same exclusions as the int8/fp8 modes
from torchao.quantization import quantize_
from torchao.prototype.mx_formats import NVFP4WeightOnlyConfig
from .diffusion_transformer_quant import (
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 13783a588d4..7d55b206456 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -155,6 +155,7 @@ from .diffusion_precision import (
quantize_text_encoders,
te_quant_needs_resident_weights,
te_quant_supported,
+ te_quant_unsupported_reason,
torchao_quantize_importable,
)
from .video_families import (
@@ -545,10 +546,7 @@ def _assert_video_precision_for_target(
"quantisations"
)
elif te_effective is not None and not te_quant_supported(target, te_effective):
- te_reason = (
- "this device does not have the tensor cores that backend needs (a CUDA GPU in "
- "bf16, plus fp8 / int8 / NVFP4 support depending on the mode)"
- )
+ te_reason = te_quant_unsupported_reason(te_effective)
elif te_quant_needs_resident_weights(te_effective) and forces_offload:
# Same fence on the encoder: the loader reports those modes unsupported once offload is active, and by then the
# resident model is gone. Layerwise fp8 is a dtype cast.
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 5bbe063c51c..fa2748757ab 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -3972,7 +3972,8 @@ class DiffusionLoadRequest(BaseModel):
"~2x smaller, "
"CUDA cc>=8.9), fp8_dynamic (torchao compute fp8 on the tensor cores, ~2x + faster, "
"cc>=8.9), int8 (torchao compute int8 with per-family keep-bf16 layers; falls back to "
- "fp8 where no schedule exists; cc>=8.0), or nvfp4 (~4x smaller, Blackwell sm_100+). A "
+ "fp8 where no schedule exists; cc>=8.0), or nvfp4 (torchao weight-only, ~3.5x smaller, "
+ "dequantised to bf16 per forward so the prompt encode is slower; cc>=8.0). A "
"memory-vs-quality tradeoff (shifts fine detail), not free; pairs well with balanced mode. "
"Fails CLOSED when NOTHING could be cast (409, or a load-progress error); an int8 request "
"downgraded to fp8 loads and is reported through the status resolved record instead.",
@@ -5093,9 +5094,10 @@ class VideoLoadRequest(BaseModel):
"the largest resident component. fp8 = diffusers layerwise casting (memory only, cc >= "
"8.9); fp8_dynamic = torchao per-row fp8 COMPUTE on the tensor cores (cc >= 8.9); int8 = "
"torchao int8 COMPUTE with per-family keep-bf16 selection (cc >= 8.0; falls back to fp8 "
- "for a family without a measured schedule); nvfp4 = torchao 4-bit weight-only (Blackwell "
- "sm_100+). null/auto leaves the choice to the backend, which keeps the encoder dense on "
- "every family except MiniMax-H3, where it takes the hosted quantized conditioner; "
+ "for a family without a measured schedule); nvfp4 = torchao 4-bit weight-only, "
+ "dequantised to bf16 per forward (cc >= 8.0). null/auto leaves the choice to the backend, "
+ "which keeps the encoder dense on every family except MiniMax-H3, where it takes the "
+ "hosted quantized conditioner; "
"none/off always keeps the released bf16 encoder, which on MiniMax-H3 is the only way "
"to ask for it. Mirrors the image backend's field.",
)
diff --git a/studio/backend/tests/test_dense_quant_rocm_gate_9396.py b/studio/backend/tests/test_dense_quant_rocm_gate_9396.py
index acbd0869c5d..3b4f66942f3 100644
--- a/studio/backend/tests/test_dense_quant_rocm_gate_9396.py
+++ b/studio/backend/tests/test_dense_quant_rocm_gate_9396.py
@@ -134,6 +134,7 @@ def test_layerwise_fp8_text_encoder_still_supported_on_rocm(monkeypatch):
def test_torchao_text_encoder_modes_still_supported_on_cuda(monkeypatch, mode):
_stub_torch(monkeypatch, hip = None, version_str = "2.10.0+cu128", cc = (10, 0))
monkeypatch.setattr(dp, "is_stubbed", lambda name: False)
+ monkeypatch.setattr(dp, "nvfp4_weight_only_importable", lambda: True)
assert dp.te_quant_supported(_target(), mode) is True
diff --git a/studio/backend/tests/test_diffusion_precision.py b/studio/backend/tests/test_diffusion_precision.py
index d40f3545fda..c9bd45e4f19 100644
--- a/studio/backend/tests/test_diffusion_precision.py
+++ b/studio/backend/tests/test_diffusion_precision.py
@@ -20,6 +20,7 @@ from core.inference.diffusion_precision import (
TE_QUANT_FP8,
TE_QUANT_FP8_DYNAMIC,
TE_QUANT_INT8,
+ TE_QUANT_MODES,
TE_QUANT_NVFP4,
_cast_int8_selective,
_cast_nvfp4,
@@ -30,6 +31,7 @@ from core.inference.diffusion_precision import (
te_quant_is_auto,
quantize_text_encoders,
te_quant_supported,
+ te_quant_unsupported_reason,
)
@@ -47,7 +49,9 @@ def _stub_torch(
*,
with_fp8 = True,
cc = (10, 0),
+ nvfp4_config = True,
):
+ monkeypatch.setattr(dp, "nvfp4_weight_only_importable", lambda: nvfp4_config)
torch = types.ModuleType("torch")
torch.bfloat16 = "bfloat16"
torch.float16 = "float16"
@@ -114,14 +118,28 @@ def test_fp8_supported_requires_cuda_bf16_and_fp8(monkeypatch):
assert te_quant_supported(_target(dtype = "float16"), TE_QUANT_FP8) is False
-def test_nvfp4_supported_requires_blackwell(monkeypatch):
- _stub_torch(monkeypatch, cc = (10, 0))
- assert te_quant_supported(_target(), TE_QUANT_NVFP4) is True
- # Hopper (cc 9.0) has no NVFP4 tensor cores.
- _stub_torch(monkeypatch, cc = (9, 0))
[email protected]("cc", [(8, 0), (8, 6), (8, 9), (9, 0), (10, 0), (12, 0)])
+def test_nvfp4_supported_without_fp4_cores(monkeypatch, cc):
+ _stub_torch(monkeypatch, cc = cc)
+ assert te_quant_supported(_target(cc = cc), TE_QUANT_NVFP4) is True
+
+
+def test_nvfp4_supported_requires_cuda_bf16_and_fp8_dtype(monkeypatch):
+ _stub_torch(monkeypatch, cc = (8, 6))
+ assert te_quant_supported(_target(device = "cpu"), TE_QUANT_NVFP4) is False
+ # pre-ampere resolves float16, which torchao's nvfp4 quantiser rejects
+ assert te_quant_supported(_target(dtype = "float16"), TE_QUANT_NVFP4) is False
+ _stub_torch(monkeypatch, with_fp8 = False, cc = (8, 6))
assert te_quant_supported(_target(), TE_QUANT_NVFP4) is False
+def test_nvfp4_unsupported_when_torchao_lacks_the_weight_only_config(monkeypatch):
+ # studio pins torchao 0.14 for torch 2.9 and older, which has no NVFP4WeightOnlyConfig
+ _stub_torch(monkeypatch, cc = (8, 6), nvfp4_config = False)
+ assert te_quant_supported(_target(cc = (8, 6)), TE_QUANT_NVFP4) is False
+ assert te_quant_supported(_target(cc = (8, 6)), TE_QUANT_INT8) is True
+
+
def test_int8_supported_requires_sm80(monkeypatch):
# int8 tensor cores (torch._int_mm) need Ampere sm_80+.
_stub_torch(monkeypatch, cc = (8, 0))
@@ -180,18 +198,54 @@ def test_quantize_nvfp4_uses_torchao(monkeypatch):
assert recorder == [("nvfp4", te)]
-def test_quantize_nvfp4_unsupported_on_hopper_is_noop(monkeypatch):
- _stub_torch(monkeypatch, cc = (9, 0))
+def test_quantize_nvfp4_engages_on_ampere(monkeypatch):
+ _stub_torch(monkeypatch, cc = (8, 6))
+ recorder: list = []
+ _stub_casters(monkeypatch, recorder)
+ te = object()
+ pipe = types.SimpleNamespace(text_encoder = te)
+ outcome = quantize_text_encoders(pipe, _target(cc = (8, 6)), mode = "nvfp4")
+ assert outcome.mode == TE_QUANT_NVFP4 and outcome.status == "applied"
+ assert recorder == [("nvfp4", te)]
+
+
+def test_quantize_nvfp4_unsupported_on_float16_is_noop(monkeypatch):
+ _stub_torch(monkeypatch, cc = (7, 5))
recorder: list = []
_stub_casters(monkeypatch, recorder)
pipe = types.SimpleNamespace(text_encoder = object())
- outcome = quantize_text_encoders(pipe, _target(cc = (9, 0)), mode = "nvfp4")
+ outcome = quantize_text_encoders(pipe, _target(dtype = "float16", cc = (7, 5)), mode = "nvfp4")
assert outcome.mode is None
- # An unsupported request is now REPORTED rather than silently skipped.
- assert outcome.status == "unsupported" and "nvfp4" in outcome.reason
+ assert outcome.status == "unsupported"
+ assert "'nvfp4' needs an NVIDIA GPU that runs bf16" in outcome.reason
assert recorder == []
+def test_nvfp4_weight_only_probe_reads_the_installed_torchao(monkeypatch):
+ def _rejects_this_torch():
+ raise RuntimeError("requires PyTorch 2.8 or later")
+
+ probe = dp.nvfp4_weight_only_importable.__wrapped__
+ tq = types.ModuleType("core.inference.diffusion_transformer_quant")
+ tq._quiet_config = lambda config_cls, **kwargs: config_cls(**kwargs)
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_transformer_quant", tq)
+ mx = types.ModuleType("torchao.prototype.mx_formats")
+ monkeypatch.setitem(sys.modules, "torchao", types.ModuleType("torchao"))
+ monkeypatch.setitem(sys.modules, "torchao.prototype", types.ModuleType("torchao.prototype"))
+ monkeypatch.setitem(sys.modules, "torchao.prototype.mx_formats", mx)
+ assert probe() is False
+ mx.NVFP4WeightOnlyConfig = _rejects_this_torch
+ assert probe() is False
+ mx.NVFP4WeightOnlyConfig = lambda: "nvfp4cfg"
+ assert probe() is True
+
+
[email protected]("mode", TE_QUANT_MODES)
+def test_every_mode_has_an_unsupported_reason(mode):
+ reason = te_quant_unsupported_reason(mode)
+ assert f"'{mode}' needs " in reason and "Blackwell" not in reason
+
+
def test_quantize_tolerates_caster_failure(monkeypatch):
_stub_torch(monkeypatch)
hooks = types.ModuleType("diffusers.hooks")
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index 74e7fa742e8..67a1d30f67e 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -4505,7 +4505,7 @@ export function ImagesPage({
)}
<AdvancedSelect
label="Text encoder precision"
- hint="Lower precision reduces text-encoder memory but can change image quality. Supported modes depend on the GPU and model. Default lets the model choose, which on Qwen-Image 2.1 means its hosted FP8 encoder (8.75 GB rather than 16.3); pick Dense (bf16) to pin the released encoder. The loaded build below reports what was applied."
+ hint={`Shrinks the text encoder to save memory, at some cost to image quality. Default lets the model choose, which on Qwen-Image 2.1 means its hosted FP8 encoder (8.75 GB rather than 16.3); pick Dense (bf16) to pin the released encoder. FP8 (storage) is the safe pick and the only one that works with CPU offload. FP8 (compute) needs an RTX 40 series or newer.${nvfp4Diffusion ? " NVFP4 is the smallest." : ""} The loaded build below reports what was applied.`}
badge={<ResolvedBadge status={status} controlKey="text_encoder_quant" />}
value={textEncoderQuant}
onValueChange={(v) => setTextEncoderQuant(v as typeof textEncoderQuant)}
@@ -4517,7 +4517,7 @@ export function ImagesPage({
["fp8", "FP8 (storage)"],
["fp8_dynamic", "FP8 (compute)"],
["int8", "INT8"],
- ["nvfp4", "NVFP4 (Blackwell)"],
+ ["nvfp4", "NVFP4"],
] as [string, string][],
nvfp4Diffusion,
)}
diff --git a/tests/studio/playwright_image_text_encoder.py b/tests/studio/playwright_image_text_encoder.py
index 748cb07283c..3a4e25151f5 100644
--- a/tests/studio/playwright_image_text_encoder.py
+++ b/tests/studio/playwright_image_text_encoder.py
@@ -230,7 +230,7 @@ def main():
for label, requested, displayed in [
("FP8 (compute)", "fp8_dynamic", "FP8 (compute)"),
- ("NVFP4 (Blackwell)", "nvfp4", "NVFP4 (Blackwell)"),
+ ("NVFP4", "nvfp4", "NVFP4"),
("INT8", "int8", "FP8 (storage)"),
("Default", None, "Default"),
]: