Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: auto runs compiled int8 W8A8 for Qwen-Image families on RDNA (#17013)
Daniel Han6db37aa046cc
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 93b92d91db5..3a1ea937789 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -522,6 +522,7 @@ from .diffusion_transformer_quant import (mark_source_precision,nvfp4_gated_base,native_int8_act,+ rdna_auto_w8a8_reason,native_offload_host,native_quant_host,native_quant_scheme,@@ -8600,9 +8601,33 @@ class DiffusionBackend:auto_bf16_when_resident_reason(getattr(fam, "name", None))== SMALL_DIT_BF16_WHEN_RESIDENT)+ # ROCm RDNA AUTO W8A8 (FamilyProfile.rdna_auto_w8a8); resident only, offload / low-VRAM keeps bf16.+ rdna_auto_w8a8 = (+ rdna_auto_w8a8_reason(target, getattr(fam, "name", None))+ if transformer_quant == TQ_AUTO+ and kind == "pipeline"+ and pipeline_seed_scheme is None+ and not _memory_request_forces_offload(memory_mode, cpu_offload)+ # Compiling tiers only; the fused W8A8 linear compiles regardless.+ and (+ speed_mode is None+ or normalize_speed_mode(speed_mode) in (SPEED_DEFAULT, SPEED_MAX)+ )+ and compile_eligible(target, is_gguf = False, family = fam, probe = False)+ else None+ )+ if rdna_auto_w8a8 is not None:+ logger.info(+ "diffusion.transformer_quant: auto picks native int8 W8A8 on this RDNA GPU (%s; "+ "UNSLOTH_NATIVE_INT8_ACT=0 keeps bf16)",+ rdna_auto_w8a8,+ )+ transformer_quant = TQ_INT8native_scheme = (native_quant_scheme(- target, transformer_quant_pinned, family = getattr(fam, "name", None)+ target,+ TQ_INT8 if rdna_auto_w8a8 is not None else transformer_quant_pinned,+ family = getattr(fam, "name", None),)if kind == "pipeline"else None@@ -10338,7 +10363,7 @@ class DiffusionBackend:{"offload": plan.offload_policy != OFFLOAD_NONE,"act_int8": native_scheme == TQ_INT8- and native_int8_act(target),+ and (native_int8_act(target) or rdna_auto_w8a8 is not None),}if native_scheme is not Noneelse {}@@ -10514,6 +10539,18 @@ class DiffusionBackend:"(quantized transformer must be compiled; eager is ~30x slower)")effective_speed = SPEED_DEFAULT+ if (+ rdna_auto_w8a8 is not None+ and transformer_quant_engaged is not None+ and speed_mode is None+ and effective_speed == SPEED_OFF+ and compile_eligible(target, is_gguf = False, family = fam, probe = False)+ ):+ logger.info(+ "diffusion.speed: speed_mode=default from render 1 (auto int8 W8A8 on RDNA is "+ "measured compiled)"+ )+ effective_speed = SPEED_DEFAULTif (small_dit_kept_bf16and speed_mode is Nonediff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.pyindex b0949c62659..4b879749416 100644--- a/studio/backend/core/inference/diffusion_families.py+++ b/studio/backend/core/inference/diffusion_families.py@@ -299,6 +299,12 @@ _QWENIMAGE_MOD_WEIGHT_ONLY = ("img_mod", "txt_mod")# pre-floor plain fp8 at LPIPS 0.712 with SSIM 0.016 and mean luma 0. Keeping the deny cost real speed: it pinned qwen# to int8 at 1.10x bf16 when compiled fp8 measures 1.21x._QWENIMAGE_SCHEME_DENY = frozenset({"mxfp8", "nvfp4"})+# rdna_auto_w8a8: gfx1151, ROCm 7.13, 1024^2, both arms compiled, LPIPS vs compiled bf16. Wan failed the 0.10 LPIPS bar.+_QWENIMAGE21_RDNA_AUTO_W8A8 = "compiled int8 W8A8 measured 83.3 vs 95.1 s/image against compiled bf16 on gfx1151 (25 steps), LPIPS 0.020"+_QWENIMAGE_RDNA_AUTO_W8A8 = (+ "compiled int8 W8A8 measured 133.1 vs 141.2 s/image against compiled bf16 on gfx1151 (20 steps CFG 4), "+ "LPIPS 0.014"+)# FLUX.3 Klein transformer-quant keep-list, every scheme (FamilyProfile.quant_exclude_tokens)._FLUX3_QUANT_KEEP = ("modulation",@@ -891,6 +897,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (te_int8_skip = _QWENIMAGE_TE_INT8_SKIP,preview_factors = "qwen_image",auto_static_skip = (("qwen/qwen-image", (("default", 2), ("max", 3), ("steps", 20))),),+ rdna_auto_w8a8 = _QWENIMAGE_RDNA_AUTO_W8A8,),filter_reduction_configs_archs = _QWEN_REDUCTION_RACE_ARCHS,comfy_flow_shift = 3.1, # ComfyUI ModelSamplingAuraFlow 3.1 (Qwen-Image templates)@@ -955,6 +962,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (# 0.0642 against 0.1032 with the generic tokens alone, paired over 16 calibration prompts, -0.0390 CI95# [-0.0708, -0.0072], better on 14/16.int8_exclude_tokens = ("txt_in",),+ rdna_auto_w8a8 = _QWENIMAGE21_RDNA_AUTO_W8A8,int8_convrot = ConvRotSpec(group = 256,suffixes = (diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.pyindex e5c792d6f07..3a0ecf79057 100644--- a/studio/backend/core/inference/diffusion_native_quant.py+++ b/studio/backend/core/inference/diffusion_native_quant.py@@ -320,6 +320,26 @@ def _rocm_fused_w8a8_refusal(torch: Any) -> Optional[str]:return f"ROCm {arch or 'unknown arch'}"+def rocm_rdna_w8a8_ready() -> bool:+ """ROCm RDNA device where the compiled W8A8 linear can run (UNSLOTH_NATIVE_INT8_FUSED not off)."""+ if _FUSED_STATE["failed"] is not None or int8_fused_disabled():+ return False+ try:+ import torch++ from .rocm_bf16 import is_rocm_torch++ if not is_rocm_torch(torch) or not torch.cuda.is_available():+ return False+ if _rocm_fused_w8a8_refusal(torch) is not None:+ return False+ from .diffusion_speed import torch_compile_runtime_available++ return bool(torch_compile_runtime_available())+ except Exception: # noqa: BLE001 - unknown device: keep the weight-only default+ return False++def _fused_w8a8() -> Optional[Callable[..., Any]]:"""Compiled ``_w8a8_rows``, or None if disabled, no inductor runtime, a ROCm arch other than RDNA3 / RDNA4, oralready failed."""diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 9c9a873f668..1da8410756f 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -17,7 +17,12 @@ from typing import Any, Callable, Optional# stdlib-only module (no torch), so this stays inside the "imported lazily" promise above.from core._torchao_stub import is_stubbed, torch_is_rocm-from .diffusion_native_quant import int8_act_disabled, int8_act_requested, is_native_linear+from .diffusion_native_quant import (+ int8_act_disabled,+ int8_act_requested,+ is_native_linear,+ rocm_rdna_w8a8_ready,+)from .diffusion_nvfp4_flag import (nvfp4_blocked,nvfp4_diffusion_enabled,@@ -1220,6 +1225,28 @@ def native_int8_act(target: Any) -> bool:return not int8_act_disabled()+def _rocm_cuda(target: Any) -> bool:+ return getattr(target, "device", None) == "cuda" and torch_is_rocm()+++_FAMILY_RDNA_AUTO_W8A8: dict[str, str] = _profile_table("rdna_auto_w8a8", quant = True)+++def rdna_auto_w8a8_reason(target: Any, family: Optional[str]) -> Optional[str]:+ """Measured reason AUTO picks W8A8 for ``family`` on ROCm RDNA, else None. Kill switch UNSLOTH_NATIVE_INT8_ACT=0."""+ reason = _FAMILY_RDNA_AUTO_W8A8.get(str(family or "").strip().lower())+ if reason is None or not _rocm_cuda(target) or not native_quant_host(target):+ return None+ # Linux only: Windows ROCm has no compile tier.+ if not _sys.platform.startswith("linux"):+ return None+ if int8_act_disabled() or not rocm_rdna_w8a8_ready():+ return None+ if native_quant_scheme(target, TQ_INT8, family = family) != TQ_INT8:+ return None+ return reason++def select_transformer_quant_scheme(target: Any,requested: Optional[str],diff --git a/studio/backend/core/inference/family_profile.py b/studio/backend/core/inference/family_profile.pyindex e4bb01aa369..a4ec5a6df9e 100644--- a/studio/backend/core/inference/family_profile.py+++ b/studio/backend/core/inference/family_profile.py@@ -80,6 +80,8 @@ class FamilyProfile:auto_scheme_skip: frozenset[str] = frozenset()# Reason auto keeps bf16 while bf16 fits (INT8 still picked once bf16 would offload); None = not this family.auto_bf16_when_resident: Optional[str] = None+ # ROCm RDNA AUTO W8A8 evidence (gfx1151); None = bf16. Kill switch UNSLOTH_NATIVE_INT8_ACT=0.+ rdna_auto_w8a8: Optional[str] = None# Video: prequant DiT schemes AUTO seeds even where bf16 fits resident (else auto keeps resident bf16).auto_prequant_when_resident: frozenset[str] = frozenset()# Text-encoder int8 keep-bf16 schedule (skip_first, skip_last) decoder blocks; None = int8 falls back to fp8.diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex d962beae5fa..48512d91c37 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -13277,6 +13277,129 @@ def test_auto_on_amd_stays_bf16(fake_runtime, tmp_path, monkeypatch):backend.unload()+def _stub_rdna_w8a8_host(backend, monkeypatch):+ """AMD host stub on an RDNA device; records the quantise kwargs."""+ from core.inference import diffusion as dmod+ from core.inference import diffusion_native_quant as nq+ from core.inference import diffusion_transformer_quant as tq++ calls = _stub_amd_weight_only_host(backend, monkeypatch)+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: True)+ monkeypatch.setattr(tq, "rocm_rdna_w8a8_ready", lambda: True)+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)+ monkeypatch.setattr(tq._sys, "platform", "linux")+ recorded = dmod.quantize_transformer++ def _quantize(+ pipe,+ target,+ *,+ mode,+ family = None,+ **kwargs,+ ):+ scheme = recorded(pipe, target, mode = mode, family = family, **kwargs)+ calls[-1]["act_int8"] = kwargs.get("act_int8")+ return scheme++ monkeypatch.setattr(dmod, "quantize_transformer", _quantize)+ return calls+++def test_auto_on_rdna_runs_compiled_w8a8_for_a_measured_family(fake_runtime, tmp_path, monkeypatch):+ backend = DiffusionBackend()+ calls = _stub_rdna_w8a8_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)+ )+ assert [(c["scheme"], c["act_int8"]) for c in calls] == [("int8", True)]+ assert status["transformer_quant"] == "int8"+ assert status["speed_mode"] == "default"+ backend.unload()+++def test_auto_on_rdna_kill_switch_keeps_bf16(fake_runtime, tmp_path, monkeypatch):+ from core.inference import diffusion_native_quant as nq++ backend = DiffusionBackend()+ calls = _stub_rdna_w8a8_host(backend, monkeypatch)+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, "0")+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)+ )+ assert calls == []+ assert status["transformer_quant"] is None+ backend.unload()+++def test_auto_on_rdna_keeps_bf16_when_rocm_compile_is_off(fake_runtime, tmp_path, monkeypatch):+ from core.inference import diffusion as dmod++ backend = DiffusionBackend()+ calls = _stub_rdna_w8a8_host(backend, monkeypatch)+ monkeypatch.setattr(dmod, "compile_eligible", lambda *a, **k: False)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)+ )+ assert calls == []+ assert status["transformer_quant"] is None+ backend.unload()+++def test_auto_on_rdna_with_the_eager_tier_stays_bf16(fake_runtime, tmp_path, monkeypatch):+ backend = DiffusionBackend()+ calls = _stub_rdna_w8a8_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ speed_mode = "eager",+ _base_local_dir = str(tmp_path),+ )+ assert calls == []+ assert status["transformer_quant"] is None+ backend.unload()+++def test_auto_on_rdna_keeps_bf16_for_an_unmeasured_family(fake_runtime, tmp_path, monkeypatch):+ backend = DiffusionBackend()+ calls = _stub_rdna_w8a8_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-Edit-2511", model_kind = "pipeline", _base_local_dir = str(tmp_path)+ )+ assert calls == []+ assert status["transformer_quant"] is None+ backend.unload()+++def test_auto_on_rdna_with_speed_off_stays_bf16(fake_runtime, tmp_path, monkeypatch):+ backend = DiffusionBackend()+ calls = _stub_rdna_w8a8_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ speed_mode = "off",+ _base_local_dir = str(tmp_path),+ )+ assert calls == []+ assert status["transformer_quant"] is None+ assert status["speed_mode"] == "off"+ backend.unload()+++def test_auto_on_rdna_honours_an_explicit_speed_tier(fake_runtime, tmp_path, monkeypatch):+ backend = DiffusionBackend()+ calls = _stub_rdna_w8a8_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ speed_mode = "max",+ _base_local_dir = str(tmp_path),+ )+ assert [(c["scheme"], c["act_int8"]) for c in calls] == [("int8", True)]+ assert status["speed_mode"] == "max"+ backend.unload()++def test_an_explicit_scheme_on_nvidia_keeps_the_torchao_path_and_its_wording(fake_runtime, tmp_path, monkeypatch):diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.pyindex 678fef585e9..9c886dd14c5 100644--- a/studio/backend/tests/test_diffusion_native_quant.py+++ b/studio/backend/tests/test_diffusion_native_quant.py@@ -185,6 +185,57 @@ def test_w8a8_default_differs_by_route(monkeypatch, env, expect_amd, expect_nvidassert tq.native_int8_act(_target()) is expect_nvidia+def test_explicit_int8_keeps_its_weight_only_default_on_rdna(rocm, monkeypatch):+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)+ monkeypatch.setattr(tq, "rocm_rdna_w8a8_ready", lambda: True)+ assert tq.native_int8_act(_target()) is False+++def test_rdna_auto_w8a8_reason_is_per_family_with_a_kill_switch(rocm, monkeypatch):+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)+ monkeypatch.setattr(tq._sys, "platform", "linux")+ monkeypatch.setattr(tq, "rocm_rdna_w8a8_ready", lambda: True)+ monkeypatch.setitem(tq._FAMILY_RDNA_AUTO_W8A8, "toy-family", "measured")+ assert tq.rdna_auto_w8a8_reason(_target(), "toy-family") == "measured"+ assert tq.rdna_auto_w8a8_reason(_target(), "Toy-Family ") == "measured"+ assert tq.rdna_auto_w8a8_reason(_target(), "other-family") is None+ assert tq.rdna_auto_w8a8_reason(_target(dtype = torch.float16), "toy-family") is None+ assert tq.rdna_auto_w8a8_reason(_target("cpu"), "toy-family") is None+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, "0")+ assert tq.rdna_auto_w8a8_reason(_target(), "toy-family") is None+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV)+ monkeypatch.setattr(tq, "rocm_rdna_w8a8_ready", lambda: False)+ assert tq.rdna_auto_w8a8_reason(_target(), "toy-family") is None+ monkeypatch.setattr(tq, "rocm_rdna_w8a8_ready", lambda: True)+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "toy-family", frozenset({tq.TQ_INT8}))+ assert tq.rdna_auto_w8a8_reason(_target(), "toy-family") is None+++def test_rdna_auto_w8a8_ships_only_for_the_measured_families():+ # Wan, small DiTs, edit / layered stay bf16.+ assert set(tq._FAMILY_RDNA_AUTO_W8A8) == {"qwen-image", "qwen-image-2.1"}+++def test_rdna_auto_w8a8_reason_is_linux_only(rocm, monkeypatch):+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)+ monkeypatch.setattr(tq, "rocm_rdna_w8a8_ready", lambda: True)+ monkeypatch.setitem(tq._FAMILY_RDNA_AUTO_W8A8, "toy-family", "measured")+ monkeypatch.setattr(tq._sys, "platform", "win32")+ assert tq.rdna_auto_w8a8_reason(_target(), "toy-family") is None+++def test_rdna_auto_w8a8_reason_is_rocm_only(nvidia, monkeypatch):+ monkeypatch.setattr(tq, "rocm_rdna_w8a8_ready", lambda: True)+ monkeypatch.setitem(tq._FAMILY_RDNA_AUTO_W8A8, "toy-family", "measured")+ assert tq.rdna_auto_w8a8_reason(_target(), "toy-family") is None+++def test_rocm_rdna_w8a8_ready_is_false_without_a_rocm_gpu(monkeypatch):+ monkeypatch.setattr(nq, "_FUSED_STATE", {"fn": None, "failed": None})+ monkeypatch.setenv(nq.NATIVE_INT8_FUSED_ENV, "0")+ assert nq.rocm_rdna_w8a8_ready() is False++def test_family_deny_list_still_applies(rocm, monkeypatch):monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "toy-family", frozenset({tq.TQ_INT8}))assert tq.native_quant_scheme(_target(), "int8", family = "toy-family") is None