Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: Ada FP8 unit-scale dispatcher installs only where it engages, chunked eager scaling (#14665)
Daniel Han487ee600f369
diff --git a/studio/backend/core/inference/diffusion_fp8_unit_scale.py b/studio/backend/core/inference/diffusion_fp8_unit_scale.pyindex 4c1381f9c9e..bda64dbdf78 100644--- a/studio/backend/core/inference/diffusion_fp8_unit_scale.py+++ b/studio/backend/core/inference/diffusion_fp8_unit_scale.py@@ -2,7 +2,8 @@# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0"""sm_89 precise-accumulate fp8: unit-scale _scaled_mm beats torchao row-wise (fast accumulate stays-on torchao, no gain); UNSLOTH_DIFFUSION_FP8_UNIT_SCALE=0 off."""+on torchao, no gain); UNSLOTH_DIFFUSION_FP8_UNIT_SCALE=0 off. Installed only where it can+engage: installing tags the process-wide compile cache key."""from __future__ import annotations@@ -13,6 +14,10 @@ UNIT_SCALE_ENV = "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE"MIN_ROWS_ENV = "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE_MIN_ROWS"# Unit-scale wins from M = 256 up (L4: row-wise 2.5-4x slower from 512); below, tens of microseconds either way._DEFAULT_MIN_ROWS = 256+# Eager epilogue chunk size, avoids a full fp32 [M, N] buffer.+_EAGER_EPILOGUE_ELEMS = 1 << 24+_OFF_WORDS = ("0", "off", "false", "no", "none")+_ON_WORDS = ("1", "on", "true", "yes", "force")_ADA_CAPS = ((8, 9),)_COUNTS = {"unit_scale": 0, "rowwise": 0}@@ -21,9 +26,9 @@ _STATE: dict = {"installed": False, "orig": None, "caps": {}}def _mode() -> str:v = (os.environ.get(UNIT_SCALE_ENV) or "").strip().lower()- if v in ("0", "off", "false", "no"):+ if v in _OFF_WORDS:return "off"- if v in ("1", "on", "true", "yes", "force"):+ if v in _ON_WORDS:return "force"return "auto"@@ -44,6 +49,41 @@ def _compiling() -> bool:return False+def _scale_epilogue(+ acc: Any, a_scale: Any, b_scale: Any, bias: Optional[Any], output_dtype: Any+) -> Any:+ """``(acc * a_scale * b_scale + bias)`` in fp32, rounded once to ``output_dtype``. Eager runs it per row chunk+ in place (bitwise identical) to bound peak memory."""+ import torch++ if _compiling():+ out = acc * a_scale.float()+ out.mul_(b_scale.float())+ if bias is not None:+ out.add_(bias.float())+ return out.to(output_dtype)+ sa, sb = a_scale.float(), b_scale.float()+ bf = bias.float() if bias is not None else None+ rows = int(acc.shape[0])+ step = max(1, _EAGER_EPILOGUE_ELEMS // max(1, int(acc.shape[1])))+ out = acc if acc.dtype == output_dtype else torch.empty_like(acc, dtype = output_dtype)+ for start in range(0, rows, step):+ stop = min(rows, start + step)+ tmp = acc[start:stop] * sa[start:stop]+ tmp.mul_(sb)+ if bf is not None:+ tmp.add_(bf)+ out[start:stop].copy_(tmp)+ # Free before the next chunk, else two are live.+ del tmp+ return out+++def can_engage(fast_accum: Optional[bool]) -> bool:+ """False only for fast accumulate outside ``force``; None (unknown) installs."""+ return _mode() == "force" or fast_accum is not True++def stats() -> dict:return {**_COUNTS, "installed": bool(_STATE["installed"])}@@ -82,12 +122,13 @@ def _device_caps() -> dict:return caps-def install(logger: Any = None) -> bool:- """Patch torchao's fp8 addmm with the dispatcher; idempotent, False when off / no qualifying GPU / layout moved."""+def install(logger: Any = None, *, fast_accum: Optional[bool] = None) -> bool:+ """Patch torchao's fp8 addmm with the dispatcher; idempotent, False when off / no qualifying GPU / layout moved /+ it cannot engage."""if _STATE["installed"]:return Truemode = _mode()- if mode == "off":+ if mode == "off" or not can_engage(fast_accum):return Falsecaps = _device_caps()if mode == "auto" and not any(c in _ADA_CAPS for c in caps.values()):@@ -147,12 +188,7 @@ def install(logger: Any = None) -> bool:)if isinstance(acc, tuple): # torch < 2.5 returns (out, amax)acc = acc[0]- # bf16 * fp32 promotes without an upcast copy; in place saves an fp32 [M, N] buffer eagerly, same result.- out = acc * a_scale.float()- out.mul_(b_scale.float())- if bias is not None:- out.add_(bias.float())- return out.to(output_dtype)+ return _scale_epilogue(acc, a_scale, b_scale, bias, output_dtype)if not _compiling():_COUNTS["rowwise"] += 1return orig(diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex a4c153d85df..73631af1eba 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -261,8 +261,7 @@ def _load_prequant_checkpoint(path: str, **kwargs: Any) -> Any:ckpt = load_prequant_safetensors(path, mmap = bool(kwargs.get("mmap")))else:ckpt = _torch_load_prequant(path, **kwargs)- if not _int8_only(ckpt):- _install_fp8_unit_scale()+ _install_fp8_unit_scale_for(ckpt)return ckpt@@ -282,14 +281,23 @@ def _int8_only(ckpt: Any) -> bool:return False-def _install_fp8_unit_scale() -> None:+def _install_fp8_unit_scale(fast_accum: Optional[bool] = None) -> None:try:from .diffusion_fp8_unit_scale import install- install()+ install(fast_accum = fast_accum)except Exception: # noqa: BLE001 - never block a load on a speed pathpass+def _install_fp8_unit_scale_for(ckpt: Any) -> None:+ """Skip int8 checkpoints and ones baked at fast accumulate (the builder's default)."""+ meta = (ckpt.get("metadata") if isinstance(ckpt, dict) else None) or {}+ if str(meta.get("scheme") or "").lower() == "int8" or _int8_only(ckpt):+ return+ fast_accum = meta.get("fast_accum")+ _install_fp8_unit_scale(fast_accum = fast_accum if isinstance(fast_accum, bool) else None)++# 0 reads a pickle checkpoint headed for an accelerator into host memory instead of mapping it._PREQUANT_MMAP_ENV = "UNSLOTH_DIFFUSION_PREQUANT_MMAP"@@ -2283,7 +2291,9 @@ def load_prequantized_transformer(logger = logger,)# ComfyUI files never pass through _load_prequant_checkpoint, which installs this for Studio's own.- _install_fp8_unit_scale()+ from .diffusion_transformer_quant import _resolve_fast_accum++ _install_fp8_unit_scale(fast_accum = _resolve_fast_accum(fast_accum))from .diffusion_convrot import declares_rotation, is_rotated_linear, warm_rotation_cachecomfy_rotated = any(is_rotated_linear(m) for m in transformer.modules())diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 221a1630348..4be72f681ad 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -1793,12 +1793,12 @@ def _quiet_config(cls: Any, **kwargs: Any) -> Any:return cls(**kwargs)-def _install_fp8_unit_scale() -> None:+def _install_fp8_unit_scale(fast_accum: Optional[bool] = None) -> None:try:import loggingfrom .diffusion_fp8_unit_scale import install- install(logger = logging.getLogger(__name__))+ install(logger = logging.getLogger(__name__), fast_accum = fast_accum)except Exception: # noqa: BLE001 - never block a load on a speed pathpass@@ -1818,7 +1818,7 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:# activation_value_lb floors the scale: an all-zero row otherwise gives NaN.from torchao.quantization import PerRow- _install_fp8_unit_scale()+ _install_fp8_unit_scale(fast_accum = _resolve_fast_accum(fast_accum))fp8_kwargs: dict = {"granularity": PerRow()}config_params = _inspect.signature(Float8DynamicActivationFloat8WeightConfig).parametersif "activation_value_lb" in config_params:diff --git a/studio/backend/tests/test_diffusion_fp8_unit_scale.py b/studio/backend/tests/test_diffusion_fp8_unit_scale.pyindex e5a9d52355a..a6c9b6b4dc4 100644--- a/studio/backend/tests/test_diffusion_fp8_unit_scale.py+++ b/studio/backend/tests/test_diffusion_fp8_unit_scale.py@@ -177,7 +177,7 @@ def test_every_prequant_container_installs_the_dispatch(monkeypatch, container):from core.inference import prequant_safetensors as pscalls = []- monkeypatch.setattr(us, "install", lambda logger = None: calls.append(container) or True)+ monkeypatch.setattr(us, "install", lambda logger = None, **kw: calls.append(container) or True)monkeypatch.setattr(ps, "is_safetensors_checkpoint", lambda path: container == "safetensors")monkeypatch.setattr(ps, "load_prequant_safetensors", lambda path, mmap = False: {"format": "fp8"})monkeypatch.setattr(dp, "_torch_load_prequant", lambda path, **kwargs: {"format": "fp8"})@@ -381,3 +381,93 @@ def test_installed_dispatcher_keys_studio_compile_bundles(monkeypatch):assert cc.cache_key(plain, {}) != cc.cache_key(tagged, {})us.uninstall()assert cc.environment_fingerprint() == plain+++def test_auto_never_installs_for_fast_accumulate(monkeypatch):+ # Installing at fast accumulate would only split every later compile's cache key.+ import torch.compiler.config as ccfg++ monkeypatch.setattr(ccfg, "cache_key_tag", "")+ _caps(monkeypatch, {0: (8, 9)})+ before = f8t.addmm_float8_unwrapped_inference+ assert us.install(fast_accum = True) is False+ assert f8t.addmm_float8_unwrapped_inference is before and ccfg.cache_key_tag == ""+ assert us.install(fast_accum = False) is True+ us.uninstall()+ assert us.install(fast_accum = None) is True+ us.uninstall()+ monkeypatch.setenv(us.UNIT_SCALE_ENV, "1")+ assert us.install(fast_accum = True) is True+++def test_fp8_quant_config_installs_only_for_precise_accumulate(monkeypatch):+ from core.inference import diffusion_transformer_quant as tq++ _caps(monkeypatch, {0: (8, 9)})+ tq._make_quant_config(tq.TQ_FP8)+ assert us.stats()["installed"] is False+ tq._make_quant_config(tq.TQ_FP8, fast_accum = True)+ assert us.stats()["installed"] is False+ tq._make_quant_config(tq.TQ_FP8, fast_accum = False)+ assert us.stats()["installed"] is True+++ "metadata, installs",+ [+ ({"scheme": "int8"}, False),+ ({"scheme": "fp8", "fast_accum": True}, False),+ ({"scheme": "fp8", "fast_accum": False}, True),+ ({"scheme": "nvfp4"}, True),+ ({}, True),+ ],+)+def test_prequant_checkpoint_installs_only_when_its_fp8_layers_can_engage(+ monkeypatch, metadata, installs+):+ from core.inference import diffusion_prequant as dp+ from core.inference import prequant_safetensors as ps++ _caps(monkeypatch, {0: (8, 9)})+ ckpt = {"format": dp.PREQUANT_FORMAT, "state_dict": {}, "metadata": metadata}+ monkeypatch.setattr(ps, "is_safetensors_checkpoint", lambda path: True)+ monkeypatch.setattr(ps, "load_prequant_safetensors", lambda path, mmap = False: ckpt)+ assert dp._load_prequant_checkpoint("model.safetensors") is ckpt+ assert us.stats()["installed"] is installs+++def test_every_off_word_of_the_compile_cache_key_turns_the_dispatcher_off(monkeypatch):+ from core.inference import diffusion_compile_cache as cc++ _caps(monkeypatch, {0: (8, 9)})+ for word in cc._OFF_WORDS:+ monkeypatch.setenv(us.UNIT_SCALE_ENV, word.upper())+ assert cc.graph_switches().get(us.UNIT_SCALE_ENV) == "off"+ assert us._mode() == "off" and us.install(fast_accum = False) is False+ for word in cc._ON_WORDS:+ monkeypatch.setenv(us.UNIT_SCALE_ENV, word)+ assert us._mode() == "force"++[email protected]("bias", [False, True])+def test_eager_epilogue_writes_into_the_accumulator_bitwise_as_before(monkeypatch, bias):+ produced = []++ def scaled_mm(*args, **kwargs):+ acc = _emulated_scaled_mm(*args, **kwargs)+ produced.append(acc)+ return acc++ fn = _install_emulated(monkeypatch, scaled_mm)+ monkeypatch.setattr(us, "_EAGER_EPILOGUE_ELEMS", 48 * 300)+ a, sa, b, sb = _operands(2048)+ bv = torch.randn(48).to(torch.bfloat16) if bias else None+ got = fn(_Dev(a), sa, b, sb, torch.bfloat16, bias = bv)+ acc = produced[-1]+ assert got.data_ptr() == acc.data_ptr()+ unit = _emulated_scaled_mm(a, b, torch.tensor(1.0), torch.tensor(1.0), torch.bfloat16)+ want = unit * sa.float()+ want.mul_(sb.float())+ if bv is not None:+ want.add_(bv.float())+ assert torch.equal(got, want.to(torch.bfloat16))diff --git a/studio/backend/tests/test_diffusion_prequant_cache_compile_key.py b/studio/backend/tests/test_diffusion_prequant_cache_compile_key.pyindex 94c31b62b74..84b1276950e 100644--- a/studio/backend/tests/test_diffusion_prequant_cache_compile_key.py+++ b/studio/backend/tests/test_diffusion_prequant_cache_compile_key.py@@ -108,7 +108,7 @@ def test_an_fp8_or_unknown_read_back_still_installs_the_dispatcher(tmp_path, monimport core.inference.prequant_safetensors as PScalls = []- monkeypatch.setattr(P, "_install_fp8_unit_scale", lambda: calls.append(1))+ monkeypatch.setattr(P, "_install_fp8_unit_scale", lambda **_: calls.append(1))monkeypatch.setattr(PS, "is_safetensors_checkpoint", lambda path: True)fp8 = type("Float8Tensor", (), {})()int8 = type("Int8Tensor", (), {})()