Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: Ada FP8 unit-scale dispatcher installs only where it engages, chunked eager scaling (#14665)

Daniel Han487ee600f369
diff --git a/studio/backend/core/inference/diffusion_fp8_unit_scale.py b/studio/backend/core/inference/diffusion_fp8_unit_scale.py
index 4c1381f9c9e..bda64dbdf78 100644
--- a/studio/backend/core/inference/diffusion_fp8_unit_scale.py
+++ b/studio/backend/core/inference/diffusion_fp8_unit_scale.py
@@ -2,7 +2,8 @@
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""sm_89 precise-accumulate fp8: unit-scale _scaled_mm beats torchao row-wise (fast accumulate stays
-on torchao, no gain); UNSLOTH_DIFFUSION_FP8_UNIT_SCALE=0 off."""
+on torchao, no gain); UNSLOTH_DIFFUSION_FP8_UNIT_SCALE=0 off. Installed only where it can
+engage: installing tags the process-wide compile cache key."""
from __future__ import annotations
@@ -13,6 +14,10 @@ UNIT_SCALE_ENV = "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE"
MIN_ROWS_ENV = "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE_MIN_ROWS"
# Unit-scale wins from M = 256 up (L4: row-wise 2.5-4x slower from 512); below, tens of microseconds either way.
_DEFAULT_MIN_ROWS = 256
+# Eager epilogue chunk size, avoids a full fp32 [M, N] buffer.
+_EAGER_EPILOGUE_ELEMS = 1 << 24
+_OFF_WORDS = ("0", "off", "false", "no", "none")
+_ON_WORDS = ("1", "on", "true", "yes", "force")
_ADA_CAPS = ((8, 9),)
_COUNTS = {"unit_scale": 0, "rowwise": 0}
@@ -21,9 +26,9 @@ _STATE: dict = {"installed": False, "orig": None, "caps": {}}
def _mode() -> str:
v = (os.environ.get(UNIT_SCALE_ENV) or "").strip().lower()
- if v in ("0", "off", "false", "no"):
+ if v in _OFF_WORDS:
return "off"
- if v in ("1", "on", "true", "yes", "force"):
+ if v in _ON_WORDS:
return "force"
return "auto"
@@ -44,6 +49,41 @@ def _compiling() -> bool:
return False
+def _scale_epilogue(
+ acc: Any, a_scale: Any, b_scale: Any, bias: Optional[Any], output_dtype: Any
+) -> Any:
+ """``(acc * a_scale * b_scale + bias)`` in fp32, rounded once to ``output_dtype``. Eager runs it per row chunk
+ in place (bitwise identical) to bound peak memory."""
+ import torch
+
+ if _compiling():
+ out = acc * a_scale.float()
+ out.mul_(b_scale.float())
+ if bias is not None:
+ out.add_(bias.float())
+ return out.to(output_dtype)
+ sa, sb = a_scale.float(), b_scale.float()
+ bf = bias.float() if bias is not None else None
+ rows = int(acc.shape[0])
+ step = max(1, _EAGER_EPILOGUE_ELEMS // max(1, int(acc.shape[1])))
+ out = acc if acc.dtype == output_dtype else torch.empty_like(acc, dtype = output_dtype)
+ for start in range(0, rows, step):
+ stop = min(rows, start + step)
+ tmp = acc[start:stop] * sa[start:stop]
+ tmp.mul_(sb)
+ if bf is not None:
+ tmp.add_(bf)
+ out[start:stop].copy_(tmp)
+ # Free before the next chunk, else two are live.
+ del tmp
+ return out
+
+
+def can_engage(fast_accum: Optional[bool]) -> bool:
+ """False only for fast accumulate outside ``force``; None (unknown) installs."""
+ return _mode() == "force" or fast_accum is not True
+
+
def stats() -> dict:
return {**_COUNTS, "installed": bool(_STATE["installed"])}
@@ -82,12 +122,13 @@ def _device_caps() -> dict:
return caps
-def install(logger: Any = None) -> bool:
- """Patch torchao's fp8 addmm with the dispatcher; idempotent, False when off / no qualifying GPU / layout moved."""
+def install(logger: Any = None, *, fast_accum: Optional[bool] = None) -> bool:
+ """Patch torchao's fp8 addmm with the dispatcher; idempotent, False when off / no qualifying GPU / layout moved /
+ it cannot engage."""
if _STATE["installed"]:
return True
mode = _mode()
- if mode == "off":
+ if mode == "off" or not can_engage(fast_accum):
return False
caps = _device_caps()
if mode == "auto" and not any(c in _ADA_CAPS for c in caps.values()):
@@ -147,12 +188,7 @@ def install(logger: Any = None) -> bool:
)
if isinstance(acc, tuple): # torch < 2.5 returns (out, amax)
acc = acc[0]
- # bf16 * fp32 promotes without an upcast copy; in place saves an fp32 [M, N] buffer eagerly, same result.
- out = acc * a_scale.float()
- out.mul_(b_scale.float())
- if bias is not None:
- out.add_(bias.float())
- return out.to(output_dtype)
+ return _scale_epilogue(acc, a_scale, b_scale, bias, output_dtype)
if not _compiling():
_COUNTS["rowwise"] += 1
return orig(
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index a4c153d85df..73631af1eba 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -261,8 +261,7 @@ def _load_prequant_checkpoint(path: str, **kwargs: Any) -> Any:
ckpt = load_prequant_safetensors(path, mmap = bool(kwargs.get("mmap")))
else:
ckpt = _torch_load_prequant(path, **kwargs)
- if not _int8_only(ckpt):
- _install_fp8_unit_scale()
+ _install_fp8_unit_scale_for(ckpt)
return ckpt
@@ -282,14 +281,23 @@ def _int8_only(ckpt: Any) -> bool:
return False
-def _install_fp8_unit_scale() -> None:
+def _install_fp8_unit_scale(fast_accum: Optional[bool] = None) -> None:
try:
from .diffusion_fp8_unit_scale import install
- install()
+ install(fast_accum = fast_accum)
except Exception: # noqa: BLE001 - never block a load on a speed path
pass
+def _install_fp8_unit_scale_for(ckpt: Any) -> None:
+ """Skip int8 checkpoints and ones baked at fast accumulate (the builder's default)."""
+ meta = (ckpt.get("metadata") if isinstance(ckpt, dict) else None) or {}
+ if str(meta.get("scheme") or "").lower() == "int8" or _int8_only(ckpt):
+ return
+ fast_accum = meta.get("fast_accum")
+ _install_fp8_unit_scale(fast_accum = fast_accum if isinstance(fast_accum, bool) else None)
+
+
# 0 reads a pickle checkpoint headed for an accelerator into host memory instead of mapping it.
_PREQUANT_MMAP_ENV = "UNSLOTH_DIFFUSION_PREQUANT_MMAP"
@@ -2283,7 +2291,9 @@ def load_prequantized_transformer(
logger = logger,
)
# ComfyUI files never pass through _load_prequant_checkpoint, which installs this for Studio's own.
- _install_fp8_unit_scale()
+ from .diffusion_transformer_quant import _resolve_fast_accum
+
+ _install_fp8_unit_scale(fast_accum = _resolve_fast_accum(fast_accum))
from .diffusion_convrot import declares_rotation, is_rotated_linear, warm_rotation_cache
comfy_rotated = any(is_rotated_linear(m) for m in transformer.modules())
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 221a1630348..4be72f681ad 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -1793,12 +1793,12 @@ def _quiet_config(cls: Any, **kwargs: Any) -> Any:
return cls(**kwargs)
-def _install_fp8_unit_scale() -> None:
+def _install_fp8_unit_scale(fast_accum: Optional[bool] = None) -> None:
try:
import logging
from .diffusion_fp8_unit_scale import install
- install(logger = logging.getLogger(__name__))
+ install(logger = logging.getLogger(__name__), fast_accum = fast_accum)
except Exception: # noqa: BLE001 - never block a load on a speed path
pass
@@ -1818,7 +1818,7 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
# activation_value_lb floors the scale: an all-zero row otherwise gives NaN.
from torchao.quantization import PerRow
- _install_fp8_unit_scale()
+ _install_fp8_unit_scale(fast_accum = _resolve_fast_accum(fast_accum))
fp8_kwargs: dict = {"granularity": PerRow()}
config_params = _inspect.signature(Float8DynamicActivationFloat8WeightConfig).parameters
if "activation_value_lb" in config_params:
diff --git a/studio/backend/tests/test_diffusion_fp8_unit_scale.py b/studio/backend/tests/test_diffusion_fp8_unit_scale.py
index e5a9d52355a..a6c9b6b4dc4 100644
--- a/studio/backend/tests/test_diffusion_fp8_unit_scale.py
+++ b/studio/backend/tests/test_diffusion_fp8_unit_scale.py
@@ -177,7 +177,7 @@ def test_every_prequant_container_installs_the_dispatch(monkeypatch, container):
from core.inference import prequant_safetensors as ps
calls = []
- monkeypatch.setattr(us, "install", lambda logger = None: calls.append(container) or True)
+ monkeypatch.setattr(us, "install", lambda logger = None, **kw: calls.append(container) or True)
monkeypatch.setattr(ps, "is_safetensors_checkpoint", lambda path: container == "safetensors")
monkeypatch.setattr(ps, "load_prequant_safetensors", lambda path, mmap = False: {"format": "fp8"})
monkeypatch.setattr(dp, "_torch_load_prequant", lambda path, **kwargs: {"format": "fp8"})
@@ -381,3 +381,93 @@ def test_installed_dispatcher_keys_studio_compile_bundles(monkeypatch):
assert cc.cache_key(plain, {}) != cc.cache_key(tagged, {})
us.uninstall()
assert cc.environment_fingerprint() == plain
+
+
+def test_auto_never_installs_for_fast_accumulate(monkeypatch):
+ # Installing at fast accumulate would only split every later compile's cache key.
+ import torch.compiler.config as ccfg
+
+ monkeypatch.setattr(ccfg, "cache_key_tag", "")
+ _caps(monkeypatch, {0: (8, 9)})
+ before = f8t.addmm_float8_unwrapped_inference
+ assert us.install(fast_accum = True) is False
+ assert f8t.addmm_float8_unwrapped_inference is before and ccfg.cache_key_tag == ""
+ assert us.install(fast_accum = False) is True
+ us.uninstall()
+ assert us.install(fast_accum = None) is True
+ us.uninstall()
+ monkeypatch.setenv(us.UNIT_SCALE_ENV, "1")
+ assert us.install(fast_accum = True) is True
+
+
+def test_fp8_quant_config_installs_only_for_precise_accumulate(monkeypatch):
+ from core.inference import diffusion_transformer_quant as tq
+
+ _caps(monkeypatch, {0: (8, 9)})
+ tq._make_quant_config(tq.TQ_FP8)
+ assert us.stats()["installed"] is False
+ tq._make_quant_config(tq.TQ_FP8, fast_accum = True)
+ assert us.stats()["installed"] is False
+ tq._make_quant_config(tq.TQ_FP8, fast_accum = False)
+ assert us.stats()["installed"] is True
+
+
+ "metadata, installs",
+ [
+ ({"scheme": "int8"}, False),
+ ({"scheme": "fp8", "fast_accum": True}, False),
+ ({"scheme": "fp8", "fast_accum": False}, True),
+ ({"scheme": "nvfp4"}, True),
+ ({}, True),
+ ],
+)
+def test_prequant_checkpoint_installs_only_when_its_fp8_layers_can_engage(
+ monkeypatch, metadata, installs
+):
+ from core.inference import diffusion_prequant as dp
+ from core.inference import prequant_safetensors as ps
+
+ _caps(monkeypatch, {0: (8, 9)})
+ ckpt = {"format": dp.PREQUANT_FORMAT, "state_dict": {}, "metadata": metadata}
+ monkeypatch.setattr(ps, "is_safetensors_checkpoint", lambda path: True)
+ monkeypatch.setattr(ps, "load_prequant_safetensors", lambda path, mmap = False: ckpt)
+ assert dp._load_prequant_checkpoint("model.safetensors") is ckpt
+ assert us.stats()["installed"] is installs
+
+
+def test_every_off_word_of_the_compile_cache_key_turns_the_dispatcher_off(monkeypatch):
+ from core.inference import diffusion_compile_cache as cc
+
+ _caps(monkeypatch, {0: (8, 9)})
+ for word in cc._OFF_WORDS:
+ monkeypatch.setenv(us.UNIT_SCALE_ENV, word.upper())
+ assert cc.graph_switches().get(us.UNIT_SCALE_ENV) == "off"
+ assert us._mode() == "off" and us.install(fast_accum = False) is False
+ for word in cc._ON_WORDS:
+ monkeypatch.setenv(us.UNIT_SCALE_ENV, word)
+ assert us._mode() == "force"
+
+
[email protected]("bias", [False, True])
+def test_eager_epilogue_writes_into_the_accumulator_bitwise_as_before(monkeypatch, bias):
+ produced = []
+
+ def scaled_mm(*args, **kwargs):
+ acc = _emulated_scaled_mm(*args, **kwargs)
+ produced.append(acc)
+ return acc
+
+ fn = _install_emulated(monkeypatch, scaled_mm)
+ monkeypatch.setattr(us, "_EAGER_EPILOGUE_ELEMS", 48 * 300)
+ a, sa, b, sb = _operands(2048)
+ bv = torch.randn(48).to(torch.bfloat16) if bias else None
+ got = fn(_Dev(a), sa, b, sb, torch.bfloat16, bias = bv)
+ acc = produced[-1]
+ assert got.data_ptr() == acc.data_ptr()
+ unit = _emulated_scaled_mm(a, b, torch.tensor(1.0), torch.tensor(1.0), torch.bfloat16)
+ want = unit * sa.float()
+ want.mul_(sb.float())
+ if bv is not None:
+ want.add_(bv.float())
+ assert torch.equal(got, want.to(torch.bfloat16))
diff --git a/studio/backend/tests/test_diffusion_prequant_cache_compile_key.py b/studio/backend/tests/test_diffusion_prequant_cache_compile_key.py
index 94c31b62b74..84b1276950e 100644
--- a/studio/backend/tests/test_diffusion_prequant_cache_compile_key.py
+++ b/studio/backend/tests/test_diffusion_prequant_cache_compile_key.py
@@ -108,7 +108,7 @@ def test_an_fp8_or_unknown_read_back_still_installs_the_dispatcher(tmp_path, mon
import core.inference.prequant_safetensors as PS
calls = []
- monkeypatch.setattr(P, "_install_fp8_unit_scale", lambda: calls.append(1))
+ monkeypatch.setattr(P, "_install_fp8_unit_scale", lambda **_: calls.append(1))
monkeypatch.setattr(PS, "is_safetensors_checkpoint", lambda path: True)
fp8 = type("Float8Tensor", (), {})()
int8 = type("Int8Tensor", (), {})()