Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run Ada FP8 precise accumulate through a unit-scale GEMM plus a fused scale epilogue (#13793)

Daniel Hanedb8e677704f
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index 81db420ed1e..5298b404b1c 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -280,6 +280,12 @@ def environment_fingerprint() -> dict[str, Any]:
fp["gpu_name"] = torch.cuda.get_device_name(index)
cap = torch.cuda.get_device_capability(index)
fp["gpu_capability"] = f"sm_{cap[0]}{cap[1]}"
+ # torch keys artifacts on this tag (set by diffusion_fp8_unit_scale); a bundle from another tag would load,
+ # miss, recompile and never be rewritten. Only a set tag joins the key, so untagged bundles stay valid.
+ tag = getattr(getattr(torch, "compiler", None), "config", None)
+ tag = getattr(tag, "cache_key_tag", None)
+ if isinstance(tag, str) and tag:
+ fp["torch_cache_key_tag"] = tag
except Exception: # noqa: BLE001 - best-effort
pass
return fp
@@ -304,6 +310,8 @@ _GRAPH_SWITCHES = {
"UNSLOTH_STATIC_STREAM_MERGE_DETECT": "off",
"UNSLOTH_TORCHAO_INDUCTOR_CONFIG": "off",
"UNSLOTH_NVFP4_BACKEND": "raw:auto",
+ "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE": "auto",
+ "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE_MIN_ROWS": "rows:256",
# Unset is off; "auto" protects the head + last step, so "auto" is NOT the default here.
"UNSLOTH_NVFP4_PROTECT_STEPS": "raw",
}
@@ -316,6 +324,13 @@ def _switch_value(name: str, default: str) -> Optional[str]:
raw = (os.environ.get(name) or "").strip().lower()
if not raw:
return None
+ if default.startswith("rows:"):
+ # A row threshold keys on the value the reader uses (max(1, int)); junk falls back to the default there.
+ try:
+ rows = str(max(1, int(raw)))
+ except ValueError:
+ return None
+ return None if rows == default.partition(":")[2] else rows
if default.startswith("raw"):
return None if raw in _OFF_WORDS or raw == default.partition(":")[2] else raw
value = "off" if raw in _OFF_WORDS else "on" if raw in _ON_WORDS else None
diff --git a/studio/backend/core/inference/diffusion_fp8_unit_scale.py b/studio/backend/core/inference/diffusion_fp8_unit_scale.py
new file mode 100644
index 00000000000..919f0db5add
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_fp8_unit_scale.py
@@ -0,0 +1,205 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""sm_89 precise-accumulate fp8 GEMM: unit-scale ``torch._scaled_mm`` + fp32 row x column scale epilogue.
+
+On sm_89 torchao's row-wise ``_scaled_mm`` hits a slow CUTLASS stream-K kernel. L4, torch 2.14.1, 1024 px linears:
+* precise accumulate: row-wise 29-32 TFLOP/s, per-tensor 102-134 TFLOP/s; end to end klein-4B 1.034 -> 0.480 s/step,
+ Z-Image-Turbo 1.708 -> 0.772 s/step.
+* fast accumulate: row-wise 85-108, per-tensor 91-127 TFLOP/s, no end-to-end gain, so left on torchao (bit-identical).
+Only difference vs row-wise: one bf16 rounding of the unscaled accumulator (relative error <= 2^-9).
+
+``UNSLOTH_DIFFUSION_FP8_UNIT_SCALE``: ``0`` off, ``1`` every arch + both accumulate modes (testing), else auto.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Optional
+
+UNIT_SCALE_ENV = "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE"
+MIN_ROWS_ENV = "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE_MIN_ROWS"
+# Unit-scale wins from M = 256 up (L4: row-wise 2.5-4x slower from 512); below, tens of microseconds either way.
+_DEFAULT_MIN_ROWS = 256
+_ADA_CAPS = ((8, 9),)
+
+_COUNTS = {"unit_scale": 0, "rowwise": 0}
+_STATE: dict = {"installed": False, "orig": None, "caps": {}}
+
+
+def _mode() -> str:
+ v = (os.environ.get(UNIT_SCALE_ENV) or "").strip().lower()
+ if v in ("0", "off", "false", "no"):
+ return "off"
+ if v in ("1", "on", "true", "yes", "force"):
+ return "force"
+ return "auto"
+
+
+def _min_rows() -> int:
+ try:
+ return max(1, int(os.environ.get(MIN_ROWS_ENV, "") or _DEFAULT_MIN_ROWS))
+ except ValueError:
+ return _DEFAULT_MIN_ROWS
+
+
+def _compiling() -> bool:
+ # Counters are eager-only: a module-global mutated inside a traced region makes Dynamo guard on it and recompile.
+ try:
+ import torch
+ return bool(torch.compiler.is_compiling())
+ except Exception: # noqa: BLE001 - no torch.compiler: nothing is being traced
+ return False
+
+
+def stats() -> dict:
+ return {**_COUNTS, "installed": bool(_STATE["installed"])}
+
+
+_CACHE_TAG_PREFIX = "unsloth_fp8_unit_scale="
+
+
+def _set_compile_cache_tag(tag: Optional[str]) -> None:
+ """Key torch's FX/AOTAutograd caches on the dispatcher: the Dynamo graph is identical with or without the patch,
+ so untagged, a graph cached on the other side of install / the kill switch is reused. ``None`` removes our tag."""
+ try:
+ import torch.compiler.config as _cc
+ except Exception: # noqa: BLE001 - torch without compiler config: no persistent compile cache to split
+ return
+ current = getattr(_cc, "cache_key_tag", None)
+ if not isinstance(current, str):
+ return
+ parts = [p for p in current.split(";") if p and not p.startswith(_CACHE_TAG_PREFIX)]
+ if tag:
+ parts.append(_CACHE_TAG_PREFIX + tag)
+ try:
+ _cc.cache_key_tag = ";".join(parts)
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _device_caps() -> dict:
+ import torch
+
+ caps = {}
+ try:
+ if torch.cuda.is_available():
+ for i in range(torch.cuda.device_count()):
+ caps[i] = tuple(torch.cuda.get_device_capability(i))
+ except Exception: # noqa: BLE001 - no CUDA runtime: nothing qualifies
+ return {}
+ return caps
+
+
+def install(logger: Any = None) -> bool:
+ """Patch torchao's fp8 addmm with the dispatcher; idempotent, False when off / no qualifying GPU / layout moved."""
+ if _STATE["installed"]:
+ return True
+ mode = _mode()
+ if mode == "off":
+ return False
+ caps = _device_caps()
+ if mode == "auto" and not any(c in _ADA_CAPS for c in caps.values()):
+ return False
+ try:
+ from torchao.quantization.quantize_.workflows.float8 import float8_tensor as _f8t
+ except Exception: # noqa: BLE001 - torchao absent / stubbed / moved
+ return False
+ orig = getattr(_f8t, "addmm_float8_unwrapped_inference", None)
+ if orig is None or not callable(orig):
+ return False
+ if getattr(orig, "_unsloth_unit_scale", False):
+ _STATE.update(installed = True)
+ return True
+ min_rows = _min_rows()
+ force = mode == "force"
+ qualifying = frozenset(i for i, c in caps.items() if force or c in _ADA_CAPS)
+
+ import torch
+
+ def addmm_float8_unwrapped_inference(
+ a_data: Any,
+ a_scale: Any,
+ b_data: Any,
+ b_scale: Any,
+ output_dtype: Any,
+ output_scale: Optional[Any] = None,
+ bias: Optional[Any] = None,
+ use_fast_accum: bool = False,
+ ) -> Any:
+ if (
+ (force or not use_fast_accum)
+ and output_scale is None
+ and output_dtype == torch.bfloat16
+ and a_data.device.type == "cuda"
+ and a_data.device.index in qualifying
+ and a_data.dim() == 2
+ and b_data.dim() == 2
+ and a_scale.dim() == 2
+ and b_scale.dim() == 2
+ and a_scale.shape[1] == 1
+ and b_scale.shape[0] == 1
+ and a_scale.shape[0] == a_data.shape[0]
+ and b_scale.shape[1] == b_data.shape[1]
+ and a_data.shape[0] >= min_rows
+ ):
+ if not _compiling():
+ _COUNTS["unit_scale"] += 1
+ one = torch.ones((), device = a_data.device, dtype = torch.float32)
+ acc = torch._scaled_mm(
+ a_data,
+ b_data,
+ scale_a = one,
+ scale_b = one,
+ out_dtype = torch.bfloat16,
+ use_fast_accum = use_fast_accum,
+ )
+ if isinstance(acc, tuple): # torch < 2.5 returns (out, amax)
+ acc = acc[0]
+ # bf16 * fp32 promotes without an upcast copy; in place saves an fp32 [M, N] buffer eagerly, same result.
+ out = acc * a_scale.float()
+ out.mul_(b_scale.float())
+ if bias is not None:
+ out.add_(bias.float())
+ return out.to(output_dtype)
+ if not _compiling():
+ _COUNTS["rowwise"] += 1
+ return orig(
+ a_data,
+ a_scale,
+ b_data,
+ b_scale,
+ output_dtype,
+ output_scale = output_scale,
+ bias = bias,
+ use_fast_accum = use_fast_accum,
+ )
+
+ addmm_float8_unwrapped_inference._unsloth_unit_scale = True # type: ignore[attr-defined]
+ addmm_float8_unwrapped_inference._unsloth_orig = orig # type: ignore[attr-defined]
+ _f8t.addmm_float8_unwrapped_inference = addmm_float8_unwrapped_inference
+ _STATE.update(installed = True, orig = orig, caps = caps)
+ _set_compile_cache_tag(f"{min_rows}{':force' if force else ''}")
+ if logger is not None:
+ try:
+ logger.info(
+ "fp8: unit-scale GEMM + row/column scale epilogue on %s (rows >= %d, %s=0 disables)",
+ "every CUDA device" if force else "sm_89 precise accumulate",
+ min_rows,
+ UNIT_SCALE_ENV,
+ )
+ except Exception: # noqa: BLE001
+ pass
+ return True
+
+
+def uninstall() -> None:
+ if not _STATE["installed"]:
+ return
+ try:
+ from torchao.quantization.quantize_.workflows.float8 import float8_tensor as _f8t
+ if _STATE["orig"] is not None:
+ _f8t.addmm_float8_unwrapped_inference = _STATE["orig"]
+ finally:
+ _STATE.update(installed = False, orig = None, caps = {})
+ _set_compile_cache_tag(None)
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index dd5a284a819..acb65d36a07 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -346,8 +346,19 @@ def _load_prequant_checkpoint(path: str, **kwargs: Any) -> Any:
from .prequant_safetensors import is_safetensors_checkpoint, load_prequant_safetensors
if is_safetensors_checkpoint(path):
- return load_prequant_safetensors(path, mmap = bool(kwargs.get("mmap")))
- return _torch_load_prequant(path, **kwargs)
+ ckpt = load_prequant_safetensors(path, mmap = bool(kwargs.get("mmap")))
+ else:
+ ckpt = _torch_load_prequant(path, **kwargs)
+ _install_fp8_unit_scale()
+ return ckpt
+
+
+def _install_fp8_unit_scale() -> None:
+ try:
+ from .diffusion_fp8_unit_scale import install
+ install()
+ except Exception: # noqa: BLE001 - never block a load on a speed path
+ pass
# 0 reads a pickle checkpoint headed for an accelerator into host memory instead of mapping it.
@@ -2381,6 +2392,8 @@ def load_prequantized_transformer(
config_subfolder = config_subfolder,
logger = logger,
)
+ # ComfyUI files never pass through _load_prequant_checkpoint, which installs this for Studio's own.
+ _install_fp8_unit_scale()
from .diffusion_convrot import declares_rotation, is_rotated_linear, warm_rotation_cache
comfy_rotated = any(is_rotated_linear(m) for m in transformer.modules())
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index a4c131f99a5..ad1ff8e1c08 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -1736,6 +1736,16 @@ def _quiet_config(cls: Any, **kwargs: Any) -> Any:
return cls(**kwargs)
+def _install_fp8_unit_scale() -> None:
+ try:
+ import logging
+
+ from .diffusion_fp8_unit_scale import install
+ install(logger = logging.getLogger(__name__))
+ except Exception: # noqa: BLE001 - never block a load on a speed path
+ pass
+
+
def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
"""The torchao dynamic-activation config for ``scheme`` (lazy imports per branch).
``fast_accum`` applies to fp8 only: None auto-detects by GPU class, True/False force it."""
@@ -1755,6 +1765,7 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
# scale 0, NaN qdata, black frames.
from torchao.quantization import PerRow
+ _install_fp8_unit_scale()
fp8_kwargs: dict = {"granularity": PerRow()}
config_params = _inspect.signature(Float8DynamicActivationFloat8WeightConfig).parameters
if "activation_value_lb" in config_params:
diff --git a/studio/backend/tests/test_diffusion_compile_cache_variant.py b/studio/backend/tests/test_diffusion_compile_cache_variant.py
index 2b1f8070bba..3ec4ca2ae7d 100644
--- a/studio/backend/tests/test_diffusion_compile_cache_variant.py
+++ b/studio/backend/tests/test_diffusion_compile_cache_variant.py
@@ -125,14 +125,21 @@ def test_an_instance_forward_swap_keys_apart():
def test_graph_switches_key_apart_only_off_their_default(monkeypatch, name, default, quant):
m = _int8() if quant else _model()
k0 = _key(m, quant)
+ rows = default.partition(":")[2] if default.startswith("rows:") else None
for same in (
("",)
+ (() if default == "raw" else ("auto",))
+ ((default,) if default in ("on", "off") else ())
+ + ((rows,) if rows else ())
):
monkeypatch.setenv(name, same)
assert _key(m, quant) == k0, same
- flipped = {"on": "0", "off": "1", "auto": "0", "raw": "torchao", "raw:auto": "torchao"}[default]
+ if rows:
+ flipped = "1"
+ else:
+ flipped = {"on": "0", "off": "1", "auto": "0", "raw": "torchao", "raw:auto": "torchao"}[
+ default
+ ]
monkeypatch.setenv(name, flipped)
assert _key(m, quant) != k0
assert name in cc.model_fingerprint(transformer = m, quant = quant, **_FP_KW)["switches"]
diff --git a/studio/backend/tests/test_diffusion_fp8_unit_scale.py b/studio/backend/tests/test_diffusion_fp8_unit_scale.py
new file mode 100644
index 00000000000..0b88aca1f4c
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_fp8_unit_scale.py
@@ -0,0 +1,380 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""sm_89 fp8 unit-scale dispatch: install gating, kill switch, fallback, and the epilogue math (CPU, emulated GEMM)."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+pytest.importorskip("torchao")
+from torchao.quantization.quantize_.workflows.float8 import float8_tensor as f8t # noqa: E402
+
+from core.inference import diffusion_fp8_unit_scale as us # noqa: E402
+
+
[email protected](autouse = True)
+def _clean(monkeypatch):
+ us.uninstall()
+ monkeypatch.delenv(us.UNIT_SCALE_ENV, raising = False)
+ monkeypatch.delenv(us.MIN_ROWS_ENV, raising = False)
+ for k in us._COUNTS:
+ us._COUNTS[k] = 0
+ original = f8t.addmm_float8_unwrapped_inference
+ yield
+ us.uninstall()
+ f8t.addmm_float8_unwrapped_inference = original
+
+
+def _caps(monkeypatch, caps):
+ monkeypatch.setattr(us, "_device_caps", lambda: dict(caps))
+
+
+def test_kill_switch_never_installs(monkeypatch):
+ _caps(monkeypatch, {0: (8, 9)})
+ monkeypatch.setenv(us.UNIT_SCALE_ENV, "0")
+ before = f8t.addmm_float8_unwrapped_inference
+ assert us.install() is False
+ assert f8t.addmm_float8_unwrapped_inference is before
+
+
[email protected]("cap", [(8, 0), (8, 6), (9, 0), (10, 0), (12, 0)])
+def test_auto_leaves_other_arches_untouched(monkeypatch, cap):
+ _caps(monkeypatch, {0: cap})
+ before = f8t.addmm_float8_unwrapped_inference
+ assert us.install() is False
+ assert f8t.addmm_float8_unwrapped_inference is before
+ assert us.stats()["installed"] is False
+
+
+def test_auto_installs_on_ada_idempotent_and_reversible(monkeypatch):
+ _caps(monkeypatch, {0: (8, 9)})
+ before = f8t.addmm_float8_unwrapped_inference
+ assert us.install() is True
+ wrapped = f8t.addmm_float8_unwrapped_inference
+ assert wrapped is not before and wrapped._unsloth_orig is before
+ assert us.install() is True
+ assert f8t.addmm_float8_unwrapped_inference is wrapped
+ us.uninstall()
+ assert f8t.addmm_float8_unwrapped_inference is before
+
+
+def _operands(
+ m,
+ k = 64,
+ n = 48,
+ seed = 0,
+):
+ g = torch.Generator().manual_seed(seed)
+ a = (torch.randn(m, k, generator = g) * 3).to(torch.float8_e4m3fn)
+ b = (torch.randn(n, k, generator = g) * 3).to(torch.float8_e4m3fn).t()
+ sa = torch.rand(m, 1, generator = g) + 0.1
+ sb = torch.rand(1, n, generator = g) + 0.1
+ return a, sa, b, sb
+
+
+def test_non_cuda_calls_fall_back_to_torchao(monkeypatch):
+ _caps(monkeypatch, {0: (8, 9)})
+ seen = []
+ monkeypatch.setattr(
+ f8t, "addmm_float8_unwrapped_inference", lambda *a, **k: seen.append((a, k)) or "orig"
+ )
+ assert us.install() is True
+ a, sa, b, sb = _operands(2048)
+ out = f8t.addmm_float8_unwrapped_inference(a, sa, b, sb, torch.bfloat16, use_fast_accum = True)
+ assert out == "orig" and len(seen) == 1 and seen[0][1]["use_fast_accum"] is True
+ assert us.stats()["unit_scale"] == 0 and us.stats()["rowwise"] == 1
+
+
+class _FakeDevice:
+ type = "cuda"
+ index = 0
+
+
+class _Dev:
+ def __init__(self, t):
+ self.t = t
+ self.device = _FakeDevice()
+ self.shape = t.shape
+
+ def dim(self):
+ return self.t.dim()
+
+
+def _emulated_scaled_mm(
+ a,
+ b,
+ scale_a,
+ scale_b,
+ out_dtype,
+ use_fast_accum = False,
+ **_,
+):
+ a = a.t if isinstance(a, _Dev) else a
+ acc = (a.float() * scale_a.float()) @ (b.float() * scale_b.float())
+ return acc.to(out_dtype)
+
+
[email protected]("bias", [False, True])
+def test_epilogue_matches_rowwise_math(monkeypatch, bias):
+ _caps(monkeypatch, {0: (8, 9)})
+ rowwise_calls = []
+
+ def rowwise(
+ a,
+ sa,
+ b,
+ sb,
+ output_dtype,
+ output_scale = None,
+ bias = None,
+ use_fast_accum = False,
+ ):
+ rowwise_calls.append(a.shape[0])
+ a = a.t if isinstance(a, _Dev) else a
+ out = (a.float() @ b.float()) * sa * sb
+ return (out + bias.float() if bias is not None else out).to(output_dtype)
+
+ monkeypatch.setattr(f8t, "addmm_float8_unwrapped_inference", rowwise)
+ monkeypatch.setattr(torch, "_scaled_mm", _emulated_scaled_mm)
+ monkeypatch.setattr(
+ torch, "ones", lambda *a, device = None, dtype = None: torch.tensor(1.0, dtype = dtype)
+ )
+ monkeypatch.setenv(us.MIN_ROWS_ENV, "1024")
+ assert us.install() is True
+ fn = f8t.addmm_float8_unwrapped_inference
+ a, sa, b, sb = _operands(2048)
+ bv = torch.randn(48).to(torch.bfloat16) if bias else None
+ got = fn(_Dev(a), sa, b, sb, torch.bfloat16, bias = bv)
+ want = rowwise(a, sa, b, sb, torch.bfloat16, bias = bv)
+ assert us.stats()["unit_scale"] == 1
+ # One extra bf16 rounding of the unscaled accumulator (bias adds after it, so measure against the output range).
+ err = (got.float() - want.float()).abs().max()
+ assert err <= 2**-7 * want.float().abs().max()
+ assert torch.isfinite(got).all()
+
+ small = _operands(256)
+ fn(_Dev(a), sa, b, sb, torch.bfloat16, use_fast_accum = True)
+ fn(_Dev(small[0]), small[1], small[2], small[3], torch.bfloat16)
+ fn(_Dev(a), sa, b, sb, torch.float32)
+ fn(_Dev(a), sa, b, sb, torch.bfloat16, output_scale = torch.tensor(1.0))
+ assert us.stats()["unit_scale"] == 1 and len(rowwise_calls) == 5
+
+
+def test_min_rows_env(monkeypatch):
+ monkeypatch.setenv(us.MIN_ROWS_ENV, "4096")
+ assert us._min_rows() == 4096
+ monkeypatch.setenv(us.MIN_ROWS_ENV, "junk")
+ assert us._min_rows() == us._DEFAULT_MIN_ROWS
+
+
[email protected]("container", ["pickle", "safetensors"])
+def test_every_prequant_container_installs_the_dispatch(monkeypatch, container):
+ # The pickle container and the safetensors rebuild-off path never reach prequant_native's torchao API.
+ from core.inference import diffusion_fp8_unit_scale as us
+ from core.inference import diffusion_prequant as dp
+ from core.inference import prequant_safetensors as ps
+
+ calls = []
+ monkeypatch.setattr(us, "install", lambda logger = None: calls.append(container) or True)
+ monkeypatch.setattr(ps, "is_safetensors_checkpoint", lambda path: container == "safetensors")
+ monkeypatch.setattr(ps, "load_prequant_safetensors", lambda path, mmap = False: {"format": "fp8"})
+ monkeypatch.setattr(dp, "_torch_load_prequant", lambda path, **kwargs: {"format": "fp8"})
+ assert dp._load_prequant_checkpoint("model.bin", map_location = "cpu") == {"format": "fp8"}
+ assert calls == [container]
+
+
+def _install_emulated(monkeypatch, scaled_mm = _emulated_scaled_mm):
+ _caps(monkeypatch, {0: (8, 9)})
+ monkeypatch.setattr(f8t, "addmm_float8_unwrapped_inference", lambda *a, **k: "orig")
+ monkeypatch.setattr(torch, "_scaled_mm", scaled_mm)
+ monkeypatch.setattr(
+ torch, "ones", lambda *a, device = None, dtype = None: torch.tensor(1.0, dtype = dtype)
+ )
+ assert us.install() is True
+ return f8t.addmm_float8_unwrapped_inference
+
+
+def test_counters_stay_untouched_while_compiling(monkeypatch):
+ # A module-global mutated inside a traced region would make Dynamo guard on it and recompile every call.
+ fn = _install_emulated(monkeypatch)
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ a, sa, b, sb = _operands(2048)
+ fn(_Dev(a), sa, b, sb, torch.bfloat16)
+ fn(_Dev(a), sa, b, sb, torch.bfloat16, use_fast_accum = True)
+ assert us.stats()["unit_scale"] == 0 and us.stats()["rowwise"] == 0
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: False)
+ fn(_Dev(a), sa, b, sb, torch.bfloat16)
+ assert us.stats()["unit_scale"] == 1
+
+
+def test_legacy_scaled_mm_tuple_is_unwrapped(monkeypatch):
+ def legacy(*args, **kwargs):
+ return _emulated_scaled_mm(*args, **kwargs), torch.tensor(0.0)
+
+ fn = _install_emulated(monkeypatch, legacy)
+ a, sa, b, sb = _operands(2048)
+ out = fn(_Dev(a), sa, b, sb, torch.bfloat16)
+ assert isinstance(out, torch.Tensor) and out.shape == (2048, 48)
+
+
+def test_unit_scale_switches_key_the_compile_cache(monkeypatch):
+ from core.inference import diffusion_compile_cache as cc
+
+ assert cc.graph_switches() == {}
+ monkeypatch.setenv(us.UNIT_SCALE_ENV, "0")
+ monkeypatch.setenv(us.MIN_ROWS_ENV, "1024")
+ assert cc.graph_switches() == {us.UNIT_SCALE_ENV: "off", us.MIN_ROWS_ENV: "1024"}
+ monkeypatch.setenv(us.MIN_ROWS_ENV, "256")
+ assert us.MIN_ROWS_ENV not in cc.graph_switches()
+ # Keyed on the value _min_rows() uses: 0 runs as 1, so it must not collide with the 256 default.
+ for raw, keyed in (("0", "1"), ("1", "1"), ("junk", None)):
+ monkeypatch.setenv(us.MIN_ROWS_ENV, raw)
+ assert cc.graph_switches().get(us.MIN_ROWS_ENV) == keyed
+ assert us._min_rows() == (int(keyed) if keyed else us._DEFAULT_MIN_ROWS)
+
+
+def test_comfy_prequant_load_installs_the_dispatch():
+ # The ComfyUI branch of load_prequantized_transformer bypasses _load_prequant_checkpoint.
+ import ast
+ import inspect
+
+ from core.inference import diffusion_prequant as dp
+
+ tree = ast.parse(inspect.getsource(dp.load_prequantized_transformer))
+ for node in ast.walk(tree):
+ if isinstance(node, ast.If) and "comfy_scan is not None" in ast.unparse(node.test):
+ calls = [ast.unparse(n.func) for n in ast.walk(node) if isinstance(n, ast.Call)]
+ assert "load_comfy_prequant" in calls and "_install_fp8_unit_scale" in calls
+ assert calls.index("load_comfy_prequant") < calls.index("_install_fp8_unit_scale")
+ return
+ raise AssertionError("ComfyUI branch not found")
+
+
+def test_install_tags_the_torch_compile_cache_key(monkeypatch):
+ # torch's FX graph / AOTAutograd caches key on the Dynamo graph, which is the same with or without the patch.
+ import torch.compiler.config as ccfg
+
+ monkeypatch.setattr(ccfg, "cache_key_tag", "user-tag")
+ _caps(monkeypatch, {0: (8, 9)})
+ monkeypatch.setenv(us.MIN_ROWS_ENV, "512")
+ assert us.install() is True
+ assert ccfg.cache_key_tag == f"user-tag;{us._CACHE_TAG_PREFIX}512"
+ us.uninstall()
+ assert ccfg.cache_key_tag == "user-tag"
+ monkeypatch.setenv(us.UNIT_SCALE_ENV, "1")
+ assert us.install() is True
+ assert ccfg.cache_key_tag == f"user-tag;{us._CACHE_TAG_PREFIX}512:force"
+ us.uninstall()
+ monkeypatch.setenv(us.UNIT_SCALE_ENV, "0")
+ assert us.install() is False
+ assert ccfg.cache_key_tag == "user-tag"
+
+
+_COMPILE_CHILD = r"""
+import json, sys, torch, torch.nn as nn
+from torchao.float8 import Float8MMConfig
+from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow, quantize_
+from torch._inductor.utils import run_and_get_code
+from core.inference import diffusion_fp8_unit_scale as us
+installed = us.install() if sys.argv[1] == "head" else False
+
+
+def scaled_mm_scale_args(code):
+ # Positional args 3 and 4 (scale_a, scale_b) of each generated extern _scaled_mm call, split paren-aware.
+ out = []
+ for line in code.splitlines():
+ if "extern_kernels._scaled_mm(" not in line or line.lstrip().startswith("#"):
+ continue
+ body, depth, args, cur = line.split("extern_kernels._scaled_mm(", 1)[1], 0, [], ""
+ for ch in body:
+ if ch == "," and depth == 0:
+ args.append(cur.strip()); cur = ""; continue
+ if ch == ")" and depth == 0:
+ break
+ depth += ch in "([" ; depth -= ch in ")]"
+ cur += ch
+ args.append(cur.strip())
+ out.append(args[2:4])
+ return out
+torch.manual_seed(0)
+lin = nn.Linear(256, 512, device = "cuda", dtype = torch.bfloat16)
+kw = {}
+try: # Studio pins the plain-torch kernel (diffusion_transformer_quant._make_quant_config)
+ from torchao.quantization.quantize_.common.kernel_preference import KernelPreference
+ kw["kernel_preference"] = KernelPreference.TORCH
+except Exception:
+ pass
+quantize_(lin, Float8DynamicActivationFloat8WeightConfig(
+ granularity = PerRow(), mm_config = Float8MMConfig(use_fast_accum = False), **kw))
+x = torch.randn(512, 256, device = "cuda", dtype = torch.bfloat16)
+with torch.no_grad():
+ _, codes = run_and_get_code(torch.compile(lin), x)
+calls = [a for c in codes for a in scaled_mm_scale_args(c)]
+# The unit-scale graph feeds one shared ones() scalar as both scales; the row-wise graph feeds two distinct scales.
+print(json.dumps({"installed": installed, "calls": len(calls), "unit": all(a == b for a, b in calls)}))
+"""
+
+
+def _fp8_gpu() -> bool:
+ try:
+ return torch.cuda.is_available() and torch.cuda.get_device_capability() >= (8, 9)
+ except Exception: # noqa: BLE001
+ return False
+
+
[email protected](not _fp8_gpu(), reason = "needs an fp8 _scaled_mm GPU (sm_89+)")
+def test_compiled_graph_after_a_cached_rowwise_one_uses_the_dispatcher(tmp_path):
+ # Base process compiles first into a shared inductor cache; the installed process must not reuse that graph.
+ import json
+ import os
+ import subprocess
+ import sys
+ from pathlib import Path
+
+ backend = Path(__file__).resolve().parents[1]
+ env = {
+ **os.environ,
+ "TORCHINDUCTOR_CACHE_DIR": str(tmp_path / "inductor"),
+ us.UNIT_SCALE_ENV: "1",
+ us.MIN_ROWS_ENV: "1",
+ "PYTHONPATH": os.pathsep.join(filter(None, [str(backend), os.environ.get("PYTHONPATH")])),
+ }
+ env.pop("TORCH_COMPILE_CACHE_KEY_TAG", None)
+ runs = {}
+ for arm in ("base", "head"):
+ proc = subprocess.run(
+ [sys.executable, "-c", _COMPILE_CHILD, arm],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ cwd = str(backend),
+ )
+ assert proc.returncode == 0, proc.stderr[-3000:]
+ runs[arm] = json.loads(proc.stdout.strip().splitlines()[-1])
+ assert (
+ runs["head"]["installed"] is True and runs["base"]["calls"] and runs["head"]["calls"]
+ ), runs
+ assert runs["base"]["unit"] is False, runs
+ # Without its own cache key the installed run reloads the base run's row-wise graph from the shared cache.
+ assert runs["head"]["unit"] is True, runs
+
+
+def test_installed_dispatcher_keys_studio_compile_bundles(monkeypatch):
+ # A bundle saved without the dispatcher holds artifacts torch no longer looks up once the tag is set.
+ import torch.compiler.config as ccfg
+
+ from core.inference import diffusion_compile_cache as cc
+
+ monkeypatch.setattr(ccfg, "cache_key_tag", "")
+ plain = cc.environment_fingerprint()
+ assert "torch_cache_key_tag" not in plain
+ _caps(monkeypatch, {0: (8, 9)})
+ assert us.install() is True
+ tagged = cc.environment_fingerprint()
+ assert tagged["torch_cache_key_tag"] == f"{us._CACHE_TAG_PREFIX}{us._DEFAULT_MIN_ROWS}"
+ assert cc.cache_key(plain, {}) != cc.cache_key(tagged, {})
+ us.uninstall()
+ assert cc.environment_fingerprint() == plain