Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""sm_89 fp8 unit-scale dispatch: install gating, kill switch, fallback, and the epilogue math (CPU, emulated GEMM)."""
from __future__ import annotations
import pytest
import torch
pytest.importorskip("torchao")
from torchao.quantization.quantize_.workflows.float8 import float8_tensor as f8t # noqa: E402
from core.inference import diffusion_compile_config as compile_config # noqa: E402
from core.inference import diffusion_fp8_unit_scale as us # noqa: E402
@pytest.fixture(autouse = True)
def _clean(monkeypatch):
us.uninstall()
compile_config._reset_for_tests()
monkeypatch.delenv(us.UNIT_SCALE_ENV, raising = False)
monkeypatch.delenv(us.MIN_ROWS_ENV, raising = False)
for k in us._COUNTS:
us._COUNTS[k] = 0
original = f8t.addmm_float8_unwrapped_inference
yield
us.uninstall()
compile_config._reset_for_tests()
f8t.addmm_float8_unwrapped_inference = original
def _caps(monkeypatch, caps):
monkeypatch.setattr(us, "_device_caps", lambda: dict(caps))
def test_kill_switch_never_installs(monkeypatch):
_caps(monkeypatch, {0: (8, 9)})
monkeypatch.setenv(us.UNIT_SCALE_ENV, "0")
before = f8t.addmm_float8_unwrapped_inference
assert us.install() is False
assert f8t.addmm_float8_unwrapped_inference is before
@pytest.mark.parametrize("cap", [(8, 0), (8, 6), (9, 0), (10, 0), (12, 0)])
def test_auto_leaves_other_arches_untouched(monkeypatch, cap):
_caps(monkeypatch, {0: cap})
before = f8t.addmm_float8_unwrapped_inference
assert us.install() is False
assert f8t.addmm_float8_unwrapped_inference is before
assert us.stats()["installed"] is False
def test_auto_installs_on_ada_idempotent_and_reversible(monkeypatch):
_caps(monkeypatch, {0: (8, 9)})
before = f8t.addmm_float8_unwrapped_inference
assert us.install() is True
wrapped = f8t.addmm_float8_unwrapped_inference
assert wrapped is not before and wrapped._unsloth_orig is before
assert us.install() is True
assert f8t.addmm_float8_unwrapped_inference is wrapped
us.uninstall()
assert f8t.addmm_float8_unwrapped_inference is before
def _operands(
m,
k = 64,
n = 48,
seed = 0,
):
g = torch.Generator().manual_seed(seed)
a = (torch.randn(m, k, generator = g) * 3).to(torch.float8_e4m3fn)
b = (torch.randn(n, k, generator = g) * 3).to(torch.float8_e4m3fn).t()
sa = torch.rand(m, 1, generator = g) + 0.1
sb = torch.rand(1, n, generator = g) + 0.1
return a, sa, b, sb
def test_non_cuda_calls_fall_back_to_torchao(monkeypatch):
_caps(monkeypatch, {0: (8, 9)})
seen = []
monkeypatch.setattr(
f8t, "addmm_float8_unwrapped_inference", lambda *a, **k: seen.append((a, k)) or "orig"
)
assert us.install() is True
a, sa, b, sb = _operands(2048)
out = f8t.addmm_float8_unwrapped_inference(a, sa, b, sb, torch.bfloat16, use_fast_accum = True)
assert out == "orig" and len(seen) == 1 and seen[0][1]["use_fast_accum"] is True
assert us.stats()["unit_scale"] == 0 and us.stats()["rowwise"] == 1
class _FakeDevice:
type = "cuda"
index = 0
class _Dev:
def __init__(self, t):
self.t = t
self.device = _FakeDevice()
self.shape = t.shape
def dim(self):
return self.t.dim()
def _emulated_scaled_mm(
a,
b,
scale_a,
scale_b,
out_dtype,
use_fast_accum = False,
**_,
):
a = a.t if isinstance(a, _Dev) else a
acc = (a.float() * scale_a.float()) @ (b.float() * scale_b.float())
return acc.to(out_dtype)
@pytest.mark.parametrize("bias", [False, True])
def test_epilogue_matches_rowwise_math(monkeypatch, bias):
_caps(monkeypatch, {0: (8, 9)})
rowwise_calls = []
def rowwise(
a,
sa,
b,
sb,
output_dtype,
output_scale = None,
bias = None,
use_fast_accum = False,
):
rowwise_calls.append(a.shape[0])
a = a.t if isinstance(a, _Dev) else a
out = (a.float() @ b.float()) * sa * sb
return (out + bias.float() if bias is not None else out).to(output_dtype)
monkeypatch.setattr(f8t, "addmm_float8_unwrapped_inference", rowwise)
monkeypatch.setattr(torch, "_scaled_mm", _emulated_scaled_mm)
monkeypatch.setattr(
torch, "ones", lambda *a, device = None, dtype = None: torch.tensor(1.0, dtype = dtype)
)
monkeypatch.setenv(us.MIN_ROWS_ENV, "1024")
assert us.install() is True
fn = f8t.addmm_float8_unwrapped_inference
a, sa, b, sb = _operands(2048)
bv = torch.randn(48).to(torch.bfloat16) if bias else None
got = fn(_Dev(a), sa, b, sb, torch.bfloat16, bias = bv)
want = rowwise(a, sa, b, sb, torch.bfloat16, bias = bv)
assert us.stats()["unit_scale"] == 1
# One extra bf16 rounding of the unscaled accumulator (bias adds after it, so measure against the output range).
err = (got.float() - want.float()).abs().max()
assert err <= 2**-7 * want.float().abs().max()
assert torch.isfinite(got).all()
small = _operands(256)
fn(_Dev(a), sa, b, sb, torch.bfloat16, use_fast_accum = True)
fn(_Dev(small[0]), small[1], small[2], small[3], torch.bfloat16)
fn(_Dev(a), sa, b, sb, torch.float32)
fn(_Dev(a), sa, b, sb, torch.bfloat16, output_scale = torch.tensor(1.0))
assert us.stats()["unit_scale"] == 1 and len(rowwise_calls) == 5
def test_min_rows_env(monkeypatch):
monkeypatch.setenv(us.MIN_ROWS_ENV, "4096")
assert us._min_rows() == 4096
monkeypatch.setenv(us.MIN_ROWS_ENV, "junk")
assert us._min_rows() == us._DEFAULT_MIN_ROWS
@pytest.mark.parametrize("container", ["pickle", "safetensors"])
def test_every_prequant_container_installs_the_dispatch(monkeypatch, container):
# The pickle container and the safetensors rebuild-off path never reach prequant_native's torchao API.
from core.inference import diffusion_fp8_unit_scale as us
from core.inference import diffusion_prequant as dp
from core.inference import prequant_safetensors as ps
calls = []
monkeypatch.setattr(us, "install", lambda logger = None, **kw: calls.append(container) or True)
monkeypatch.setattr(ps, "is_safetensors_checkpoint", lambda path: container == "safetensors")
monkeypatch.setattr(ps, "load_prequant_safetensors", lambda path, mmap = False: {"format": "fp8"})
monkeypatch.setattr(dp, "_torch_load_prequant", lambda path, **kwargs: {"format": "fp8"})
assert dp._load_prequant_checkpoint("model.bin", map_location = "cpu") == {"format": "fp8"}
assert calls == [container]
def _install_emulated(monkeypatch, scaled_mm = _emulated_scaled_mm):
_caps(monkeypatch, {0: (8, 9)})
monkeypatch.setattr(f8t, "addmm_float8_unwrapped_inference", lambda *a, **k: "orig")
monkeypatch.setattr(torch, "_scaled_mm", scaled_mm)
monkeypatch.setattr(
torch, "ones", lambda *a, device = None, dtype = None: torch.tensor(1.0, dtype = dtype)
)
assert us.install() is True
return f8t.addmm_float8_unwrapped_inference
def test_counters_stay_untouched_while_compiling(monkeypatch):
# A module-global mutated inside a traced region would make Dynamo guard on it and recompile every call.
fn = _install_emulated(monkeypatch)
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
a, sa, b, sb = _operands(2048)
fn(_Dev(a), sa, b, sb, torch.bfloat16)
fn(_Dev(a), sa, b, sb, torch.bfloat16, use_fast_accum = True)
assert us.stats()["unit_scale"] == 0 and us.stats()["rowwise"] == 0
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: False)
fn(_Dev(a), sa, b, sb, torch.bfloat16)
assert us.stats()["unit_scale"] == 1
def test_legacy_scaled_mm_tuple_is_unwrapped(monkeypatch):
def legacy(*args, **kwargs):
return _emulated_scaled_mm(*args, **kwargs), torch.tensor(0.0)
fn = _install_emulated(monkeypatch, legacy)
a, sa, b, sb = _operands(2048)
out = fn(_Dev(a), sa, b, sb, torch.bfloat16)
assert isinstance(out, torch.Tensor) and out.shape == (2048, 48)
def test_unit_scale_switches_key_the_compile_cache(monkeypatch):
from core.inference import diffusion_compile_cache as cc
assert cc.graph_switches() == {}
monkeypatch.setenv(us.UNIT_SCALE_ENV, "0")
monkeypatch.setenv(us.MIN_ROWS_ENV, "1024")
assert cc.graph_switches() == {us.UNIT_SCALE_ENV: "off", us.MIN_ROWS_ENV: "1024"}
monkeypatch.setenv(us.MIN_ROWS_ENV, "256")
assert us.MIN_ROWS_ENV not in cc.graph_switches()
# Keyed on the value _min_rows() uses: 0 runs as 1, so it must not collide with the 256 default.
for raw, keyed in (("0", "1"), ("1", "1"), ("junk", None)):
monkeypatch.setenv(us.MIN_ROWS_ENV, raw)
assert cc.graph_switches().get(us.MIN_ROWS_ENV) == keyed
assert us._min_rows() == (int(keyed) if keyed else us._DEFAULT_MIN_ROWS)
def test_comfy_prequant_load_installs_the_dispatch():
# The ComfyUI branch of load_prequantized_transformer bypasses _load_prequant_checkpoint.
import ast
import inspect
from core.inference import diffusion_prequant as dp
tree = ast.parse(inspect.getsource(dp.load_prequantized_transformer))
for node in ast.walk(tree):
if isinstance(node, ast.If) and "comfy_scan is not None" in ast.unparse(node.test):
calls = [ast.unparse(n.func) for n in ast.walk(node) if isinstance(n, ast.Call)]
assert "load_comfy_prequant" in calls and "_install_fp8_unit_scale" in calls
assert calls.index("load_comfy_prequant") < calls.index("_install_fp8_unit_scale")
return
raise AssertionError("ComfyUI branch not found")
def test_install_tags_the_torch_compile_cache_key(monkeypatch):
# torch's FX graph / AOTAutograd caches key on the Dynamo graph, which is the same with or without the patch.
import torch.compiler.config as ccfg
monkeypatch.setattr(ccfg, "cache_key_tag", "user-tag")
_caps(monkeypatch, {0: (8, 9)})
monkeypatch.setenv(us.MIN_ROWS_ENV, "512")
assert us.install() is True
assert ccfg.cache_key_tag == f"user-tag;{us._CACHE_TAG_PREFIX}512"
us.uninstall()
assert ccfg.cache_key_tag == "user-tag"
monkeypatch.setenv(us.UNIT_SCALE_ENV, "1")
assert us.install() is True
assert ccfg.cache_key_tag == f"user-tag;{us._CACHE_TAG_PREFIX}512:force"
us.uninstall()
monkeypatch.setenv(us.UNIT_SCALE_ENV, "0")
assert us.install() is False
assert ccfg.cache_key_tag == "user-tag"
_COMPILE_CHILD = r"""
import json, sys, torch, torch.nn as nn
from torchao.float8 import Float8MMConfig
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow, quantize_
from torch._inductor.utils import run_and_get_code
from core.inference import diffusion_fp8_unit_scale as us
installed = us.install() if sys.argv[1] == "head" else False
def scaled_mm_scale_args(code):
# Positional args 3 and 4 (scale_a, scale_b) of each generated extern _scaled_mm call, split paren-aware.
out = []
for line in code.splitlines():
if "extern_kernels._scaled_mm(" not in line or line.lstrip().startswith("#"):
continue
body, depth, args, cur = line.split("extern_kernels._scaled_mm(", 1)[1], 0, [], ""
for ch in body:
if ch == "," and depth == 0:
args.append(cur.strip()); cur = ""; continue
if ch == ")" and depth == 0:
break
depth += ch in "([" ; depth -= ch in ")]"
cur += ch
args.append(cur.strip())
out.append(args[2:4])
return out
torch.manual_seed(0)
lin = nn.Linear(256, 512, device = "cuda", dtype = torch.bfloat16)
kw = {}
try: # Studio pins the plain-torch kernel (diffusion_transformer_quant._make_quant_config)
from torchao.quantization.quantize_.common.kernel_preference import KernelPreference
kw["kernel_preference"] = KernelPreference.TORCH
except Exception:
pass
quantize_(lin, Float8DynamicActivationFloat8WeightConfig(
granularity = PerRow(), mm_config = Float8MMConfig(use_fast_accum = False), **kw))
x = torch.randn(512, 256, device = "cuda", dtype = torch.bfloat16)
with torch.no_grad():
_, codes = run_and_get_code(torch.compile(lin), x)
calls = [a for c in codes for a in scaled_mm_scale_args(c)]
# The unit-scale graph feeds one shared ones() scalar as both scales; the row-wise graph feeds two distinct scales.
print(json.dumps({"installed": installed, "calls": len(calls), "unit": all(a == b for a, b in calls)}))
"""
def _fp8_gpu() -> bool:
try:
# ROCm capability is a gfx version, and Studio never runs torchao fp8 on ROCm.
if getattr(torch.version, "hip", None):
return False
return torch.cuda.is_available() and torch.cuda.get_device_capability() >= (8, 9)
except Exception: # noqa: BLE001
return False
@pytest.mark.skipif(not _fp8_gpu(), reason = "needs an fp8 _scaled_mm GPU (sm_89+)")
def test_compiled_graph_after_a_cached_rowwise_one_uses_the_dispatcher(tmp_path):
# Base process compiles first into a shared inductor cache; the installed process must not reuse that graph.
import json
import os
import subprocess
import sys
from pathlib import Path
backend = Path(__file__).resolve().parents[1]
env = {
**os.environ,
"TORCHINDUCTOR_CACHE_DIR": str(tmp_path / "inductor"),
us.UNIT_SCALE_ENV: "1",
us.MIN_ROWS_ENV: "1",
"PYTHONPATH": os.pathsep.join(filter(None, [str(backend), os.environ.get("PYTHONPATH")])),
}
env.pop("TORCH_COMPILE_CACHE_KEY_TAG", None)
runs = {}
for arm in ("base", "head"):
proc = subprocess.run(
[sys.executable, "-c", _COMPILE_CHILD, arm],
env = env,
capture_output = True,
text = True,
timeout = 600,
cwd = str(backend),
)
assert proc.returncode == 0, proc.stderr[-3000:]
runs[arm] = json.loads(proc.stdout.strip().splitlines()[-1])
assert (
runs["head"]["installed"] is True and runs["base"]["calls"] and runs["head"]["calls"]
), runs
assert runs["base"]["unit"] is False, runs
# Without its own cache key the installed run reloads the base run's row-wise graph from the shared cache.
assert runs["head"]["unit"] is True, runs
def test_installed_dispatcher_keys_studio_compile_bundles(monkeypatch):
# A bundle saved without the dispatcher holds artifacts torch no longer looks up once the tag is set.
import torch.compiler.config as ccfg
from core.inference import diffusion_compile_cache as cc
monkeypatch.setattr(ccfg, "cache_key_tag", "")
plain = cc.environment_fingerprint()
assert "torch_cache_key_tag" not in plain
_caps(monkeypatch, {0: (8, 9)})
assert us.install() is True
tagged = cc.environment_fingerprint()
assert tagged["torch_cache_key_tag"] == f"{us._CACHE_TAG_PREFIX}{us._DEFAULT_MIN_ROWS}"
assert cc.cache_key(plain, {}) != cc.cache_key(tagged, {})
us.uninstall()
assert cc.environment_fingerprint() == plain
def test_auto_never_installs_for_fast_accumulate(monkeypatch):
# Installing at fast accumulate would only split every later compile's cache key.
import torch.compiler.config as ccfg
monkeypatch.setattr(ccfg, "cache_key_tag", "")
_caps(monkeypatch, {0: (8, 9)})
before = f8t.addmm_float8_unwrapped_inference
assert us.install(fast_accum = True) is False
assert f8t.addmm_float8_unwrapped_inference is before and ccfg.cache_key_tag == ""
assert us.install(fast_accum = False) is True
us.uninstall()
assert us.install(fast_accum = None) is True
us.uninstall()
monkeypatch.setenv(us.UNIT_SCALE_ENV, "1")
assert us.install(fast_accum = True) is True
def test_fp8_quant_config_installs_only_for_precise_accumulate(monkeypatch):
from core.inference import diffusion_transformer_quant as tq
_caps(monkeypatch, {0: (8, 9)})
tq._make_quant_config(tq.TQ_FP8)
assert us.stats()["installed"] is False
tq._make_quant_config(tq.TQ_FP8, fast_accum = True)
assert us.stats()["installed"] is False
tq._make_quant_config(tq.TQ_FP8, fast_accum = False)
assert us.stats()["installed"] is True
@pytest.mark.parametrize(
"metadata, installs",
[
({"scheme": "int8"}, False),
({"scheme": "fp8", "fast_accum": True}, False),
({"scheme": "fp8", "fast_accum": False}, True),
({"scheme": "nvfp4"}, True),
({}, True),
],
)
def test_prequant_checkpoint_installs_only_when_its_fp8_layers_can_engage(
monkeypatch, metadata, installs
):
from core.inference import diffusion_prequant as dp
from core.inference import prequant_safetensors as ps
_caps(monkeypatch, {0: (8, 9)})
ckpt = {"format": dp.PREQUANT_FORMAT, "state_dict": {}, "metadata": metadata}
monkeypatch.setattr(ps, "is_safetensors_checkpoint", lambda path: True)
monkeypatch.setattr(ps, "load_prequant_safetensors", lambda path, mmap = False: ckpt)
assert dp._load_prequant_checkpoint("model.safetensors") is ckpt
assert us.stats()["installed"] is installs
def test_every_off_word_of_the_compile_cache_key_turns_the_dispatcher_off(monkeypatch):
from core.inference import diffusion_compile_cache as cc
_caps(monkeypatch, {0: (8, 9)})
for word in cc._OFF_WORDS:
monkeypatch.setenv(us.UNIT_SCALE_ENV, word.upper())
assert cc.graph_switches().get(us.UNIT_SCALE_ENV) == "off"
assert us._mode() == "off" and us.install(fast_accum = False) is False
for word in cc._ON_WORDS:
monkeypatch.setenv(us.UNIT_SCALE_ENV, word)
assert us._mode() == "force"
@pytest.mark.parametrize("bias", [False, True])
def test_eager_epilogue_writes_into_the_accumulator_bitwise_as_before(monkeypatch, bias):
produced = []
def scaled_mm(*args, **kwargs):
acc = _emulated_scaled_mm(*args, **kwargs)
produced.append(acc)
return acc
fn = _install_emulated(monkeypatch, scaled_mm)
monkeypatch.setattr(us, "_EAGER_EPILOGUE_ELEMS", 48 * 300)
a, sa, b, sb = _operands(2048)
bv = torch.randn(48).to(torch.bfloat16) if bias else None
got = fn(_Dev(a), sa, b, sb, torch.bfloat16, bias = bv)
acc = produced[-1]
assert got.data_ptr() == acc.data_ptr()
unit = _emulated_scaled_mm(a, b, torch.tensor(1.0), torch.tensor(1.0), torch.bfloat16)
want = unit * sa.float()
want.mul_(sb.float())
if bv is not None:
want.add_(bv.float())
assert torch.equal(got, want.to(torch.bfloat16))