Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run Ada FP8 precise accumulate through a unit-scale GEMM plus a fused scale epilogue (#13793)
Daniel Hanedb8e677704f
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex 81db420ed1e..5298b404b1c 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -280,6 +280,12 @@ def environment_fingerprint() -> dict[str, Any]:fp["gpu_name"] = torch.cuda.get_device_name(index)cap = torch.cuda.get_device_capability(index)fp["gpu_capability"] = f"sm_{cap[0]}{cap[1]}"+ # torch keys artifacts on this tag (set by diffusion_fp8_unit_scale); a bundle from another tag would load,+ # miss, recompile and never be rewritten. Only a set tag joins the key, so untagged bundles stay valid.+ tag = getattr(getattr(torch, "compiler", None), "config", None)+ tag = getattr(tag, "cache_key_tag", None)+ if isinstance(tag, str) and tag:+ fp["torch_cache_key_tag"] = tagexcept Exception: # noqa: BLE001 - best-effortpassreturn fp@@ -304,6 +310,8 @@ _GRAPH_SWITCHES = {"UNSLOTH_STATIC_STREAM_MERGE_DETECT": "off","UNSLOTH_TORCHAO_INDUCTOR_CONFIG": "off","UNSLOTH_NVFP4_BACKEND": "raw:auto",+ "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE": "auto",+ "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE_MIN_ROWS": "rows:256",# Unset is off; "auto" protects the head + last step, so "auto" is NOT the default here."UNSLOTH_NVFP4_PROTECT_STEPS": "raw",}@@ -316,6 +324,13 @@ def _switch_value(name: str, default: str) -> Optional[str]:raw = (os.environ.get(name) or "").strip().lower()if not raw:return None+ if default.startswith("rows:"):+ # A row threshold keys on the value the reader uses (max(1, int)); junk falls back to the default there.+ try:+ rows = str(max(1, int(raw)))+ except ValueError:+ return None+ return None if rows == default.partition(":")[2] else rowsif default.startswith("raw"):return None if raw in _OFF_WORDS or raw == default.partition(":")[2] else rawvalue = "off" if raw in _OFF_WORDS else "on" if raw in _ON_WORDS else Nonediff --git a/studio/backend/core/inference/diffusion_fp8_unit_scale.py b/studio/backend/core/inference/diffusion_fp8_unit_scale.pynew file mode 100644index 00000000000..919f0db5add--- /dev/null+++ b/studio/backend/core/inference/diffusion_fp8_unit_scale.py@@ -0,0 +1,205 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""sm_89 precise-accumulate fp8 GEMM: unit-scale ``torch._scaled_mm`` + fp32 row x column scale epilogue.++On sm_89 torchao's row-wise ``_scaled_mm`` hits a slow CUTLASS stream-K kernel. L4, torch 2.14.1, 1024 px linears:+* precise accumulate: row-wise 29-32 TFLOP/s, per-tensor 102-134 TFLOP/s; end to end klein-4B 1.034 -> 0.480 s/step,+ Z-Image-Turbo 1.708 -> 0.772 s/step.+* fast accumulate: row-wise 85-108, per-tensor 91-127 TFLOP/s, no end-to-end gain, so left on torchao (bit-identical).+Only difference vs row-wise: one bf16 rounding of the unscaled accumulator (relative error <= 2^-9).++``UNSLOTH_DIFFUSION_FP8_UNIT_SCALE``: ``0`` off, ``1`` every arch + both accumulate modes (testing), else auto.+"""++from __future__ import annotations++import os+from typing import Any, Optional++UNIT_SCALE_ENV = "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE"+MIN_ROWS_ENV = "UNSLOTH_DIFFUSION_FP8_UNIT_SCALE_MIN_ROWS"+# Unit-scale wins from M = 256 up (L4: row-wise 2.5-4x slower from 512); below, tens of microseconds either way.+_DEFAULT_MIN_ROWS = 256+_ADA_CAPS = ((8, 9),)++_COUNTS = {"unit_scale": 0, "rowwise": 0}+_STATE: dict = {"installed": False, "orig": None, "caps": {}}+++def _mode() -> str:+ v = (os.environ.get(UNIT_SCALE_ENV) or "").strip().lower()+ if v in ("0", "off", "false", "no"):+ return "off"+ if v in ("1", "on", "true", "yes", "force"):+ return "force"+ return "auto"+++def _min_rows() -> int:+ try:+ return max(1, int(os.environ.get(MIN_ROWS_ENV, "") or _DEFAULT_MIN_ROWS))+ except ValueError:+ return _DEFAULT_MIN_ROWS+++def _compiling() -> bool:+ # Counters are eager-only: a module-global mutated inside a traced region makes Dynamo guard on it and recompile.+ try:+ import torch+ return bool(torch.compiler.is_compiling())+ except Exception: # noqa: BLE001 - no torch.compiler: nothing is being traced+ return False+++def stats() -> dict:+ return {**_COUNTS, "installed": bool(_STATE["installed"])}+++_CACHE_TAG_PREFIX = "unsloth_fp8_unit_scale="+++def _set_compile_cache_tag(tag: Optional[str]) -> None:+ """Key torch's FX/AOTAutograd caches on the dispatcher: the Dynamo graph is identical with or without the patch,+ so untagged, a graph cached on the other side of install / the kill switch is reused. ``None`` removes our tag."""+ try:+ import torch.compiler.config as _cc+ except Exception: # noqa: BLE001 - torch without compiler config: no persistent compile cache to split+ return+ current = getattr(_cc, "cache_key_tag", None)+ if not isinstance(current, str):+ return+ parts = [p for p in current.split(";") if p and not p.startswith(_CACHE_TAG_PREFIX)]+ if tag:+ parts.append(_CACHE_TAG_PREFIX + tag)+ try:+ _cc.cache_key_tag = ";".join(parts)+ except Exception: # noqa: BLE001+ pass+++def _device_caps() -> dict:+ import torch++ caps = {}+ try:+ if torch.cuda.is_available():+ for i in range(torch.cuda.device_count()):+ caps[i] = tuple(torch.cuda.get_device_capability(i))+ except Exception: # noqa: BLE001 - no CUDA runtime: nothing qualifies+ return {}+ return caps+++def install(logger: Any = None) -> bool:+ """Patch torchao's fp8 addmm with the dispatcher; idempotent, False when off / no qualifying GPU / layout moved."""+ if _STATE["installed"]:+ return True+ mode = _mode()+ if mode == "off":+ return False+ caps = _device_caps()+ if mode == "auto" and not any(c in _ADA_CAPS for c in caps.values()):+ return False+ try:+ from torchao.quantization.quantize_.workflows.float8 import float8_tensor as _f8t+ except Exception: # noqa: BLE001 - torchao absent / stubbed / moved+ return False+ orig = getattr(_f8t, "addmm_float8_unwrapped_inference", None)+ if orig is None or not callable(orig):+ return False+ if getattr(orig, "_unsloth_unit_scale", False):+ _STATE.update(installed = True)+ return True+ min_rows = _min_rows()+ force = mode == "force"+ qualifying = frozenset(i for i, c in caps.items() if force or c in _ADA_CAPS)++ import torch++ def addmm_float8_unwrapped_inference(+ a_data: Any,+ a_scale: Any,+ b_data: Any,+ b_scale: Any,+ output_dtype: Any,+ output_scale: Optional[Any] = None,+ bias: Optional[Any] = None,+ use_fast_accum: bool = False,+ ) -> Any:+ if (+ (force or not use_fast_accum)+ and output_scale is None+ and output_dtype == torch.bfloat16+ and a_data.device.type == "cuda"+ and a_data.device.index in qualifying+ and a_data.dim() == 2+ and b_data.dim() == 2+ and a_scale.dim() == 2+ and b_scale.dim() == 2+ and a_scale.shape[1] == 1+ and b_scale.shape[0] == 1+ and a_scale.shape[0] == a_data.shape[0]+ and b_scale.shape[1] == b_data.shape[1]+ and a_data.shape[0] >= min_rows+ ):+ if not _compiling():+ _COUNTS["unit_scale"] += 1+ one = torch.ones((), device = a_data.device, dtype = torch.float32)+ acc = torch._scaled_mm(+ a_data,+ b_data,+ scale_a = one,+ scale_b = one,+ out_dtype = torch.bfloat16,+ use_fast_accum = use_fast_accum,+ )+ if isinstance(acc, tuple): # torch < 2.5 returns (out, amax)+ acc = acc[0]+ # bf16 * fp32 promotes without an upcast copy; in place saves an fp32 [M, N] buffer eagerly, same result.+ out = acc * a_scale.float()+ out.mul_(b_scale.float())+ if bias is not None:+ out.add_(bias.float())+ return out.to(output_dtype)+ if not _compiling():+ _COUNTS["rowwise"] += 1+ return orig(+ a_data,+ a_scale,+ b_data,+ b_scale,+ output_dtype,+ output_scale = output_scale,+ bias = bias,+ use_fast_accum = use_fast_accum,+ )++ addmm_float8_unwrapped_inference._unsloth_unit_scale = True # type: ignore[attr-defined]+ addmm_float8_unwrapped_inference._unsloth_orig = orig # type: ignore[attr-defined]+ _f8t.addmm_float8_unwrapped_inference = addmm_float8_unwrapped_inference+ _STATE.update(installed = True, orig = orig, caps = caps)+ _set_compile_cache_tag(f"{min_rows}{':force' if force else ''}")+ if logger is not None:+ try:+ logger.info(+ "fp8: unit-scale GEMM + row/column scale epilogue on %s (rows >= %d, %s=0 disables)",+ "every CUDA device" if force else "sm_89 precise accumulate",+ min_rows,+ UNIT_SCALE_ENV,+ )+ except Exception: # noqa: BLE001+ pass+ return True+++def uninstall() -> None:+ if not _STATE["installed"]:+ return+ try:+ from torchao.quantization.quantize_.workflows.float8 import float8_tensor as _f8t+ if _STATE["orig"] is not None:+ _f8t.addmm_float8_unwrapped_inference = _STATE["orig"]+ finally:+ _STATE.update(installed = False, orig = None, caps = {})+ _set_compile_cache_tag(None)diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex dd5a284a819..acb65d36a07 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -346,8 +346,19 @@ def _load_prequant_checkpoint(path: str, **kwargs: Any) -> Any:from .prequant_safetensors import is_safetensors_checkpoint, load_prequant_safetensorsif is_safetensors_checkpoint(path):- return load_prequant_safetensors(path, mmap = bool(kwargs.get("mmap")))- return _torch_load_prequant(path, **kwargs)+ ckpt = load_prequant_safetensors(path, mmap = bool(kwargs.get("mmap")))+ else:+ ckpt = _torch_load_prequant(path, **kwargs)+ _install_fp8_unit_scale()+ return ckpt+++def _install_fp8_unit_scale() -> None:+ try:+ from .diffusion_fp8_unit_scale import install+ install()+ except Exception: # noqa: BLE001 - never block a load on a speed path+ pass# 0 reads a pickle checkpoint headed for an accelerator into host memory instead of mapping it.@@ -2381,6 +2392,8 @@ def load_prequantized_transformer(config_subfolder = config_subfolder,logger = logger,)+ # ComfyUI files never pass through _load_prequant_checkpoint, which installs this for Studio's own.+ _install_fp8_unit_scale()from .diffusion_convrot import declares_rotation, is_rotated_linear, warm_rotation_cachecomfy_rotated = any(is_rotated_linear(m) for m in transformer.modules())diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex a4c131f99a5..ad1ff8e1c08 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -1736,6 +1736,16 @@ def _quiet_config(cls: Any, **kwargs: Any) -> Any:return cls(**kwargs)+def _install_fp8_unit_scale() -> None:+ try:+ import logging++ from .diffusion_fp8_unit_scale import install+ install(logger = logging.getLogger(__name__))+ except Exception: # noqa: BLE001 - never block a load on a speed path+ pass++def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:"""The torchao dynamic-activation config for ``scheme`` (lazy imports per branch).``fast_accum`` applies to fp8 only: None auto-detects by GPU class, True/False force it."""@@ -1755,6 +1765,7 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:# scale 0, NaN qdata, black frames.from torchao.quantization import PerRow+ _install_fp8_unit_scale()fp8_kwargs: dict = {"granularity": PerRow()}config_params = _inspect.signature(Float8DynamicActivationFloat8WeightConfig).parametersif "activation_value_lb" in config_params:diff --git a/studio/backend/tests/test_diffusion_compile_cache_variant.py b/studio/backend/tests/test_diffusion_compile_cache_variant.pyindex 2b1f8070bba..3ec4ca2ae7d 100644--- a/studio/backend/tests/test_diffusion_compile_cache_variant.py+++ b/studio/backend/tests/test_diffusion_compile_cache_variant.py@@ -125,14 +125,21 @@ def test_an_instance_forward_swap_keys_apart():def test_graph_switches_key_apart_only_off_their_default(monkeypatch, name, default, quant):m = _int8() if quant else _model()k0 = _key(m, quant)+ rows = default.partition(":")[2] if default.startswith("rows:") else Nonefor same in (("",)+ (() if default == "raw" else ("auto",))+ ((default,) if default in ("on", "off") else ())+ + ((rows,) if rows else ())):monkeypatch.setenv(name, same)assert _key(m, quant) == k0, same- flipped = {"on": "0", "off": "1", "auto": "0", "raw": "torchao", "raw:auto": "torchao"}[default]+ if rows:+ flipped = "1"+ else:+ flipped = {"on": "0", "off": "1", "auto": "0", "raw": "torchao", "raw:auto": "torchao"}[+ default+ ]monkeypatch.setenv(name, flipped)assert _key(m, quant) != k0assert name in cc.model_fingerprint(transformer = m, quant = quant, **_FP_KW)["switches"]diff --git a/studio/backend/tests/test_diffusion_fp8_unit_scale.py b/studio/backend/tests/test_diffusion_fp8_unit_scale.pynew file mode 100644index 00000000000..0b88aca1f4c--- /dev/null+++ b/studio/backend/tests/test_diffusion_fp8_unit_scale.py@@ -0,0 +1,380 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""sm_89 fp8 unit-scale dispatch: install gating, kill switch, fallback, and the epilogue math (CPU, emulated GEMM)."""++from __future__ import annotations++import pytest+import torch++pytest.importorskip("torchao")+from torchao.quantization.quantize_.workflows.float8 import float8_tensor as f8t # noqa: E402++from core.inference import diffusion_fp8_unit_scale as us # noqa: E402++[email protected](autouse = True)+def _clean(monkeypatch):+ us.uninstall()+ monkeypatch.delenv(us.UNIT_SCALE_ENV, raising = False)+ monkeypatch.delenv(us.MIN_ROWS_ENV, raising = False)+ for k in us._COUNTS:+ us._COUNTS[k] = 0+ original = f8t.addmm_float8_unwrapped_inference+ yield+ us.uninstall()+ f8t.addmm_float8_unwrapped_inference = original+++def _caps(monkeypatch, caps):+ monkeypatch.setattr(us, "_device_caps", lambda: dict(caps))+++def test_kill_switch_never_installs(monkeypatch):+ _caps(monkeypatch, {0: (8, 9)})+ monkeypatch.setenv(us.UNIT_SCALE_ENV, "0")+ before = f8t.addmm_float8_unwrapped_inference+ assert us.install() is False+ assert f8t.addmm_float8_unwrapped_inference is before++[email protected]("cap", [(8, 0), (8, 6), (9, 0), (10, 0), (12, 0)])+def test_auto_leaves_other_arches_untouched(monkeypatch, cap):+ _caps(monkeypatch, {0: cap})+ before = f8t.addmm_float8_unwrapped_inference+ assert us.install() is False+ assert f8t.addmm_float8_unwrapped_inference is before+ assert us.stats()["installed"] is False+++def test_auto_installs_on_ada_idempotent_and_reversible(monkeypatch):+ _caps(monkeypatch, {0: (8, 9)})+ before = f8t.addmm_float8_unwrapped_inference+ assert us.install() is True+ wrapped = f8t.addmm_float8_unwrapped_inference+ assert wrapped is not before and wrapped._unsloth_orig is before+ assert us.install() is True+ assert f8t.addmm_float8_unwrapped_inference is wrapped+ us.uninstall()+ assert f8t.addmm_float8_unwrapped_inference is before+++def _operands(+ m,+ k = 64,+ n = 48,+ seed = 0,+):+ g = torch.Generator().manual_seed(seed)+ a = (torch.randn(m, k, generator = g) * 3).to(torch.float8_e4m3fn)+ b = (torch.randn(n, k, generator = g) * 3).to(torch.float8_e4m3fn).t()+ sa = torch.rand(m, 1, generator = g) + 0.1+ sb = torch.rand(1, n, generator = g) + 0.1+ return a, sa, b, sb+++def test_non_cuda_calls_fall_back_to_torchao(monkeypatch):+ _caps(monkeypatch, {0: (8, 9)})+ seen = []+ monkeypatch.setattr(+ f8t, "addmm_float8_unwrapped_inference", lambda *a, **k: seen.append((a, k)) or "orig"+ )+ assert us.install() is True+ a, sa, b, sb = _operands(2048)+ out = f8t.addmm_float8_unwrapped_inference(a, sa, b, sb, torch.bfloat16, use_fast_accum = True)+ assert out == "orig" and len(seen) == 1 and seen[0][1]["use_fast_accum"] is True+ assert us.stats()["unit_scale"] == 0 and us.stats()["rowwise"] == 1+++class _FakeDevice:+ type = "cuda"+ index = 0+++class _Dev:+ def __init__(self, t):+ self.t = t+ self.device = _FakeDevice()+ self.shape = t.shape++ def dim(self):+ return self.t.dim()+++def _emulated_scaled_mm(+ a,+ b,+ scale_a,+ scale_b,+ out_dtype,+ use_fast_accum = False,+ **_,+):+ a = a.t if isinstance(a, _Dev) else a+ acc = (a.float() * scale_a.float()) @ (b.float() * scale_b.float())+ return acc.to(out_dtype)++[email protected]("bias", [False, True])+def test_epilogue_matches_rowwise_math(monkeypatch, bias):+ _caps(monkeypatch, {0: (8, 9)})+ rowwise_calls = []++ def rowwise(+ a,+ sa,+ b,+ sb,+ output_dtype,+ output_scale = None,+ bias = None,+ use_fast_accum = False,+ ):+ rowwise_calls.append(a.shape[0])+ a = a.t if isinstance(a, _Dev) else a+ out = (a.float() @ b.float()) * sa * sb+ return (out + bias.float() if bias is not None else out).to(output_dtype)++ monkeypatch.setattr(f8t, "addmm_float8_unwrapped_inference", rowwise)+ monkeypatch.setattr(torch, "_scaled_mm", _emulated_scaled_mm)+ monkeypatch.setattr(+ torch, "ones", lambda *a, device = None, dtype = None: torch.tensor(1.0, dtype = dtype)+ )+ monkeypatch.setenv(us.MIN_ROWS_ENV, "1024")+ assert us.install() is True+ fn = f8t.addmm_float8_unwrapped_inference+ a, sa, b, sb = _operands(2048)+ bv = torch.randn(48).to(torch.bfloat16) if bias else None+ got = fn(_Dev(a), sa, b, sb, torch.bfloat16, bias = bv)+ want = rowwise(a, sa, b, sb, torch.bfloat16, bias = bv)+ assert us.stats()["unit_scale"] == 1+ # One extra bf16 rounding of the unscaled accumulator (bias adds after it, so measure against the output range).+ err = (got.float() - want.float()).abs().max()+ assert err <= 2**-7 * want.float().abs().max()+ assert torch.isfinite(got).all()++ small = _operands(256)+ fn(_Dev(a), sa, b, sb, torch.bfloat16, use_fast_accum = True)+ fn(_Dev(small[0]), small[1], small[2], small[3], torch.bfloat16)+ fn(_Dev(a), sa, b, sb, torch.float32)+ fn(_Dev(a), sa, b, sb, torch.bfloat16, output_scale = torch.tensor(1.0))+ assert us.stats()["unit_scale"] == 1 and len(rowwise_calls) == 5+++def test_min_rows_env(monkeypatch):+ monkeypatch.setenv(us.MIN_ROWS_ENV, "4096")+ assert us._min_rows() == 4096+ monkeypatch.setenv(us.MIN_ROWS_ENV, "junk")+ assert us._min_rows() == us._DEFAULT_MIN_ROWS++[email protected]("container", ["pickle", "safetensors"])+def test_every_prequant_container_installs_the_dispatch(monkeypatch, container):+ # The pickle container and the safetensors rebuild-off path never reach prequant_native's torchao API.+ from core.inference import diffusion_fp8_unit_scale as us+ from core.inference import diffusion_prequant as dp+ from core.inference import prequant_safetensors as ps++ calls = []+ monkeypatch.setattr(us, "install", lambda logger = None: calls.append(container) or True)+ monkeypatch.setattr(ps, "is_safetensors_checkpoint", lambda path: container == "safetensors")+ monkeypatch.setattr(ps, "load_prequant_safetensors", lambda path, mmap = False: {"format": "fp8"})+ monkeypatch.setattr(dp, "_torch_load_prequant", lambda path, **kwargs: {"format": "fp8"})+ assert dp._load_prequant_checkpoint("model.bin", map_location = "cpu") == {"format": "fp8"}+ assert calls == [container]+++def _install_emulated(monkeypatch, scaled_mm = _emulated_scaled_mm):+ _caps(monkeypatch, {0: (8, 9)})+ monkeypatch.setattr(f8t, "addmm_float8_unwrapped_inference", lambda *a, **k: "orig")+ monkeypatch.setattr(torch, "_scaled_mm", scaled_mm)+ monkeypatch.setattr(+ torch, "ones", lambda *a, device = None, dtype = None: torch.tensor(1.0, dtype = dtype)+ )+ assert us.install() is True+ return f8t.addmm_float8_unwrapped_inference+++def test_counters_stay_untouched_while_compiling(monkeypatch):+ # A module-global mutated inside a traced region would make Dynamo guard on it and recompile every call.+ fn = _install_emulated(monkeypatch)+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ a, sa, b, sb = _operands(2048)+ fn(_Dev(a), sa, b, sb, torch.bfloat16)+ fn(_Dev(a), sa, b, sb, torch.bfloat16, use_fast_accum = True)+ assert us.stats()["unit_scale"] == 0 and us.stats()["rowwise"] == 0+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: False)+ fn(_Dev(a), sa, b, sb, torch.bfloat16)+ assert us.stats()["unit_scale"] == 1+++def test_legacy_scaled_mm_tuple_is_unwrapped(monkeypatch):+ def legacy(*args, **kwargs):+ return _emulated_scaled_mm(*args, **kwargs), torch.tensor(0.0)++ fn = _install_emulated(monkeypatch, legacy)+ a, sa, b, sb = _operands(2048)+ out = fn(_Dev(a), sa, b, sb, torch.bfloat16)+ assert isinstance(out, torch.Tensor) and out.shape == (2048, 48)+++def test_unit_scale_switches_key_the_compile_cache(monkeypatch):+ from core.inference import diffusion_compile_cache as cc++ assert cc.graph_switches() == {}+ monkeypatch.setenv(us.UNIT_SCALE_ENV, "0")+ monkeypatch.setenv(us.MIN_ROWS_ENV, "1024")+ assert cc.graph_switches() == {us.UNIT_SCALE_ENV: "off", us.MIN_ROWS_ENV: "1024"}+ monkeypatch.setenv(us.MIN_ROWS_ENV, "256")+ assert us.MIN_ROWS_ENV not in cc.graph_switches()+ # Keyed on the value _min_rows() uses: 0 runs as 1, so it must not collide with the 256 default.+ for raw, keyed in (("0", "1"), ("1", "1"), ("junk", None)):+ monkeypatch.setenv(us.MIN_ROWS_ENV, raw)+ assert cc.graph_switches().get(us.MIN_ROWS_ENV) == keyed+ assert us._min_rows() == (int(keyed) if keyed else us._DEFAULT_MIN_ROWS)+++def test_comfy_prequant_load_installs_the_dispatch():+ # The ComfyUI branch of load_prequantized_transformer bypasses _load_prequant_checkpoint.+ import ast+ import inspect++ from core.inference import diffusion_prequant as dp++ tree = ast.parse(inspect.getsource(dp.load_prequantized_transformer))+ for node in ast.walk(tree):+ if isinstance(node, ast.If) and "comfy_scan is not None" in ast.unparse(node.test):+ calls = [ast.unparse(n.func) for n in ast.walk(node) if isinstance(n, ast.Call)]+ assert "load_comfy_prequant" in calls and "_install_fp8_unit_scale" in calls+ assert calls.index("load_comfy_prequant") < calls.index("_install_fp8_unit_scale")+ return+ raise AssertionError("ComfyUI branch not found")+++def test_install_tags_the_torch_compile_cache_key(monkeypatch):+ # torch's FX graph / AOTAutograd caches key on the Dynamo graph, which is the same with or without the patch.+ import torch.compiler.config as ccfg++ monkeypatch.setattr(ccfg, "cache_key_tag", "user-tag")+ _caps(monkeypatch, {0: (8, 9)})+ monkeypatch.setenv(us.MIN_ROWS_ENV, "512")+ assert us.install() is True+ assert ccfg.cache_key_tag == f"user-tag;{us._CACHE_TAG_PREFIX}512"+ us.uninstall()+ assert ccfg.cache_key_tag == "user-tag"+ monkeypatch.setenv(us.UNIT_SCALE_ENV, "1")+ assert us.install() is True+ assert ccfg.cache_key_tag == f"user-tag;{us._CACHE_TAG_PREFIX}512:force"+ us.uninstall()+ monkeypatch.setenv(us.UNIT_SCALE_ENV, "0")+ assert us.install() is False+ assert ccfg.cache_key_tag == "user-tag"+++_COMPILE_CHILD = r"""+import json, sys, torch, torch.nn as nn+from torchao.float8 import Float8MMConfig+from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow, quantize_+from torch._inductor.utils import run_and_get_code+from core.inference import diffusion_fp8_unit_scale as us+installed = us.install() if sys.argv[1] == "head" else False+++def scaled_mm_scale_args(code):+ # Positional args 3 and 4 (scale_a, scale_b) of each generated extern _scaled_mm call, split paren-aware.+ out = []+ for line in code.splitlines():+ if "extern_kernels._scaled_mm(" not in line or line.lstrip().startswith("#"):+ continue+ body, depth, args, cur = line.split("extern_kernels._scaled_mm(", 1)[1], 0, [], ""+ for ch in body:+ if ch == "," and depth == 0:+ args.append(cur.strip()); cur = ""; continue+ if ch == ")" and depth == 0:+ break+ depth += ch in "([" ; depth -= ch in ")]"+ cur += ch+ args.append(cur.strip())+ out.append(args[2:4])+ return out+torch.manual_seed(0)+lin = nn.Linear(256, 512, device = "cuda", dtype = torch.bfloat16)+kw = {}+try: # Studio pins the plain-torch kernel (diffusion_transformer_quant._make_quant_config)+ from torchao.quantization.quantize_.common.kernel_preference import KernelPreference+ kw["kernel_preference"] = KernelPreference.TORCH+except Exception:+ pass+quantize_(lin, Float8DynamicActivationFloat8WeightConfig(+ granularity = PerRow(), mm_config = Float8MMConfig(use_fast_accum = False), **kw))+x = torch.randn(512, 256, device = "cuda", dtype = torch.bfloat16)+with torch.no_grad():+ _, codes = run_and_get_code(torch.compile(lin), x)+calls = [a for c in codes for a in scaled_mm_scale_args(c)]+# The unit-scale graph feeds one shared ones() scalar as both scales; the row-wise graph feeds two distinct scales.+print(json.dumps({"installed": installed, "calls": len(calls), "unit": all(a == b for a, b in calls)}))+"""+++def _fp8_gpu() -> bool:+ try:+ return torch.cuda.is_available() and torch.cuda.get_device_capability() >= (8, 9)+ except Exception: # noqa: BLE001+ return False++[email protected](not _fp8_gpu(), reason = "needs an fp8 _scaled_mm GPU (sm_89+)")+def test_compiled_graph_after_a_cached_rowwise_one_uses_the_dispatcher(tmp_path):+ # Base process compiles first into a shared inductor cache; the installed process must not reuse that graph.+ import json+ import os+ import subprocess+ import sys+ from pathlib import Path++ backend = Path(__file__).resolve().parents[1]+ env = {+ **os.environ,+ "TORCHINDUCTOR_CACHE_DIR": str(tmp_path / "inductor"),+ us.UNIT_SCALE_ENV: "1",+ us.MIN_ROWS_ENV: "1",+ "PYTHONPATH": os.pathsep.join(filter(None, [str(backend), os.environ.get("PYTHONPATH")])),+ }+ env.pop("TORCH_COMPILE_CACHE_KEY_TAG", None)+ runs = {}+ for arm in ("base", "head"):+ proc = subprocess.run(+ [sys.executable, "-c", _COMPILE_CHILD, arm],+ env = env,+ capture_output = True,+ text = True,+ timeout = 600,+ cwd = str(backend),+ )+ assert proc.returncode == 0, proc.stderr[-3000:]+ runs[arm] = json.loads(proc.stdout.strip().splitlines()[-1])+ assert (+ runs["head"]["installed"] is True and runs["base"]["calls"] and runs["head"]["calls"]+ ), runs+ assert runs["base"]["unit"] is False, runs+ # Without its own cache key the installed run reloads the base run's row-wise graph from the shared cache.+ assert runs["head"]["unit"] is True, runs+++def test_installed_dispatcher_keys_studio_compile_bundles(monkeypatch):+ # A bundle saved without the dispatcher holds artifacts torch no longer looks up once the tag is set.+ import torch.compiler.config as ccfg++ from core.inference import diffusion_compile_cache as cc++ monkeypatch.setattr(ccfg, "cache_key_tag", "")+ plain = cc.environment_fingerprint()+ assert "torch_cache_key_tag" not in plain+ _caps(monkeypatch, {0: (8, 9)})+ assert us.install() is True+ tagged = cc.environment_fingerprint()+ assert tagged["torch_cache_key_tag"] == f"{us._CACHE_TAG_PREFIX}{us._DEFAULT_MIN_ROWS}"+ assert cc.cache_key(plain, {}) != cc.cache_key(tagged, {})+ us.uninstall()+ assert cc.environment_fingerprint() == plain