Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: widen stored fp8 / int8 weights without full float32 temporaries (#17068)

Daniel Han57d5fffbcf61
diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.py
index d8f6c14d8b7..36a220298f6 100644
--- a/studio/backend/core/inference/diffusion_comfy_quant.py
+++ b/studio/backend/core/inference/diffusion_comfy_quant.py
@@ -12,6 +12,7 @@ from dataclasses import dataclass, field
from typing import Any, Optional
from .diffusion_comfy_block import BLOCK_SIZES, MXFP8, NVFP4, tiled_scale_numel
+from .diffusion_fp8_dequant import chunk_rows, dequant_rows, fused_dequant_enabled
from .diffusion_single_file_converters import host_ram_budget_scope
from .diffusion_small_host import FP8_STORAGE_HOST_RESERVE_MIB
@@ -594,10 +595,23 @@ def _dequant(codes: Any, scale: Any, group: int, dtype: Any) -> Any:
from .diffusion_single_file_converters import charge_host_ram
rows = codes.shape[0]
+ out_size = torch.empty((), dtype = dtype).element_size()
+ if not group and codes.dim() == 2 and rows and fused_dequant_enabled():
+ cols = codes.shape[1]
+ step = chunk_rows(rows, cols, codes.device)
+ if step:
+ # Row chunks into the output: the output plus one float32 chunk is the peak.
+ charge_host_ram(codes.numel() * out_size + step * cols * 4)
+ return dequant_rows(
+ codes,
+ scale.to(torch.float32).reshape(-1).expand(rows),
+ dtype,
+ lambda c: c.to(torch.float32, copy = True),
+ triton_ok = False,
+ )
# Peak, not just the output: the float32 codes and their float32 product are live together (8 bytes per
# element); ConvRot's batched matmul holds the product, a float32 copy of it and its float32 result (12,
# measured); last the product and the output.
- out_size = torch.empty((), dtype = dtype).element_size()
charge_host_ram(codes.numel() * max(12 if group else 8, 4 + out_size))
weight = codes.to(torch.float32) * scale.to(torch.float32).reshape(-1, 1).expand(rows, 1)
if group:
diff --git a/studio/backend/core/inference/diffusion_fp8_dequant.py b/studio/backend/core/inference/diffusion_fp8_dequant.py
new file mode 100644
index 00000000000..91829de068f
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_fp8_dequant.py
@@ -0,0 +1,148 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Eager widening of int8 / stored-fp8 codes with a per-row scale, without full float32 temporaries: one Triton
+kernel on CUDA / ROCm, bounded row chunks elsewhere. Bitwise equal to ``(codes.float() * scale[:, None]).to(dtype)``
+for every non-NaN value; torch imported lazily"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Callable, Optional
+
+# ``0`` widens a whole layer at once again (two float32 matrices per call).
+FP8_DEQUANT_FUSED_ENV = "UNSLOTH_DIFFUSION_FP8_DEQUANT_FUSED"
+# float32 scratch per chunk; a layer whose whole widen fits stays one expression.
+CHUNK_SCRATCH_BYTES_CPU = 16 << 20
+CHUNK_SCRATCH_BYTES_ACCEL = 64 << 20
+_TRITON_BLOCK = 1024
+_TRITON_STATE: dict = {"kernel": None, "failed": None}
+_BF16_NAN: dict = {}
+# Specialisations launched once in eager, so a CUDA-graph capture may replay-record them.
+_WARM: set = set()
+_FMT_ID = {None: 0, "float8_e4m3fn": 1, "float8_e5m2": 2}
+
+
+def fused_dequant_enabled() -> bool:
+ return os.environ.get(FP8_DEQUANT_FUSED_ENV, "1").strip() != "0"
+
+
+def chunk_scratch_bytes(device: Any) -> int:
+ return CHUNK_SCRATCH_BYTES_CPU if device.type == "cpu" else CHUNK_SCRATCH_BYTES_ACCEL
+
+
+def chunk_rows(
+ rows: int,
+ cols: int,
+ device: Any,
+ per_elem: int = 4,
+) -> int:
+ """Rows per chunk under the scratch budget, or 0 when the old two-temporary widen (8 B/elem) fits it."""
+ budget = chunk_scratch_bytes(device)
+ if rows * cols * 8 <= budget:
+ return 0
+ return min(rows, max(1, budget // max(1, cols * per_elem)))
+
+
+def _build_kernel() -> Any:
+ from .diffusion_fp8_dequant_triton import dequant_rows_kernel
+ return dequant_rows_kernel
+
+
+def _bf16_nan(device: Any) -> int:
+ import torch
+
+ key = str(device)
+ bits = _BF16_NAN.get(key)
+ if bits is None:
+ nan = torch.full((1,), float("nan"), dtype = torch.float32, device = device)
+ bits = _BF16_NAN[key] = int(nan.to(torch.bfloat16).view(torch.int16).item()) & 0xFFFF
+ return bits
+
+
+def launch_triton(codes: Any, scale: Any, out: Any, fmt: Optional[str]) -> None:
+ """``out[r, c] = (decode(codes[r, c]) * scale[r]).to(out.dtype)``; contiguous 2-D codes / out, fp32 scale."""
+ import torch
+ import triton
+
+ kernel = _TRITON_STATE["kernel"]
+ if kernel is None:
+ kernel = _TRITON_STATE["kernel"] = _build_kernel()
+ rows, cols = codes.shape
+ grid = (rows, triton.cdiv(cols, _TRITON_BLOCK))
+ # No FMA contraction: the multiply rounds once, as torch's.
+ kernel[grid](
+ codes,
+ scale,
+ out,
+ cols,
+ FMT = _FMT_ID[fmt],
+ BF16 = out.dtype == torch.bfloat16,
+ BF16_NAN = _bf16_nan(out.device) if out.dtype == torch.bfloat16 else 0,
+ BLOCK = _TRITON_BLOCK,
+ num_warps = 4,
+ enable_fp_fusion = False,
+ )
+
+
+def _triton_ok(codes: Any, dtype: Any) -> bool:
+ import torch
+
+ if _TRITON_STATE["failed"] is not None or not codes.is_cuda or not codes.is_contiguous():
+ return False
+ if dtype not in (torch.bfloat16, torch.float16, torch.float32) or codes.dim() != 2:
+ return False
+ rows, cols = codes.shape
+ if not (0 < rows < (1 << 31) and 0 < cols and -(-cols // _TRITON_BLOCK) < 65536):
+ return False
+ # Under CUDA-graph capture only an already compiled specialisation (no JIT, no NaN probe sync).
+ return not torch.cuda.is_current_stream_capturing() or _warm_key(codes, dtype) in _WARM
+
+
+def _warm_key(codes: Any, dtype: Any) -> tuple:
+ cols = codes.shape[1]
+ return (str(codes.device), codes.dtype, dtype, cols % 16 == 0, cols == 1)
+
+
+def _triton_failed(exc: BaseException) -> None:
+ import logging
+ _TRITON_STATE["failed"] = repr(exc)
+ logging.getLogger(__name__).debug(
+ "diffusion.fp8_dequant: Triton widen unavailable (%s); chunked torch from here on", exc
+ )
+
+
+def dequant_rows(
+ codes: Any,
+ scale: Any,
+ dtype: Any,
+ widen: Callable[[Any], Any],
+ fmt: Optional[str] = None,
+ triton_ok: bool = True,
+) -> Any:
+ """``(widen(codes) * scale[:, None]).to(dtype)`` for 2-D ``codes`` and an fp32 ``scale`` of shape [rows].
+ ``widen`` maps a row slice to its exact float32 values; ``fmt`` names the fp8 bits (None: integer codes)."""
+ import torch
+
+ if triton_ok and _triton_ok(codes, dtype):
+ out = torch.empty(codes.shape, dtype = dtype, device = codes.device)
+ try:
+ launch_triton(codes, scale.contiguous(), out, fmt)
+ _WARM.add(_warm_key(codes, dtype))
+ return out
+ except Exception as exc: # noqa: BLE001 - no Triton / launch failure: chunked torch below
+ _triton_failed(exc)
+ del out
+ rows = codes.shape[0]
+ cols = codes.numel() // max(1, rows)
+ step = chunk_rows(rows, cols, codes.device, 8 if codes.device.type == "mps" else 4)
+ if not step:
+ return (widen(codes) * scale[:, None]).to(dtype)
+ out = torch.empty(codes.shape, dtype = dtype, device = codes.device)
+ for start in range(0, rows, step):
+ stop = min(rows, start + step)
+ wide = widen(codes[start:stop])
+ wide.mul_(scale[start:stop, None])
+ out[start:stop].copy_(wide)
+ del wide
+ return out
diff --git a/studio/backend/core/inference/diffusion_fp8_dequant_triton.py b/studio/backend/core/inference/diffusion_fp8_dequant_triton.py
new file mode 100644
index 00000000000..8e7285c914a
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_fp8_dequant_triton.py
@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Triton kernel for ``diffusion_fp8_dequant`` (module level: a compiled kernel resolves ``tl`` via globals)."""
+
+import triton
+import triton.language as tl
+
+
+def _fp16_bits(value):
+ """float32 -> float16 bits, round to nearest even by integer ops (overflow to inf, subnormals kept), as torch
+ casts on every device; ROCm's own conversion differs."""
+ u = value.to(tl.uint32, bitcast = True)
+ sign = (u >> 16) & 0x8000
+ a = u & 0x7FFFFFFF
+ r = a - (112 << 23)
+ normal = (r + 0x0FFF + ((r >> 13) & 1)) >> 13
+ # below 2**-14: the subnormal count, |x| * 2**24 rounded to the nearest even integer (exact scaling, one add)
+ scaled = (a.to(tl.float32, bitcast = True) * 16777216.0 + 8388608.0) - 8388608.0
+ sub = scaled.to(tl.uint32)
+ bits = tl.where(a >= 0x38800000, normal, sub)
+ bits = tl.where(a >= 0x477FF000, 0x7C00, bits)
+ # NaN: quieted, top payload bits kept, as the hardware conversion torch uses
+ bits = tl.where(a > 0x7F800000, 0x7E00 | ((a >> 13) & 0x3FF), bits)
+ return (bits | sign).to(tl.uint16)
+
+
+def dequant_rows_kernel(
+ codes_ptr,
+ scale_ptr,
+ out_ptr,
+ cols,
+ FMT: tl.constexpr,
+ BF16: tl.constexpr,
+ BF16_NAN: tl.constexpr,
+ BLOCK: tl.constexpr,
+):
+ row = tl.program_id(0).to(tl.int64)
+ col = tl.program_id(1).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
+ mask = col < cols
+ offs = row * cols + col
+ s = tl.load(scale_ptr + row)
+ c = tl.load(codes_ptr + offs, mask = mask, other = 0)
+ if FMT == 0:
+ wide = c.to(tl.float32)
+ else:
+ # fp8_bits_to_float32 by integer ops (no fp8 type: sm75, ROCm); NaN bits as torch's own cast.
+ b = c.to(tl.int32)
+ if FMT == 1:
+ e = (b >> 3) & 15
+ m = b & 7
+ normal = ((e + 120) << 23) | (m << 20)
+ sub = (m.to(tl.float32) * 0.001953125).to(tl.int32, bitcast = True)
+ special = (e == 15) & (m == 7)
+ special_bits = tl.full(b.shape, 0x7FF00000, tl.int32)
+ else:
+ e = (b >> 2) & 31
+ m = b & 3
+ normal = ((e + 112) << 23) | (m << 21)
+ sub = (m.to(tl.float32) * 0.0000152587890625).to(tl.int32, bitcast = True)
+ special = e == 31
+ special_bits = 0x7F800000 | (m << 21) | tl.where(m != 0, 0x00400000, 0)
+ mag = tl.where(e == 0, sub, normal)
+ mag = tl.where(special, special_bits, mag)
+ wide = (mag | ((b & 0x80) << 24)).to(tl.float32, bitcast = True)
+ prod = wide * s
+ if BF16:
+ # nearest even by integer ops (the interpreter truncates); NaN as torch's cast on this device
+ bits = prod.to(tl.int32, bitcast = True)
+ rne = (bits + 0x7FFF + ((bits >> 16) & 1)) >> 16
+ out = tl.where((bits & 0x7FFFFFFF) > 0x7F800000, BF16_NAN, rne).to(tl.int16)
+ out = out.to(tl.bfloat16, bitcast = True)
+ else:
+ if out_ptr.dtype.element_ty == tl.float16:
+ # nearest even by integer ops: ROCm's own fp32 -> fp16 conversion differs from torch's
+ out = _fp16_bits(prod).to(tl.float16, bitcast = True)
+ else:
+ out = prod.to(out_ptr.dtype.element_ty)
+ tl.store(out_ptr + offs, out, mask = mask)
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.py
index 3a0ecf79057..bda53720da7 100644
--- a/studio/backend/core/inference/diffusion_native_quant.py
+++ b/studio/backend/core/inference/diffusion_native_quant.py
@@ -14,6 +14,8 @@ import time
from functools import lru_cache
from typing import Any, Callable, Optional
+from .diffusion_fp8_dequant import dequant_rows, fused_dequant_enabled
+
NATIVE_INT8 = "int8"
NATIVE_FP8 = "fp8"
_QMAX = {NATIVE_INT8: 127.0, NATIVE_FP8: 448.0}
@@ -509,8 +511,25 @@ def native_linear_class():
self._row_major = _row_major_op() if self.act_int8 and not self.rot_group else None
self._int8pack_scales: dict = {}
+ def _widen(self, codes: Any) -> Any:
+ """Exact float32 values of a row slice of ``weight_q``, as the eager branches below."""
+ if self.scheme != NATIVE_FP8:
+ return codes.to(torch.float32)
+ if codes.device.type == "mps":
+ lut = fp8_lookup_table(self.fp8_format, str(codes.device))
+ return lut.index_select(0, codes.reshape(-1).int()).view(codes.shape)
+ return codes.view(getattr(torch, self.fp8_format)).to(torch.float32)
+
def _stored_weight(self, dtype: Any) -> Any:
scale = self.weight_scale.view(torch.float32)
+ if not torch.compiler.is_compiling() and fused_dequant_enabled():
+ return dequant_rows(
+ self.weight_q,
+ scale,
+ dtype,
+ self._widen,
+ self.fp8_format if self.scheme == NATIVE_FP8 else None,
+ )
if self.scheme != NATIVE_FP8:
wide = self.weight_q.to(torch.float32)
elif torch.compiler.is_compiling():
diff --git a/studio/backend/tests/test_diffusion_fp8_dequant.py b/studio/backend/tests/test_diffusion_fp8_dequant.py
new file mode 100644
index 00000000000..1266886bcce
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_fp8_dequant.py
@@ -0,0 +1,441 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Eager int8 / stored-fp8 widening without full float32 temporaries: bitwise equal to the whole-layer formula on
+the chunked torch path and the Triton kernel (under TRITON_INTERPRET=1), kill switch, compile unchanged."""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_comfy_quant as cq # noqa: E402
+import core.inference.diffusion_native_quant as nq # noqa: E402
+
+# The tree ``core`` was imported from (the subprocess imports the same one).
+BACKEND = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(nq.__file__))))
+ENV = "UNSLOTH_DIFFUSION_FP8_DEQUANT_FUSED"
+FP8 = ("float8_e4m3fn", "float8_e5m2")
+DTYPES = (torch.bfloat16, torch.float16, torch.float32)
+# 1, extremes, the e4m3 max-ish, non-powers of two (rounding), a subnormal scale, a negative one (signed zero)
+SCALES = (1.0, 2.0**-20, 2.0**20, 448.0, 1 / 448, 0.7, 3.3e-3, 1e-40, 2.0**-140, -1.5, 2.0**100)
+
+
+def _fd():
+ import core.inference.diffusion_fp8_dequant as fd
+ return fd
+
+
+def test_imports_own_tree():
+ """The modules under test are this test's own tree."""
+ root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+ for mod in (nq, cq):
+ assert os.path.realpath(mod.__file__).startswith(root + os.sep), mod.__file__
+
+
+def _old(
+ codes,
+ scale,
+ dtype,
+ fmt = None,
+):
+ """Today's eager formula, independent of the tree."""
+ wide = (
+ codes.to(torch.float32)
+ if fmt is None
+ else codes.view(getattr(torch, fmt)).to(torch.float32)
+ )
+ return (wide * scale[:, None]).to(dtype)
+
+
+def _bits(t):
+ return t.view({1: torch.int8, 2: torch.int16, 4: torch.int32}[t.element_size()])
+
+
+def _assert_bitwise(new, ref):
+ assert new.dtype == ref.dtype and new.shape == ref.shape
+ nan = torch.isnan(ref)
+ assert torch.equal(torch.isnan(new), nan)
+ assert torch.equal(_bits(new)[~nan], _bits(ref)[~nan])
+
+
+def _layer(
+ codes,
+ scale,
+ scheme,
+ dtype = torch.bfloat16,
+ bias = False,
+):
+ lin = torch.nn.Linear(codes.shape[1], codes.shape[0], bias = bias, dtype = dtype)
+ return nq.native_linear_class()(lin, scheme, codes = codes, scale = scale)
+
+
+def _exhaustive(fmt):
+ codes = torch.arange(256, dtype = torch.uint8).repeat(len(SCALES), 1).contiguous()
+ if fmt is None:
+ codes = codes.view(torch.int8)
+ return codes, torch.tensor(SCALES, dtype = torch.float32)
+
+
+def tiny_chunks(monkeypatch):
+ """One row per chunk on CPU, so every row runs the chunked path."""
+ monkeypatch.delenv(ENV, raising = False)
+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 4)
+
+
[email protected]("fmt", (*FP8, None))
[email protected]("dtype", DTYPES)
+def test_every_code_and_scale_is_bitwise_on_the_chunked_path(tiny_chunks, fmt, dtype):
+ codes, scale = _exhaustive(fmt)
+ store = codes.view(getattr(torch, fmt)) if fmt else codes
+ layer = _layer(store, scale, nq.NATIVE_FP8 if fmt else nq.NATIVE_INT8)
+ ref = _old(codes, scale, dtype, fmt)
+ _assert_bitwise(layer._stored_weight(dtype), ref)
+ # Same per-element ops as today, so NaN bits match too.
+ assert torch.equal(_bits(layer._stored_weight(dtype)), _bits(ref))
+
+
[email protected]("fmt", FP8)
+def test_integer_decode_matches_torch_cast_bits(fmt):
+ """The bit decode the kernel mirrors: fp8_bits_to_float32 for every value, torch's NaN bits in the kernel."""
+ codes = torch.arange(256, dtype = torch.uint8)
+ cast = codes.view(getattr(torch, fmt)).float()
+ _assert_bitwise(nq.fp8_bits_to_float32(codes, fmt), cast)
+
+
+_INTERP = textwrap.dedent(
+ """
+ import json, os, sys
+ os.environ["TRITON_INTERPRET"] = "1"
+ sys.path.insert(0, sys.argv[1])
+ import torch
+ import core.inference.diffusion_fp8_dequant as fd
+ assert os.path.realpath(fd.__file__).startswith(os.path.realpath(sys.argv[1]))
+ scales = torch.tensor(json.loads(sys.argv[2]), dtype = torch.float32)
+ res = []
+ for fmt in ("float8_e4m3fn", "float8_e5m2", None):
+ codes = torch.arange(256, dtype = torch.uint8).repeat(len(scales), 1).contiguous()
+ if fmt is None:
+ codes = codes.view(torch.int8)
+ wide = codes.to(torch.float32) if fmt is None else codes.view(getattr(torch, fmt)).to(torch.float32)
+ for dt in (torch.bfloat16, torch.float16, torch.float32):
+ ref = (wide * scales[:, None]).to(dt)
+ out = torch.full(codes.shape, 7, dtype = dt)
+ fd.launch_triton(codes, scales, out, fmt)
+ iv = {2: torch.int16, 4: torch.int32}[out.element_size()]
+ nan = torch.isnan(ref)
+ res.append(dict(
+ fmt = fmt, dtype = str(dt),
+ nan_pos = bool(torch.equal(torch.isnan(out), nan)),
+ values = bool(torch.equal(out.view(iv)[~nan], ref.view(iv)[~nan])),
+ nan_bits_equal = bool(torch.equal(out.view(iv)[nan], ref.view(iv)[nan])),
+ n_nan = int(nan.sum()),
+ n = int(ref.numel()),
+ ))
+ # Masked tail and int64 offsets: odd widths.
+ for cols in (1, 3, 1023, 1025):
+ codes = torch.randint(0, 256, (5, cols), dtype = torch.uint8)
+ s = torch.rand(5) + 0.1
+ ref = (codes.view(torch.float8_e4m3fn).float() * s[:, None]).to(torch.bfloat16)
+ out = torch.empty(codes.shape, dtype = torch.bfloat16)
+ fd.launch_triton(codes, s, out, "float8_e4m3fn")
+ nan = torch.isnan(ref)
+ res.append(dict(fmt = "tail", dtype = str(cols), nan_pos = bool(torch.equal(torch.isnan(out), nan)),
+ values = bool(torch.equal(out.view(torch.int16)[~nan], ref.view(torch.int16)[~nan])),
+ nan_bits_equal = None, n_nan = int(nan.sum()), n = int(ref.numel())))
+ print("RESULT " + json.dumps(res))
+ """
+)
+
+
+def test_triton_kernel_is_bitwise_under_the_interpreter():
+ pytest.importorskip("triton")
+ env = dict(os.environ, TRITON_INTERPRET = "1", CUDA_VISIBLE_DEVICES = "")
+ proc = subprocess.run(
+ [sys.executable, "-c", _INTERP, BACKEND, json.dumps(SCALES)],
+ capture_output = True,
+ text = True,
+ env = env,
+ timeout = 600,
+ )
+ line = [l for l in proc.stdout.splitlines() if l.startswith("RESULT ")]
+ if not line and "interpret" in proc.stderr.lower() and "not supported" in proc.stderr.lower():
+ # skip because this Triton build has no interpreter for the op; the GPU check covers the kernel
+ pytest.skip(f"Triton interpreter cannot run here: {proc.stderr[-400:]}")
+ assert line, proc.stderr[-4000:]
+ res = json.loads(line[0][len("RESULT ") :])
+ print(json.dumps(res, indent = 1))
+ for r in res:
+ assert r["nan_pos"] and r["values"], r
+ # NaN bits too, against CPU torch (the bfloat16 NaN constant is chosen per backend).
+ for r in res:
+ if r["fmt"] != "tail":
+ assert r["nan_bits_equal"], r
+
+
[email protected]("rows", (1, 17, 1023, 1024, 1025))
[email protected]("fmt", ("float8_e4m3fn", None))
+def test_odd_row_counts_over_several_chunks(monkeypatch, rows, fmt):
+ monkeypatch.delenv(ENV, raising = False)
+ cols = 64
+ # 100 rows per chunk: 1025 rows = 11 chunks with a 25-row tail.
+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 100 * cols * 4)
+ if rows * cols * 8 <= 100 * cols * 4:
+ assert _fd().chunk_rows(rows, cols, torch.device("cpu")) == 0
+ else:
+ assert _fd().chunk_rows(rows, cols, torch.device("cpu")) == 100
+ g = torch.Generator().manual_seed(rows)
+ codes = torch.randint(0, 256, (rows, cols), dtype = torch.uint8, generator = g)
+ if fmt is None:
+ codes = codes.view(torch.int8)
+ scale = torch.rand(rows, generator = g) * 0.01 + 1e-4
+ store = codes.view(getattr(torch, fmt)) if fmt else codes
+ layer = _layer(store, scale, nq.NATIVE_FP8 if fmt else nq.NATIVE_INT8)
+ for dtype in DTYPES:
+ _assert_bitwise(layer._stored_weight(dtype), _old(codes, scale, dtype, fmt))
+
+
+def test_kill_switch_takes_the_old_path(monkeypatch):
+ calls = []
+ real = nq.dequant_rows
+
+ def spy(*a, **k):
+ calls.append(1)
+ return real(*a, **k)
+
+ monkeypatch.setattr(nq, "dequant_rows", spy)
+ codes, scale = _exhaustive("float8_e4m3fn")
+ layer = _layer(codes.view(torch.float8_e4m3fn), scale, nq.NATIVE_FP8)
+ monkeypatch.setenv(ENV, "0")
+ old = layer._stored_weight(torch.bfloat16)
+ assert calls == []
+ monkeypatch.setenv(ENV, "1")
+ new = layer._stored_weight(torch.bfloat16)
+ assert calls == [1]
+ assert torch.equal(_bits(old), _bits(new))
+
+ seen = []
+ monkeypatch.setattr(cq, "dequant_rows", lambda *a, **k: seen.append(1) or real(*a, **k))
+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 4)
+ monkeypatch.setenv(ENV, "0")
+ cq._dequant(codes.view(torch.float8_e4m3fn), scale, 0, torch.bfloat16)
+ assert seen == []
+ monkeypatch.setenv(ENV, "1")
+ cq._dequant(codes.view(torch.float8_e4m3fn), scale, 0, torch.bfloat16)
+ assert seen == [1]
+
+
+def test_triton_failure_latches_and_falls_back(monkeypatch):
+ """A launch failure never reaches the caller: chunked torch, and Triton is not tried again."""
+ fd = _fd()
+ tries = []
+
+ def boom(*a, **k):
+ tries.append(1)
+ raise RuntimeError("no triton")
+
+ monkeypatch.setattr(fd, "launch_triton", boom)
+ monkeypatch.setattr(fd, "_TRITON_STATE", {"kernel": None, "failed": None})
+ monkeypatch.setattr(fd, "_triton_ok", lambda c, d: fd._TRITON_STATE["failed"] is None)
+ codes, scale = _exhaustive("float8_e5m2")
+ widen = lambda c: c.view(torch.float8_e5m2).to(torch.float32)
+ for _ in range(2):
+ out = fd.dequant_rows(codes, scale, torch.float16, widen, "float8_e5m2")
+ _assert_bitwise(out, _old(codes, scale, torch.float16, "float8_e5m2"))
+ assert tries == [1] and fd._TRITON_STATE["failed"]
+
+
+class _Fp32Full(torch.utils._python_dispatch.TorchDispatchMode):
+ """Counts op outputs that are full-size float32 [rows, cols] matrices."""
+
+ def __init__(self, numel):
+ super().__init__()
+ self.numel, self.hits = numel, 0
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ out = func(*args, **(kwargs or {}))
+ for t in out if isinstance(out, (tuple, list)) else (out,):
+ if isinstance(t, torch.Tensor) and t.dtype == torch.float32 and t.numel() >= self.numel:
+ self.hits += 1
+ return out
+
+
[email protected]("scheme", (nq.NATIVE_FP8, nq.NATIVE_INT8))
+def test_eager_widen_holds_no_full_fp32_matrix(monkeypatch, scheme):
+ """Base: the float32 codes and their float32 product (2 full matrices). Head: one bounded chunk."""
+ monkeypatch.delenv(ENV, raising = False)
+ rows, cols = 2048, 4096 # 32 MiB of float32 per matrix, over the 16 MiB CPU budget
+ g = torch.Generator().manual_seed(0)
+ raw = torch.randint(-127, 128, (rows, cols), dtype = torch.int8, generator = g)
+ codes = raw.view(torch.uint8).view(torch.float8_e4m3fn) if scheme == nq.NATIVE_FP8 else raw
+ if scheme == nq.NATIVE_FP8:
+ codes = torch.where(torch.isnan(codes.float()), 0, codes.float()).to(torch.float8_e4m3fn)
+ layer = _layer(codes, torch.rand(rows) + 0.5, scheme)
+ with _Fp32Full(rows * cols) as mode:
+ layer._stored_weight(torch.bfloat16)
+ assert mode.hits == 0
+ monkeypatch.setenv(ENV, "0")
+ with _Fp32Full(rows * cols) as mode:
+ layer._stored_weight(torch.bfloat16)
+ assert mode.hits == 2
+
+
[email protected]("fmt", (*FP8, "int8"))
[email protected]("per_row", (True, False))
+def test_comfy_dequant_chunked_is_bitwise_and_charges_the_true_peak(monkeypatch, fmt, per_row):
+ monkeypatch.delenv(ENV, raising = False)
+ import core.inference.diffusion_single_file_converters as sfc
+
+ charged = []
+ monkeypatch.setattr(sfc, "charge_host_ram", lambda n: charged.append(int(n)))
+ rows, cols = 1025, 96
+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 64 * cols * 4)
+ g = torch.Generator().manual_seed(1)
+ raw = torch.randint(0, 256, (rows, cols), dtype = torch.uint8, generator = g)
+ codes = raw.view(torch.int8) if fmt == "int8" else raw.view(getattr(torch, fmt))
+ scale = (torch.rand(rows if per_row else 1, generator = g) * 0.02).to(torch.float32)
+ for dtype in DTYPES:
+ charged.clear()
+ new = cq._dequant(codes, scale, 0, dtype)
+ new_charge = charged[-1]
+ monkeypatch.setenv(ENV, "0")
+ charged.clear()
+ old = cq._dequant(codes, scale, 0, dtype)
+ old_charge = charged[-1]
+ monkeypatch.delenv(ENV)
+ ref = (codes.to(torch.float32) * scale.reshape(-1, 1).expand(rows, 1)).to(dtype)
+ _assert_bitwise(new, ref)
+ assert torch.equal(_bits(new), _bits(old))
+ out_size = torch.empty((), dtype = dtype).element_size()
+ assert new_charge == rows * cols * out_size + 64 * cols * 4
+ assert old_charge == rows * cols * max(8, 4 + out_size) > new_charge
+
+
+def test_comfy_convrot_dequant_is_unchanged(monkeypatch):
+ """ConvRot (group > 0) is not row-chunked: same expression, same charge, either setting."""
+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 4)
+ import core.inference.diffusion_single_file_converters as sfc
+
+ charged = []
+ monkeypatch.setattr(sfc, "charge_host_ram", lambda n: charged.append(int(n)))
+ codes = torch.randint(-127, 128, (33, 64), dtype = torch.int8)
+ scale = torch.rand(33)
+ monkeypatch.setenv(ENV, "0")
+ old = cq._dequant(codes, scale, 16, torch.bfloat16)
+ monkeypatch.setenv(ENV, "1")
+ new = cq._dequant(codes, scale, 16, torch.bfloat16)
+ assert torch.equal(_bits(old), _bits(new))
+ assert charged == [33 * 64 * 12] * 2
+
+
[email protected]("scheme", (nq.NATIVE_FP8, nq.NATIVE_INT8))
+def test_forward_is_bitwise_new_vs_old(monkeypatch, scheme):
+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 8 * 48 * 4)
+ g = torch.Generator().manual_seed(2)
+ lin = torch.nn.Linear(48, 40, bias = True, dtype = torch.bfloat16)
+ layer = nq.native_linear_class()(lin, scheme)
+ x = torch.randn(3, 5, 48, generator = g).to(torch.bfloat16)
+ monkeypatch.setenv(ENV, "1")
+ new = layer(x)
+ w_new = layer.weight
+ monkeypatch.setenv(ENV, "0")
+ old = layer(x)
+ w_old = layer.weight
+ assert torch.equal(_bits(new), _bits(old))
+ assert torch.equal(_bits(w_new), _bits(w_old))
+
+
+def _compile_counts(layer, x, backend):
+ from torch._dynamo.testing import CompileCounterWithBackend
+
+ torch._dynamo.reset()
+ cnt = CompileCounterWithBackend(backend)
+ fn = torch.compile(layer, backend = cnt, fullgraph = True)
+ out = fn(x)
+ out2 = fn(x)
+ return cnt.frame_count, cnt.op_count, out, out2
+
+
[email protected]("backend", ("aot_eager", "inductor"))
+def test_compiled_stored_fp8_layer_graph_is_unchanged(monkeypatch, backend):
+ """The compiled branch is today's: one fullgraph frame, the same op count with the switch on or off."""
+ lin = torch.nn.Linear(64, 32, bias = True, dtype = torch.float32)
+ layer = nq.native_linear_class()(lin, nq.NATIVE_FP8)
+ x = torch.randn(4, 64)
+ counts = {}
+ for flag in ("1", "0"):
+ monkeypatch.setenv(ENV, flag)
+ try:
+ frames, ops, out, out2 = _compile_counts(layer, x, backend)
+ except Exception as exc: # noqa: BLE001 - no C++ toolchain for CPU inductor
+ if backend == "inductor":
+ # skip because CPU inductor needs a C++ toolchain; the aot_eager leg still runs
+ pytest.skip(f"CPU inductor unavailable: {exc!r}"[:300])
+ raise
+ assert frames == 1
+ assert torch.equal(out, out2)
+ counts[flag] = (frames, ops)
+ print("compile counts", backend, counts)
+ assert counts["1"] == counts["0"]
+ torch._dynamo.reset()
+
+
+def test_cuda_graph_capture_takes_only_a_warm_kernel(monkeypatch):
+ """No first-launch JIT / NaN probe sync inside a capture: chunked torch until the specialisation ran eager."""
+ import types
+
+ fd = _fd()
+ monkeypatch.setattr(fd, "_TRITON_STATE", {"kernel": None, "failed": None})
+ monkeypatch.setattr(fd, "_WARM", set())
+ codes = types.SimpleNamespace(
+ is_cuda = True,
+ is_contiguous = lambda: True,
+ dim = lambda: 2,
+ shape = (4, 32),
+ device = "cuda:0",
+ dtype = torch.uint8,
+ )
+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
+ assert fd._triton_ok(codes, torch.bfloat16)
+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
+ assert not fd._triton_ok(codes, torch.bfloat16)
+ fd._WARM.add(fd._warm_key(codes, torch.bfloat16))
+ assert fd._triton_ok(codes, torch.bfloat16)
+ assert not fd._triton_ok(codes, torch.float16)
+
+
+ not torch.cuda.is_available(), reason = "needs a CUDA / ROCm GPU to run the Triton widen"
+)
[email protected]("fmt", (*FP8, None))
[email protected]("dtype", DTYPES)
+def test_the_triton_widen_on_the_gpu_matches_the_old_formula(monkeypatch, fmt, dtype):
+ monkeypatch.delenv(ENV, raising = False)
+ codes, scale = _exhaustive(fmt)
+ codes = torch.cat(
+ [codes] * 8
+ ).contiguous() # several rows per scale, every column block masked or full
+ scale = torch.cat([scale] * 8)
+ store = codes.view(getattr(torch, fmt)) if fmt else codes
+ layer = _layer(store, scale, nq.NATIVE_FP8 if fmt else nq.NATIVE_INT8).to("cuda")
+ launches = []
+ real = _fd().launch_triton
+ monkeypatch.setattr(_fd(), "launch_triton", lambda *a: launches.append(1) or real(*a))
+ got = layer._stored_weight(dtype).cpu()
+ assert launches and _fd()._TRITON_STATE["failed"] is None, _fd()._TRITON_STATE["failed"]
+ ref = _old(codes, scale, dtype, fmt)
+ _assert_bitwise(got, ref)
diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.py
index 341f8a4d33b..f9031f9c353 100644
--- a/studio/backend/tests/test_diffusion_native_quant.py
+++ b/studio/backend/tests/test_diffusion_native_quant.py
@@ -1340,3 +1340,47 @@ def test_blocked_torchao_holds_after_peft_cached_it_available(monkeypatch):
) # newer peft's dispatcher imports this one instead
model = _toy()
peft.inject_adapter_in_model(peft.LoraConfig(r = 4, target_modules = ["to_q"]), model)
+
+
[email protected]("scheme", [nq.NATIVE_FP8, nq.NATIVE_INT8])
+def test_the_eager_stored_widen_allocates_no_full_float32_matrix(scheme):
+ """A stored fp8 / int8 layer's eager widen: the same bf16 weight, with no float32 temporary the size of the
+ whole layer (the old widen held two)."""
+ from torch.utils._python_dispatch import TorchDispatchMode
+
+ rows, cols = 2048, 4096
+ gen = torch.Generator().manual_seed(0)
+ codes = torch.randint(-127, 128, (rows, cols), dtype = torch.int8, generator = gen)
+ if scheme == nq.NATIVE_FP8:
+ codes = torch.randint(0, 0x7F, (rows, cols), dtype = torch.uint8, generator = gen).view(
+ torch.float8_e4m3fn
+ )
+ scale = torch.rand(rows, generator = gen) * 1e-2
+ lin = torch.nn.Linear(cols, rows, bias = False, dtype = torch.bfloat16)
+ layer = nq.native_linear_class()(lin, scheme, codes = codes, scale = scale)
+ want = (codes.to(torch.float32) * scale.reshape(-1, 1)).to(torch.bfloat16)
+
+ class FullFloat32(TorchDispatchMode):
+ count = 0
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ out = func(*args, **(kwargs or {}))
+ if (
+ torch.is_tensor(out)
+ and out.dtype == torch.float32
+ and out.numel() >= rows * cols
+ and func is not torch.ops.aten.view.default
+ ):
+ FullFloat32.count += 1
+ return out
+
+ with torch.no_grad(), FullFloat32():
+ got = layer._stored_weight(torch.bfloat16)
+ assert torch.equal(got.view(torch.int16), want.view(torch.int16))
+ assert FullFloat32.count == 0, FullFloat32.count