Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: widen stored fp8 / int8 weights without full float32 temporaries (#17068)
Daniel Han57d5fffbcf61
diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.pyindex d8f6c14d8b7..36a220298f6 100644--- a/studio/backend/core/inference/diffusion_comfy_quant.py+++ b/studio/backend/core/inference/diffusion_comfy_quant.py@@ -12,6 +12,7 @@ from dataclasses import dataclass, fieldfrom typing import Any, Optionalfrom .diffusion_comfy_block import BLOCK_SIZES, MXFP8, NVFP4, tiled_scale_numel+from .diffusion_fp8_dequant import chunk_rows, dequant_rows, fused_dequant_enabledfrom .diffusion_single_file_converters import host_ram_budget_scopefrom .diffusion_small_host import FP8_STORAGE_HOST_RESERVE_MIB@@ -594,10 +595,23 @@ def _dequant(codes: Any, scale: Any, group: int, dtype: Any) -> Any:from .diffusion_single_file_converters import charge_host_ramrows = codes.shape[0]+ out_size = torch.empty((), dtype = dtype).element_size()+ if not group and codes.dim() == 2 and rows and fused_dequant_enabled():+ cols = codes.shape[1]+ step = chunk_rows(rows, cols, codes.device)+ if step:+ # Row chunks into the output: the output plus one float32 chunk is the peak.+ charge_host_ram(codes.numel() * out_size + step * cols * 4)+ return dequant_rows(+ codes,+ scale.to(torch.float32).reshape(-1).expand(rows),+ dtype,+ lambda c: c.to(torch.float32, copy = True),+ triton_ok = False,+ )# Peak, not just the output: the float32 codes and their float32 product are live together (8 bytes per# element); ConvRot's batched matmul holds the product, a float32 copy of it and its float32 result (12,# measured); last the product and the output.- out_size = torch.empty((), dtype = dtype).element_size()charge_host_ram(codes.numel() * max(12 if group else 8, 4 + out_size))weight = codes.to(torch.float32) * scale.to(torch.float32).reshape(-1, 1).expand(rows, 1)if group:diff --git a/studio/backend/core/inference/diffusion_fp8_dequant.py b/studio/backend/core/inference/diffusion_fp8_dequant.pynew file mode 100644index 00000000000..91829de068f--- /dev/null+++ b/studio/backend/core/inference/diffusion_fp8_dequant.py@@ -0,0 +1,148 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Eager widening of int8 / stored-fp8 codes with a per-row scale, without full float32 temporaries: one Triton+kernel on CUDA / ROCm, bounded row chunks elsewhere. Bitwise equal to ``(codes.float() * scale[:, None]).to(dtype)``+for every non-NaN value; torch imported lazily"""++from __future__ import annotations++import os+from typing import Any, Callable, Optional++# ``0`` widens a whole layer at once again (two float32 matrices per call).+FP8_DEQUANT_FUSED_ENV = "UNSLOTH_DIFFUSION_FP8_DEQUANT_FUSED"+# float32 scratch per chunk; a layer whose whole widen fits stays one expression.+CHUNK_SCRATCH_BYTES_CPU = 16 << 20+CHUNK_SCRATCH_BYTES_ACCEL = 64 << 20+_TRITON_BLOCK = 1024+_TRITON_STATE: dict = {"kernel": None, "failed": None}+_BF16_NAN: dict = {}+# Specialisations launched once in eager, so a CUDA-graph capture may replay-record them.+_WARM: set = set()+_FMT_ID = {None: 0, "float8_e4m3fn": 1, "float8_e5m2": 2}+++def fused_dequant_enabled() -> bool:+ return os.environ.get(FP8_DEQUANT_FUSED_ENV, "1").strip() != "0"+++def chunk_scratch_bytes(device: Any) -> int:+ return CHUNK_SCRATCH_BYTES_CPU if device.type == "cpu" else CHUNK_SCRATCH_BYTES_ACCEL+++def chunk_rows(+ rows: int,+ cols: int,+ device: Any,+ per_elem: int = 4,+) -> int:+ """Rows per chunk under the scratch budget, or 0 when the old two-temporary widen (8 B/elem) fits it."""+ budget = chunk_scratch_bytes(device)+ if rows * cols * 8 <= budget:+ return 0+ return min(rows, max(1, budget // max(1, cols * per_elem)))+++def _build_kernel() -> Any:+ from .diffusion_fp8_dequant_triton import dequant_rows_kernel+ return dequant_rows_kernel+++def _bf16_nan(device: Any) -> int:+ import torch++ key = str(device)+ bits = _BF16_NAN.get(key)+ if bits is None:+ nan = torch.full((1,), float("nan"), dtype = torch.float32, device = device)+ bits = _BF16_NAN[key] = int(nan.to(torch.bfloat16).view(torch.int16).item()) & 0xFFFF+ return bits+++def launch_triton(codes: Any, scale: Any, out: Any, fmt: Optional[str]) -> None:+ """``out[r, c] = (decode(codes[r, c]) * scale[r]).to(out.dtype)``; contiguous 2-D codes / out, fp32 scale."""+ import torch+ import triton++ kernel = _TRITON_STATE["kernel"]+ if kernel is None:+ kernel = _TRITON_STATE["kernel"] = _build_kernel()+ rows, cols = codes.shape+ grid = (rows, triton.cdiv(cols, _TRITON_BLOCK))+ # No FMA contraction: the multiply rounds once, as torch's.+ kernel[grid](+ codes,+ scale,+ out,+ cols,+ FMT = _FMT_ID[fmt],+ BF16 = out.dtype == torch.bfloat16,+ BF16_NAN = _bf16_nan(out.device) if out.dtype == torch.bfloat16 else 0,+ BLOCK = _TRITON_BLOCK,+ num_warps = 4,+ enable_fp_fusion = False,+ )+++def _triton_ok(codes: Any, dtype: Any) -> bool:+ import torch++ if _TRITON_STATE["failed"] is not None or not codes.is_cuda or not codes.is_contiguous():+ return False+ if dtype not in (torch.bfloat16, torch.float16, torch.float32) or codes.dim() != 2:+ return False+ rows, cols = codes.shape+ if not (0 < rows < (1 << 31) and 0 < cols and -(-cols // _TRITON_BLOCK) < 65536):+ return False+ # Under CUDA-graph capture only an already compiled specialisation (no JIT, no NaN probe sync).+ return not torch.cuda.is_current_stream_capturing() or _warm_key(codes, dtype) in _WARM+++def _warm_key(codes: Any, dtype: Any) -> tuple:+ cols = codes.shape[1]+ return (str(codes.device), codes.dtype, dtype, cols % 16 == 0, cols == 1)+++def _triton_failed(exc: BaseException) -> None:+ import logging+ _TRITON_STATE["failed"] = repr(exc)+ logging.getLogger(__name__).debug(+ "diffusion.fp8_dequant: Triton widen unavailable (%s); chunked torch from here on", exc+ )+++def dequant_rows(+ codes: Any,+ scale: Any,+ dtype: Any,+ widen: Callable[[Any], Any],+ fmt: Optional[str] = None,+ triton_ok: bool = True,+) -> Any:+ """``(widen(codes) * scale[:, None]).to(dtype)`` for 2-D ``codes`` and an fp32 ``scale`` of shape [rows].+ ``widen`` maps a row slice to its exact float32 values; ``fmt`` names the fp8 bits (None: integer codes)."""+ import torch++ if triton_ok and _triton_ok(codes, dtype):+ out = torch.empty(codes.shape, dtype = dtype, device = codes.device)+ try:+ launch_triton(codes, scale.contiguous(), out, fmt)+ _WARM.add(_warm_key(codes, dtype))+ return out+ except Exception as exc: # noqa: BLE001 - no Triton / launch failure: chunked torch below+ _triton_failed(exc)+ del out+ rows = codes.shape[0]+ cols = codes.numel() // max(1, rows)+ step = chunk_rows(rows, cols, codes.device, 8 if codes.device.type == "mps" else 4)+ if not step:+ return (widen(codes) * scale[:, None]).to(dtype)+ out = torch.empty(codes.shape, dtype = dtype, device = codes.device)+ for start in range(0, rows, step):+ stop = min(rows, start + step)+ wide = widen(codes[start:stop])+ wide.mul_(scale[start:stop, None])+ out[start:stop].copy_(wide)+ del wide+ return outdiff --git a/studio/backend/core/inference/diffusion_fp8_dequant_triton.py b/studio/backend/core/inference/diffusion_fp8_dequant_triton.pynew file mode 100644index 00000000000..8e7285c914a--- /dev/null+++ b/studio/backend/core/inference/diffusion_fp8_dequant_triton.py@@ -0,0 +1,81 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Triton kernel for ``diffusion_fp8_dequant`` (module level: a compiled kernel resolves ``tl`` via globals)."""++import triton+import triton.language as tl+++def _fp16_bits(value):+ """float32 -> float16 bits, round to nearest even by integer ops (overflow to inf, subnormals kept), as torch+ casts on every device; ROCm's own conversion differs."""+ u = value.to(tl.uint32, bitcast = True)+ sign = (u >> 16) & 0x8000+ a = u & 0x7FFFFFFF+ r = a - (112 << 23)+ normal = (r + 0x0FFF + ((r >> 13) & 1)) >> 13+ # below 2**-14: the subnormal count, |x| * 2**24 rounded to the nearest even integer (exact scaling, one add)+ scaled = (a.to(tl.float32, bitcast = True) * 16777216.0 + 8388608.0) - 8388608.0+ sub = scaled.to(tl.uint32)+ bits = tl.where(a >= 0x38800000, normal, sub)+ bits = tl.where(a >= 0x477FF000, 0x7C00, bits)+ # NaN: quieted, top payload bits kept, as the hardware conversion torch uses+ bits = tl.where(a > 0x7F800000, 0x7E00 | ((a >> 13) & 0x3FF), bits)+ return (bits | sign).to(tl.uint16)+++def dequant_rows_kernel(+ codes_ptr,+ scale_ptr,+ out_ptr,+ cols,+ FMT: tl.constexpr,+ BF16: tl.constexpr,+ BF16_NAN: tl.constexpr,+ BLOCK: tl.constexpr,+):+ row = tl.program_id(0).to(tl.int64)+ col = tl.program_id(1).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)+ mask = col < cols+ offs = row * cols + col+ s = tl.load(scale_ptr + row)+ c = tl.load(codes_ptr + offs, mask = mask, other = 0)+ if FMT == 0:+ wide = c.to(tl.float32)+ else:+ # fp8_bits_to_float32 by integer ops (no fp8 type: sm75, ROCm); NaN bits as torch's own cast.+ b = c.to(tl.int32)+ if FMT == 1:+ e = (b >> 3) & 15+ m = b & 7+ normal = ((e + 120) << 23) | (m << 20)+ sub = (m.to(tl.float32) * 0.001953125).to(tl.int32, bitcast = True)+ special = (e == 15) & (m == 7)+ special_bits = tl.full(b.shape, 0x7FF00000, tl.int32)+ else:+ e = (b >> 2) & 31+ m = b & 3+ normal = ((e + 112) << 23) | (m << 21)+ sub = (m.to(tl.float32) * 0.0000152587890625).to(tl.int32, bitcast = True)+ special = e == 31+ special_bits = 0x7F800000 | (m << 21) | tl.where(m != 0, 0x00400000, 0)+ mag = tl.where(e == 0, sub, normal)+ mag = tl.where(special, special_bits, mag)+ wide = (mag | ((b & 0x80) << 24)).to(tl.float32, bitcast = True)+ prod = wide * s+ if BF16:+ # nearest even by integer ops (the interpreter truncates); NaN as torch's cast on this device+ bits = prod.to(tl.int32, bitcast = True)+ rne = (bits + 0x7FFF + ((bits >> 16) & 1)) >> 16+ out = tl.where((bits & 0x7FFFFFFF) > 0x7F800000, BF16_NAN, rne).to(tl.int16)+ out = out.to(tl.bfloat16, bitcast = True)+ else:+ if out_ptr.dtype.element_ty == tl.float16:+ # nearest even by integer ops: ROCm's own fp32 -> fp16 conversion differs from torch's+ out = _fp16_bits(prod).to(tl.float16, bitcast = True)+ else:+ out = prod.to(out_ptr.dtype.element_ty)+ tl.store(out_ptr + offs, out, mask = mask)diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.pyindex 3a0ecf79057..bda53720da7 100644--- a/studio/backend/core/inference/diffusion_native_quant.py+++ b/studio/backend/core/inference/diffusion_native_quant.py@@ -14,6 +14,8 @@ import timefrom functools import lru_cachefrom typing import Any, Callable, Optional+from .diffusion_fp8_dequant import dequant_rows, fused_dequant_enabled+NATIVE_INT8 = "int8"NATIVE_FP8 = "fp8"_QMAX = {NATIVE_INT8: 127.0, NATIVE_FP8: 448.0}@@ -509,8 +511,25 @@ def native_linear_class():self._row_major = _row_major_op() if self.act_int8 and not self.rot_group else Noneself._int8pack_scales: dict = {}+ def _widen(self, codes: Any) -> Any:+ """Exact float32 values of a row slice of ``weight_q``, as the eager branches below."""+ if self.scheme != NATIVE_FP8:+ return codes.to(torch.float32)+ if codes.device.type == "mps":+ lut = fp8_lookup_table(self.fp8_format, str(codes.device))+ return lut.index_select(0, codes.reshape(-1).int()).view(codes.shape)+ return codes.view(getattr(torch, self.fp8_format)).to(torch.float32)+def _stored_weight(self, dtype: Any) -> Any:scale = self.weight_scale.view(torch.float32)+ if not torch.compiler.is_compiling() and fused_dequant_enabled():+ return dequant_rows(+ self.weight_q,+ scale,+ dtype,+ self._widen,+ self.fp8_format if self.scheme == NATIVE_FP8 else None,+ )if self.scheme != NATIVE_FP8:wide = self.weight_q.to(torch.float32)elif torch.compiler.is_compiling():diff --git a/studio/backend/tests/test_diffusion_fp8_dequant.py b/studio/backend/tests/test_diffusion_fp8_dequant.pynew file mode 100644index 00000000000..1266886bcce--- /dev/null+++ b/studio/backend/tests/test_diffusion_fp8_dequant.py@@ -0,0 +1,441 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Eager int8 / stored-fp8 widening without full float32 temporaries: bitwise equal to the whole-layer formula on+the chunked torch path and the Triton kernel (under TRITON_INTERPRET=1), kill switch, compile unchanged."""++from __future__ import annotations++import json+import os+import subprocess+import sys+import textwrap++import pytest++torch = pytest.importorskip("torch")++import core.inference.diffusion_comfy_quant as cq # noqa: E402+import core.inference.diffusion_native_quant as nq # noqa: E402++# The tree ``core`` was imported from (the subprocess imports the same one).+BACKEND = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(nq.__file__))))+ENV = "UNSLOTH_DIFFUSION_FP8_DEQUANT_FUSED"+FP8 = ("float8_e4m3fn", "float8_e5m2")+DTYPES = (torch.bfloat16, torch.float16, torch.float32)+# 1, extremes, the e4m3 max-ish, non-powers of two (rounding), a subnormal scale, a negative one (signed zero)+SCALES = (1.0, 2.0**-20, 2.0**20, 448.0, 1 / 448, 0.7, 3.3e-3, 1e-40, 2.0**-140, -1.5, 2.0**100)+++def _fd():+ import core.inference.diffusion_fp8_dequant as fd+ return fd+++def test_imports_own_tree():+ """The modules under test are this test's own tree."""+ root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))+ for mod in (nq, cq):+ assert os.path.realpath(mod.__file__).startswith(root + os.sep), mod.__file__+++def _old(+ codes,+ scale,+ dtype,+ fmt = None,+):+ """Today's eager formula, independent of the tree."""+ wide = (+ codes.to(torch.float32)+ if fmt is None+ else codes.view(getattr(torch, fmt)).to(torch.float32)+ )+ return (wide * scale[:, None]).to(dtype)+++def _bits(t):+ return t.view({1: torch.int8, 2: torch.int16, 4: torch.int32}[t.element_size()])+++def _assert_bitwise(new, ref):+ assert new.dtype == ref.dtype and new.shape == ref.shape+ nan = torch.isnan(ref)+ assert torch.equal(torch.isnan(new), nan)+ assert torch.equal(_bits(new)[~nan], _bits(ref)[~nan])+++def _layer(+ codes,+ scale,+ scheme,+ dtype = torch.bfloat16,+ bias = False,+):+ lin = torch.nn.Linear(codes.shape[1], codes.shape[0], bias = bias, dtype = dtype)+ return nq.native_linear_class()(lin, scheme, codes = codes, scale = scale)+++def _exhaustive(fmt):+ codes = torch.arange(256, dtype = torch.uint8).repeat(len(SCALES), 1).contiguous()+ if fmt is None:+ codes = codes.view(torch.int8)+ return codes, torch.tensor(SCALES, dtype = torch.float32)+++def tiny_chunks(monkeypatch):+ """One row per chunk on CPU, so every row runs the chunked path."""+ monkeypatch.delenv(ENV, raising = False)+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 4)++[email protected]("fmt", (*FP8, None))[email protected]("dtype", DTYPES)+def test_every_code_and_scale_is_bitwise_on_the_chunked_path(tiny_chunks, fmt, dtype):+ codes, scale = _exhaustive(fmt)+ store = codes.view(getattr(torch, fmt)) if fmt else codes+ layer = _layer(store, scale, nq.NATIVE_FP8 if fmt else nq.NATIVE_INT8)+ ref = _old(codes, scale, dtype, fmt)+ _assert_bitwise(layer._stored_weight(dtype), ref)+ # Same per-element ops as today, so NaN bits match too.+ assert torch.equal(_bits(layer._stored_weight(dtype)), _bits(ref))++[email protected]("fmt", FP8)+def test_integer_decode_matches_torch_cast_bits(fmt):+ """The bit decode the kernel mirrors: fp8_bits_to_float32 for every value, torch's NaN bits in the kernel."""+ codes = torch.arange(256, dtype = torch.uint8)+ cast = codes.view(getattr(torch, fmt)).float()+ _assert_bitwise(nq.fp8_bits_to_float32(codes, fmt), cast)+++_INTERP = textwrap.dedent(+ """+ import json, os, sys+ os.environ["TRITON_INTERPRET"] = "1"+ sys.path.insert(0, sys.argv[1])+ import torch+ import core.inference.diffusion_fp8_dequant as fd+ assert os.path.realpath(fd.__file__).startswith(os.path.realpath(sys.argv[1]))+ scales = torch.tensor(json.loads(sys.argv[2]), dtype = torch.float32)+ res = []+ for fmt in ("float8_e4m3fn", "float8_e5m2", None):+ codes = torch.arange(256, dtype = torch.uint8).repeat(len(scales), 1).contiguous()+ if fmt is None:+ codes = codes.view(torch.int8)+ wide = codes.to(torch.float32) if fmt is None else codes.view(getattr(torch, fmt)).to(torch.float32)+ for dt in (torch.bfloat16, torch.float16, torch.float32):+ ref = (wide * scales[:, None]).to(dt)+ out = torch.full(codes.shape, 7, dtype = dt)+ fd.launch_triton(codes, scales, out, fmt)+ iv = {2: torch.int16, 4: torch.int32}[out.element_size()]+ nan = torch.isnan(ref)+ res.append(dict(+ fmt = fmt, dtype = str(dt),+ nan_pos = bool(torch.equal(torch.isnan(out), nan)),+ values = bool(torch.equal(out.view(iv)[~nan], ref.view(iv)[~nan])),+ nan_bits_equal = bool(torch.equal(out.view(iv)[nan], ref.view(iv)[nan])),+ n_nan = int(nan.sum()),+ n = int(ref.numel()),+ ))+ # Masked tail and int64 offsets: odd widths.+ for cols in (1, 3, 1023, 1025):+ codes = torch.randint(0, 256, (5, cols), dtype = torch.uint8)+ s = torch.rand(5) + 0.1+ ref = (codes.view(torch.float8_e4m3fn).float() * s[:, None]).to(torch.bfloat16)+ out = torch.empty(codes.shape, dtype = torch.bfloat16)+ fd.launch_triton(codes, s, out, "float8_e4m3fn")+ nan = torch.isnan(ref)+ res.append(dict(fmt = "tail", dtype = str(cols), nan_pos = bool(torch.equal(torch.isnan(out), nan)),+ values = bool(torch.equal(out.view(torch.int16)[~nan], ref.view(torch.int16)[~nan])),+ nan_bits_equal = None, n_nan = int(nan.sum()), n = int(ref.numel())))+ print("RESULT " + json.dumps(res))+ """+)+++def test_triton_kernel_is_bitwise_under_the_interpreter():+ pytest.importorskip("triton")+ env = dict(os.environ, TRITON_INTERPRET = "1", CUDA_VISIBLE_DEVICES = "")+ proc = subprocess.run(+ [sys.executable, "-c", _INTERP, BACKEND, json.dumps(SCALES)],+ capture_output = True,+ text = True,+ env = env,+ timeout = 600,+ )+ line = [l for l in proc.stdout.splitlines() if l.startswith("RESULT ")]+ if not line and "interpret" in proc.stderr.lower() and "not supported" in proc.stderr.lower():+ # skip because this Triton build has no interpreter for the op; the GPU check covers the kernel+ pytest.skip(f"Triton interpreter cannot run here: {proc.stderr[-400:]}")+ assert line, proc.stderr[-4000:]+ res = json.loads(line[0][len("RESULT ") :])+ print(json.dumps(res, indent = 1))+ for r in res:+ assert r["nan_pos"] and r["values"], r+ # NaN bits too, against CPU torch (the bfloat16 NaN constant is chosen per backend).+ for r in res:+ if r["fmt"] != "tail":+ assert r["nan_bits_equal"], r++[email protected]("rows", (1, 17, 1023, 1024, 1025))[email protected]("fmt", ("float8_e4m3fn", None))+def test_odd_row_counts_over_several_chunks(monkeypatch, rows, fmt):+ monkeypatch.delenv(ENV, raising = False)+ cols = 64+ # 100 rows per chunk: 1025 rows = 11 chunks with a 25-row tail.+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 100 * cols * 4)+ if rows * cols * 8 <= 100 * cols * 4:+ assert _fd().chunk_rows(rows, cols, torch.device("cpu")) == 0+ else:+ assert _fd().chunk_rows(rows, cols, torch.device("cpu")) == 100+ g = torch.Generator().manual_seed(rows)+ codes = torch.randint(0, 256, (rows, cols), dtype = torch.uint8, generator = g)+ if fmt is None:+ codes = codes.view(torch.int8)+ scale = torch.rand(rows, generator = g) * 0.01 + 1e-4+ store = codes.view(getattr(torch, fmt)) if fmt else codes+ layer = _layer(store, scale, nq.NATIVE_FP8 if fmt else nq.NATIVE_INT8)+ for dtype in DTYPES:+ _assert_bitwise(layer._stored_weight(dtype), _old(codes, scale, dtype, fmt))+++def test_kill_switch_takes_the_old_path(monkeypatch):+ calls = []+ real = nq.dequant_rows++ def spy(*a, **k):+ calls.append(1)+ return real(*a, **k)++ monkeypatch.setattr(nq, "dequant_rows", spy)+ codes, scale = _exhaustive("float8_e4m3fn")+ layer = _layer(codes.view(torch.float8_e4m3fn), scale, nq.NATIVE_FP8)+ monkeypatch.setenv(ENV, "0")+ old = layer._stored_weight(torch.bfloat16)+ assert calls == []+ monkeypatch.setenv(ENV, "1")+ new = layer._stored_weight(torch.bfloat16)+ assert calls == [1]+ assert torch.equal(_bits(old), _bits(new))++ seen = []+ monkeypatch.setattr(cq, "dequant_rows", lambda *a, **k: seen.append(1) or real(*a, **k))+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 4)+ monkeypatch.setenv(ENV, "0")+ cq._dequant(codes.view(torch.float8_e4m3fn), scale, 0, torch.bfloat16)+ assert seen == []+ monkeypatch.setenv(ENV, "1")+ cq._dequant(codes.view(torch.float8_e4m3fn), scale, 0, torch.bfloat16)+ assert seen == [1]+++def test_triton_failure_latches_and_falls_back(monkeypatch):+ """A launch failure never reaches the caller: chunked torch, and Triton is not tried again."""+ fd = _fd()+ tries = []++ def boom(*a, **k):+ tries.append(1)+ raise RuntimeError("no triton")++ monkeypatch.setattr(fd, "launch_triton", boom)+ monkeypatch.setattr(fd, "_TRITON_STATE", {"kernel": None, "failed": None})+ monkeypatch.setattr(fd, "_triton_ok", lambda c, d: fd._TRITON_STATE["failed"] is None)+ codes, scale = _exhaustive("float8_e5m2")+ widen = lambda c: c.view(torch.float8_e5m2).to(torch.float32)+ for _ in range(2):+ out = fd.dequant_rows(codes, scale, torch.float16, widen, "float8_e5m2")+ _assert_bitwise(out, _old(codes, scale, torch.float16, "float8_e5m2"))+ assert tries == [1] and fd._TRITON_STATE["failed"]+++class _Fp32Full(torch.utils._python_dispatch.TorchDispatchMode):+ """Counts op outputs that are full-size float32 [rows, cols] matrices."""++ def __init__(self, numel):+ super().__init__()+ self.numel, self.hits = numel, 0++ def __torch_dispatch__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ out = func(*args, **(kwargs or {}))+ for t in out if isinstance(out, (tuple, list)) else (out,):+ if isinstance(t, torch.Tensor) and t.dtype == torch.float32 and t.numel() >= self.numel:+ self.hits += 1+ return out++[email protected]("scheme", (nq.NATIVE_FP8, nq.NATIVE_INT8))+def test_eager_widen_holds_no_full_fp32_matrix(monkeypatch, scheme):+ """Base: the float32 codes and their float32 product (2 full matrices). Head: one bounded chunk."""+ monkeypatch.delenv(ENV, raising = False)+ rows, cols = 2048, 4096 # 32 MiB of float32 per matrix, over the 16 MiB CPU budget+ g = torch.Generator().manual_seed(0)+ raw = torch.randint(-127, 128, (rows, cols), dtype = torch.int8, generator = g)+ codes = raw.view(torch.uint8).view(torch.float8_e4m3fn) if scheme == nq.NATIVE_FP8 else raw+ if scheme == nq.NATIVE_FP8:+ codes = torch.where(torch.isnan(codes.float()), 0, codes.float()).to(torch.float8_e4m3fn)+ layer = _layer(codes, torch.rand(rows) + 0.5, scheme)+ with _Fp32Full(rows * cols) as mode:+ layer._stored_weight(torch.bfloat16)+ assert mode.hits == 0+ monkeypatch.setenv(ENV, "0")+ with _Fp32Full(rows * cols) as mode:+ layer._stored_weight(torch.bfloat16)+ assert mode.hits == 2++[email protected]("fmt", (*FP8, "int8"))[email protected]("per_row", (True, False))+def test_comfy_dequant_chunked_is_bitwise_and_charges_the_true_peak(monkeypatch, fmt, per_row):+ monkeypatch.delenv(ENV, raising = False)+ import core.inference.diffusion_single_file_converters as sfc++ charged = []+ monkeypatch.setattr(sfc, "charge_host_ram", lambda n: charged.append(int(n)))+ rows, cols = 1025, 96+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 64 * cols * 4)+ g = torch.Generator().manual_seed(1)+ raw = torch.randint(0, 256, (rows, cols), dtype = torch.uint8, generator = g)+ codes = raw.view(torch.int8) if fmt == "int8" else raw.view(getattr(torch, fmt))+ scale = (torch.rand(rows if per_row else 1, generator = g) * 0.02).to(torch.float32)+ for dtype in DTYPES:+ charged.clear()+ new = cq._dequant(codes, scale, 0, dtype)+ new_charge = charged[-1]+ monkeypatch.setenv(ENV, "0")+ charged.clear()+ old = cq._dequant(codes, scale, 0, dtype)+ old_charge = charged[-1]+ monkeypatch.delenv(ENV)+ ref = (codes.to(torch.float32) * scale.reshape(-1, 1).expand(rows, 1)).to(dtype)+ _assert_bitwise(new, ref)+ assert torch.equal(_bits(new), _bits(old))+ out_size = torch.empty((), dtype = dtype).element_size()+ assert new_charge == rows * cols * out_size + 64 * cols * 4+ assert old_charge == rows * cols * max(8, 4 + out_size) > new_charge+++def test_comfy_convrot_dequant_is_unchanged(monkeypatch):+ """ConvRot (group > 0) is not row-chunked: same expression, same charge, either setting."""+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 4)+ import core.inference.diffusion_single_file_converters as sfc++ charged = []+ monkeypatch.setattr(sfc, "charge_host_ram", lambda n: charged.append(int(n)))+ codes = torch.randint(-127, 128, (33, 64), dtype = torch.int8)+ scale = torch.rand(33)+ monkeypatch.setenv(ENV, "0")+ old = cq._dequant(codes, scale, 16, torch.bfloat16)+ monkeypatch.setenv(ENV, "1")+ new = cq._dequant(codes, scale, 16, torch.bfloat16)+ assert torch.equal(_bits(old), _bits(new))+ assert charged == [33 * 64 * 12] * 2++[email protected]("scheme", (nq.NATIVE_FP8, nq.NATIVE_INT8))+def test_forward_is_bitwise_new_vs_old(monkeypatch, scheme):+ monkeypatch.setattr(_fd(), "CHUNK_SCRATCH_BYTES_CPU", 8 * 48 * 4)+ g = torch.Generator().manual_seed(2)+ lin = torch.nn.Linear(48, 40, bias = True, dtype = torch.bfloat16)+ layer = nq.native_linear_class()(lin, scheme)+ x = torch.randn(3, 5, 48, generator = g).to(torch.bfloat16)+ monkeypatch.setenv(ENV, "1")+ new = layer(x)+ w_new = layer.weight+ monkeypatch.setenv(ENV, "0")+ old = layer(x)+ w_old = layer.weight+ assert torch.equal(_bits(new), _bits(old))+ assert torch.equal(_bits(w_new), _bits(w_old))+++def _compile_counts(layer, x, backend):+ from torch._dynamo.testing import CompileCounterWithBackend++ torch._dynamo.reset()+ cnt = CompileCounterWithBackend(backend)+ fn = torch.compile(layer, backend = cnt, fullgraph = True)+ out = fn(x)+ out2 = fn(x)+ return cnt.frame_count, cnt.op_count, out, out2++[email protected]("backend", ("aot_eager", "inductor"))+def test_compiled_stored_fp8_layer_graph_is_unchanged(monkeypatch, backend):+ """The compiled branch is today's: one fullgraph frame, the same op count with the switch on or off."""+ lin = torch.nn.Linear(64, 32, bias = True, dtype = torch.float32)+ layer = nq.native_linear_class()(lin, nq.NATIVE_FP8)+ x = torch.randn(4, 64)+ counts = {}+ for flag in ("1", "0"):+ monkeypatch.setenv(ENV, flag)+ try:+ frames, ops, out, out2 = _compile_counts(layer, x, backend)+ except Exception as exc: # noqa: BLE001 - no C++ toolchain for CPU inductor+ if backend == "inductor":+ # skip because CPU inductor needs a C++ toolchain; the aot_eager leg still runs+ pytest.skip(f"CPU inductor unavailable: {exc!r}"[:300])+ raise+ assert frames == 1+ assert torch.equal(out, out2)+ counts[flag] = (frames, ops)+ print("compile counts", backend, counts)+ assert counts["1"] == counts["0"]+ torch._dynamo.reset()+++def test_cuda_graph_capture_takes_only_a_warm_kernel(monkeypatch):+ """No first-launch JIT / NaN probe sync inside a capture: chunked torch until the specialisation ran eager."""+ import types++ fd = _fd()+ monkeypatch.setattr(fd, "_TRITON_STATE", {"kernel": None, "failed": None})+ monkeypatch.setattr(fd, "_WARM", set())+ codes = types.SimpleNamespace(+ is_cuda = True,+ is_contiguous = lambda: True,+ dim = lambda: 2,+ shape = (4, 32),+ device = "cuda:0",+ dtype = torch.uint8,+ )+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)+ assert fd._triton_ok(codes, torch.bfloat16)+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)+ assert not fd._triton_ok(codes, torch.bfloat16)+ fd._WARM.add(fd._warm_key(codes, torch.bfloat16))+ assert fd._triton_ok(codes, torch.bfloat16)+ assert not fd._triton_ok(codes, torch.float16)+++ not torch.cuda.is_available(), reason = "needs a CUDA / ROCm GPU to run the Triton widen"+)[email protected]("fmt", (*FP8, None))[email protected]("dtype", DTYPES)+def test_the_triton_widen_on_the_gpu_matches_the_old_formula(monkeypatch, fmt, dtype):+ monkeypatch.delenv(ENV, raising = False)+ codes, scale = _exhaustive(fmt)+ codes = torch.cat(+ [codes] * 8+ ).contiguous() # several rows per scale, every column block masked or full+ scale = torch.cat([scale] * 8)+ store = codes.view(getattr(torch, fmt)) if fmt else codes+ layer = _layer(store, scale, nq.NATIVE_FP8 if fmt else nq.NATIVE_INT8).to("cuda")+ launches = []+ real = _fd().launch_triton+ monkeypatch.setattr(_fd(), "launch_triton", lambda *a: launches.append(1) or real(*a))+ got = layer._stored_weight(dtype).cpu()+ assert launches and _fd()._TRITON_STATE["failed"] is None, _fd()._TRITON_STATE["failed"]+ ref = _old(codes, scale, dtype, fmt)+ _assert_bitwise(got, ref)diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.pyindex 341f8a4d33b..f9031f9c353 100644--- a/studio/backend/tests/test_diffusion_native_quant.py+++ b/studio/backend/tests/test_diffusion_native_quant.py@@ -1340,3 +1340,47 @@ def test_blocked_torchao_holds_after_peft_cached_it_available(monkeypatch):) # newer peft's dispatcher imports this one insteadmodel = _toy()peft.inject_adapter_in_model(peft.LoraConfig(r = 4, target_modules = ["to_q"]), model)++[email protected]("scheme", [nq.NATIVE_FP8, nq.NATIVE_INT8])+def test_the_eager_stored_widen_allocates_no_full_float32_matrix(scheme):+ """A stored fp8 / int8 layer's eager widen: the same bf16 weight, with no float32 temporary the size of the+ whole layer (the old widen held two)."""+ from torch.utils._python_dispatch import TorchDispatchMode++ rows, cols = 2048, 4096+ gen = torch.Generator().manual_seed(0)+ codes = torch.randint(-127, 128, (rows, cols), dtype = torch.int8, generator = gen)+ if scheme == nq.NATIVE_FP8:+ codes = torch.randint(0, 0x7F, (rows, cols), dtype = torch.uint8, generator = gen).view(+ torch.float8_e4m3fn+ )+ scale = torch.rand(rows, generator = gen) * 1e-2+ lin = torch.nn.Linear(cols, rows, bias = False, dtype = torch.bfloat16)+ layer = nq.native_linear_class()(lin, scheme, codes = codes, scale = scale)+ want = (codes.to(torch.float32) * scale.reshape(-1, 1)).to(torch.bfloat16)++ class FullFloat32(TorchDispatchMode):+ count = 0++ def __torch_dispatch__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ out = func(*args, **(kwargs or {}))+ if (+ torch.is_tensor(out)+ and out.dtype == torch.float32+ and out.numel() >= rows * cols+ and func is not torch.ops.aten.view.default+ ):+ FullFloat32.count += 1+ return out++ with torch.no_grad(), FullFloat32():+ got = layer._stored_weight(torch.bfloat16)+ assert torch.equal(got.view(torch.int16), want.view(torch.int16))+ assert FullFloat32.count == 0, FullFloat32.count