Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio GGUF: fuse Q8_0 dequant and cast into one kernel (LTX-2.5 Q8_0 denoise -12%, bit-identical) (#16884)

Daniel Hana0197ca266e1
diff --git a/studio/backend/core/inference/diffusion_gguf_compile.py b/studio/backend/core/inference/diffusion_gguf_compile.py
index a04aa8d9728..e8ef756c664 100644
--- a/studio/backend/core/inference/diffusion_gguf_compile.py
+++ b/studio/backend/core/inference/diffusion_gguf_compile.py
@@ -3,7 +3,12 @@
"""Compiles GGUF dequant with dynamic=True for the default tier and for max where the block cannot
compile; never where max compiles the block (would break it); torch and diffusers imported
-lazily; UNSLOTH_DIFFUSION_GGUF_COMPILE_DEQUANT=0 disables."""
+lazily; UNSLOTH_DIFFUSION_GGUF_COMPILE_DEQUANT=0 disables.
+
+Q8_0 GGUFLinear weights also get one fused dequant+cast kernel (UNSLOTH_DIFFUSION_GGUF_FUSED_Q8_0=0 disables): the
+stock forward runs the compiled dequant (fp16 out, several kernels) and then an eager ``.to(compute_dtype)`` copy per
+linear per forward. The fused kernel keeps the fp16 rounding of ``d * q`` before the cast (see ``_q8_0_blocks_dequant_cast``),
+so the weight it produces is bit-identical to the stock one."""
from __future__ import annotations
@@ -36,6 +41,44 @@ _compiled_dequant_installed = False
_compile_error: str | None = None
_DEQUANT_ATTR = "dequantize_gguf_tensor"
+_ENV_FUSED_Q8_0 = "UNSLOTH_DIFFUSION_GGUF_FUSED_Q8_0"
+_FORWARD_ATTR = "forward_native"
+_Q8_0 = 8 # gguf.GGMLQuantizationType.Q8_0
+_Q8_0_TYPE_SIZE = 34 # one fp16 scale + 32 int8 codes per block
+_fused_q8_0_installed = False
+
+
+def _q8_0_blocks_dequant_cast(raw: Any, dtype: Any) -> Any:
+ """Q8_0 bytes (any shape, ``n_blocks * 34`` bytes) -> ``(n_blocks, 32)`` weight in ``dtype``. Same ops and order
+ as diffusers' ``dequantize_blocks_Q8_0`` (fp16 scale * int8 code -> fp16) followed by ``.to(dtype)``; the dtype
+ views are taken on the contiguous byte tensor before slicing, so no slice is materialised.
+
+ Inductor computes a fused fp16 product in fp32 and would round it once to ``dtype`` (bf16: a different result
+ than fp16 round then bf16 round, and no fp16 overflow to inf). The int16 bitcast forces the fp16 store rounding
+ inside the kernel; the ``& -1`` keeps inductor from folding the bitcast pair away. Eager it is a no-op. The
+ result stays in the block layout: a reshape to the symbolic row width inside the graph costs a second copy."""
+ import torch # noqa: PLC0415
+
+ blocks = raw.reshape(-1, _Q8_0_TYPE_SIZE)
+ d = blocks.view(torch.int16)[:, :1].view(torch.float16)
+ x = blocks.view(torch.int8)[:, 2:]
+ half_bits = (d * x).view(torch.int16) & -1
+ return half_bits.view(torch.float16).to(dtype)
+
+
+def _q8_0_dequant_cast(
+ raw: Any,
+ dtype: Any,
+ blocks_fn: Any = None,
+) -> Any:
+ """Q8_0 bytes ``(rows, n_blocks * 34)`` -> ``(rows, n_blocks * 32)`` weight in ``dtype``; ``blocks_fn`` is the
+ (compiled) block dequant, eager by default. The final reshape is a free view of the contiguous block output."""
+ return (blocks_fn or _q8_0_blocks_dequant_cast)(raw, dtype).reshape(raw.shape[0], -1)
+
+
+def is_fused_q8_0_installed() -> bool:
+ return _fused_q8_0_installed
+
def is_compiled_dequant_installed() -> bool:
return _compiled_dequant_installed
@@ -67,6 +110,7 @@ def install_compiled_dequant(logger: Any = None) -> bool:
if apply_patch(gguf_utils, _DEQUANT_ATTR, compiled, force = True):
_compiled_dequant_installed = True
_compile_error = None
+ _install_fused_q8_0(gguf_utils, logger)
return True
return False
except Exception as exc: # noqa: BLE001 - optimisation only
@@ -108,12 +152,63 @@ def _guard(compiled: Any, eager: Any, logger: Any) -> Any:
return guarded
+def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:
+ """Patch ``GGUFLinear.forward_native`` so a Q8_0 weight is dequantised and cast in one compiled kernel; every
+ other weight (other quant types, no compute dtype) takes the stock forward unchanged."""
+ global _fused_q8_0_installed
+ if _fused_q8_0_installed or not _enabled(_ENV_FUSED_Q8_0):
+ return _fused_q8_0_installed
+ linear_cls = getattr(gguf_utils, "GGUFLinear", None)
+ if linear_cls is None or not callable(getattr(linear_cls, _FORWARD_ATTR, None)):
+ return False
+ try:
+ import torch # noqa: PLC0415
+
+ compiled = torch.compile(_q8_0_blocks_dequant_cast, dynamic = True)
+ blocks_fn = _guard(compiled, _q8_0_blocks_dequant_cast, logger)
+ stock_forward = getattr(linear_cls, _FORWARD_ATTR)
+ F = torch.nn.functional
+
+ @functools.wraps(stock_forward)
+ def forward_native(self: Any, inputs: Any) -> Any:
+ weight = self.weight
+ dtype = self.compute_dtype
+ if (
+ getattr(weight, "quant_type", None) != _Q8_0
+ or dtype is None
+ or weight.dim() != 2
+ or weight.shape[-1] % _Q8_0_TYPE_SIZE
+ ):
+ return stock_forward(self, inputs)
+ dense = _q8_0_dequant_cast(weight.as_tensor(), dtype, blocks_fn)
+ bias = self.bias.to(dtype) if self.bias is not None else None
+ return F.linear(inputs, dense, bias)
+
+ if apply_patch(linear_cls, _FORWARD_ATTR, forward_native, force = True):
+ _fused_q8_0_installed = True
+ return _fused_q8_0_installed
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "install fused Q8_0 dequant", exc)
+ return False
+
+
+def _uninstall_fused_q8_0(gguf_utils: Any) -> None:
+ global _fused_q8_0_installed
+ if not _fused_q8_0_installed:
+ return
+ linear_cls = getattr(gguf_utils, "GGUFLinear", None) if gguf_utils is not None else None
+ if linear_cls is not None:
+ revert_patch(linear_cls, _FORWARD_ATTR)
+ _fused_q8_0_installed = False
+
+
def uninstall_compiled_dequant() -> None:
- """Restore the original ``dequantize_gguf_tensor``. Idempotent."""
+ """Restore the original ``dequantize_gguf_tensor`` and ``GGUFLinear.forward_native``. Idempotent."""
global _compiled_dequant_installed
if not _compiled_dequant_installed:
return
gguf_utils = _gguf_utils()
+ _uninstall_fused_q8_0(gguf_utils)
if gguf_utils is not None:
revert_patch(gguf_utils, _DEQUANT_ATTR)
_compiled_dequant_installed = False
diff --git a/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py
new file mode 100644
index 00000000000..273894f25ae
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py
@@ -0,0 +1,185 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Fused Q8_0 dequant+cast for GGUFLinear: bit-identical to the stock dequant then cast, no graph breaks or
+recompiles, other quant types untouched, reversible. Runs on CPU."""
+
+from __future__ import annotations
+
+import pytest
+
+torch = pytest.importorskip("torch")
+np = pytest.importorskip("numpy")
+gguf = pytest.importorskip("gguf")
+gguf_utils = pytest.importorskip("diffusers.quantizers.gguf.utils")
+
+from core.inference import diffusion_gguf_compile as gc # noqa: E402
+
+Q8_0 = gguf.GGMLQuantizationType.Q8_0
+Q4_K = gguf.GGMLQuantizationType.Q4_K
+
+
[email protected](autouse = True)
+def _clean():
+ # Other Studio tests record inductor knobs (emulate_precision_casts among them) that the guarded compile
+ # re-applies; start every test from torch's defaults so results do not depend on test order.
+ from core.inference import diffusion_compile_config as cc
+
+ cc._reset_for_tests()
+ gc.uninstall_all()
+ torch._dynamo.reset()
+ with torch._inductor.config.patch(emulate_precision_casts = False):
+ yield
+ gc.uninstall_all()
+ cc._reset_for_tests()
+
+
+def _q8_0_bytes(
+ rows,
+ cols,
+ seed = 0,
+ adversarial = False,
+):
+ """Real Q8_0 blocks from gguf's own quantiser, or (adversarial) random bytes with the scale drawn over the whole
+ fp16 range, so fp16 overflow / subnormal / tie cases of the fp16 product are exercised."""
+ rng = np.random.default_rng(seed)
+ if not adversarial:
+ w = (rng.standard_normal((rows, cols)) * 0.02).astype(np.float32)
+ return torch.from_numpy(np.ascontiguousarray(gguf.quants.quantize(w, Q8_0)))
+ raw = rng.integers(0, 256, size = (rows, cols // 32 * 34), dtype = np.uint8)
+ return torch.from_numpy(raw)
+
+
+def _param(raw, qtype = Q8_0):
+ return gguf_utils.GGUFParameter(raw, quant_type = qtype)
+
+
+def _stock_dense(
+ raw,
+ dtype,
+ qtype = Q8_0,
+):
+ # diffusers' own eager path: GGUFLinear.forward_native's dequantize_gguf_tensor(weight).to(compute_dtype).
+ return gguf_utils.dequantize_gguf_tensor(_param(raw, qtype)).to(dtype)
+
+
+def _bits(t):
+ return t.view(
+ {torch.bfloat16: torch.int16, torch.float16: torch.int16, torch.float32: torch.int32}[
+ t.dtype
+ ]
+ )
+
+
+ "emulate", [False, True], ids = ["inductor_default", "emulate_precision_casts"]
+)
[email protected]("adversarial", [False, True], ids = ["gguf_quantized", "random_bytes"])
[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])
+def test_fused_q8_0_is_bit_identical_eager_and_compiled(adversarial, dtype, emulate):
+ # Studio sets emulate_precision_casts on some renders, so the fused kernel must be exact with and without it.
+ raw = _q8_0_bytes(96, 256, adversarial = adversarial)
+ ref = _stock_dense(raw, dtype)
+ eager = gc._q8_0_dequant_cast(raw, dtype)
+ with torch._inductor.config.patch(emulate_precision_casts = emulate):
+ compiled = gc._q8_0_dequant_cast(
+ raw, dtype, torch.compile(gc._q8_0_blocks_dequant_cast, dynamic = True)
+ )
+ assert eager.shape == ref.shape == compiled.shape
+ assert torch.equal(_bits(eager), _bits(ref))
+ assert torch.equal(_bits(compiled), _bits(ref))
+
+
+def _linear(
+ raw,
+ qtype,
+ out_f,
+ in_f,
+ dtype = torch.bfloat16,
+):
+ lin = gguf_utils.GGUFLinear(in_f, out_f, bias = True, compute_dtype = dtype)
+ lin.weight = _param(raw, qtype)
+ lin.bias = torch.nn.Parameter(torch.randn(out_f, generator = torch.Generator().manual_seed(1)))
+ return lin
+
+
+def test_naive_fusion_would_not_be_bit_identical():
+ """Guards the int16 bitcast in _q8_0_dequant_cast: without it inductor skips the fp16 rounding."""
+
+ def naive(raw, dtype):
+ blocks = raw.reshape(-1, 34)
+ d = blocks.view(torch.int16)[:, :1].view(torch.float16)
+ return (d * blocks.view(torch.int8)[:, 2:]).reshape(raw.shape[0], -1).to(dtype)
+
+ raw = _q8_0_bytes(96, 256, adversarial = True)
+ got = torch.compile(naive, dynamic = True)(raw, torch.bfloat16)
+ assert not torch.equal(_bits(got), _bits(_stock_dense(raw, torch.bfloat16)))
+
+
+def test_patched_q8_0_linear_skips_the_unfused_dequant_and_matches_stock(monkeypatch):
+ raw = _q8_0_bytes(64, 128, seed = 3)
+ lin = _linear(raw, Q8_0, 64, 128)
+ x = torch.randn(5, 128, generator = torch.Generator().manual_seed(2)).to(torch.bfloat16)
+ expected = lin.forward_native(x) # stock, nothing installed
+
+ assert gc.install_compiled_dequant() is True
+ assert gc.is_fused_q8_0_installed() is True
+ calls = []
+ compiled_dequant = gguf_utils.dequantize_gguf_tensor
+ monkeypatch.setattr(
+ gguf_utils, "dequantize_gguf_tensor", lambda t: calls.append(1) or compiled_dequant(t)
+ )
+ got = lin(x)
+ assert calls == [] # one fused kernel, not the compiled dequant plus an eager cast
+ assert torch.equal(_bits(got), _bits(expected))
+
+
+def test_other_quant_types_take_the_stock_forward(monkeypatch):
+ rng = np.random.default_rng(4)
+ block = gguf.GGML_QUANT_SIZES[Q4_K][1]
+ raw = rng.integers(0, 256, size = (16, 256 // 256 * block), dtype = np.uint8)
+ raw[:, :4] = 0x10 # small finite fp16 d / dmin
+ raw = torch.from_numpy(raw)
+ lin = _linear(raw, Q4_K, 16, 256)
+ x = torch.randn(3, 256, generator = torch.Generator().manual_seed(5)).to(torch.bfloat16)
+ stock_forward = gguf_utils.GGUFLinear.forward_native
+ assert gc.install_compiled_dequant() is True
+ expected = stock_forward(lin, x) # the stock forward over the same compiled dequant
+ calls = []
+ compiled_dequant = gguf_utils.dequantize_gguf_tensor
+ monkeypatch.setattr(
+ gguf_utils, "dequantize_gguf_tensor", lambda t: calls.append(1) or compiled_dequant(t)
+ )
+ got = lin(x)
+ assert calls == [1]
+ assert torch.equal(_bits(got), _bits(expected))
+
+
+def test_fused_q8_0_has_no_graph_breaks_and_one_dynamic_graph():
+ raw = _q8_0_bytes(32, 128)
+ explain = torch._dynamo.explain(gc._q8_0_blocks_dequant_cast)(raw, torch.bfloat16)
+ assert explain.graph_break_count == 0
+ assert explain.graph_count == 1
+
+ from torch._dynamo.testing import CompileCounter
+
+ counter = CompileCounter()
+ fn = torch.compile(gc._q8_0_blocks_dequant_cast, backend = counter, dynamic = True)
+ for rows, cols in [(32, 128), (48, 256), (64, 4096), (17, 64)]:
+ out = gc._q8_0_dequant_cast(_q8_0_bytes(rows, cols), torch.bfloat16, fn)
+ assert out.shape == (rows, cols)
+ assert counter.frame_count == 1 # new layer shapes reuse the graph: no recompiles
+
+
+def test_fused_q8_0_uninstall_and_kill_switch(monkeypatch):
+ stock = gguf_utils.GGUFLinear.forward_native
+ assert gc.install_compiled_dequant() is True
+ assert gguf_utils.GGUFLinear.forward_native is not stock
+ gc.uninstall_all()
+ assert gguf_utils.GGUFLinear.forward_native is stock
+ assert gc.is_fused_q8_0_installed() is False
+
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_FUSED_Q8_0", "0")
+ assert gc.install_compiled_dequant() is True
+ assert gc.is_fused_q8_0_installed() is False
+ assert gguf_utils.GGUFLinear.forward_native is stock