Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio GGUF: fuse Q8_0 dequant and cast into one kernel (LTX-2.5 Q8_0 denoise -12%, bit-identical) (#16884)
Daniel Hana0197ca266e1
diff --git a/studio/backend/core/inference/diffusion_gguf_compile.py b/studio/backend/core/inference/diffusion_gguf_compile.pyindex a04aa8d9728..e8ef756c664 100644--- a/studio/backend/core/inference/diffusion_gguf_compile.py+++ b/studio/backend/core/inference/diffusion_gguf_compile.py@@ -3,7 +3,12 @@"""Compiles GGUF dequant with dynamic=True for the default tier and for max where the block cannotcompile; never where max compiles the block (would break it); torch and diffusers imported-lazily; UNSLOTH_DIFFUSION_GGUF_COMPILE_DEQUANT=0 disables."""+lazily; UNSLOTH_DIFFUSION_GGUF_COMPILE_DEQUANT=0 disables.++Q8_0 GGUFLinear weights also get one fused dequant+cast kernel (UNSLOTH_DIFFUSION_GGUF_FUSED_Q8_0=0 disables): the+stock forward runs the compiled dequant (fp16 out, several kernels) and then an eager ``.to(compute_dtype)`` copy per+linear per forward. The fused kernel keeps the fp16 rounding of ``d * q`` before the cast (see ``_q8_0_blocks_dequant_cast``),+so the weight it produces is bit-identical to the stock one."""from __future__ import annotations@@ -36,6 +41,44 @@ _compiled_dequant_installed = False_compile_error: str | None = None_DEQUANT_ATTR = "dequantize_gguf_tensor"+_ENV_FUSED_Q8_0 = "UNSLOTH_DIFFUSION_GGUF_FUSED_Q8_0"+_FORWARD_ATTR = "forward_native"+_Q8_0 = 8 # gguf.GGMLQuantizationType.Q8_0+_Q8_0_TYPE_SIZE = 34 # one fp16 scale + 32 int8 codes per block+_fused_q8_0_installed = False+++def _q8_0_blocks_dequant_cast(raw: Any, dtype: Any) -> Any:+ """Q8_0 bytes (any shape, ``n_blocks * 34`` bytes) -> ``(n_blocks, 32)`` weight in ``dtype``. Same ops and order+ as diffusers' ``dequantize_blocks_Q8_0`` (fp16 scale * int8 code -> fp16) followed by ``.to(dtype)``; the dtype+ views are taken on the contiguous byte tensor before slicing, so no slice is materialised.++ Inductor computes a fused fp16 product in fp32 and would round it once to ``dtype`` (bf16: a different result+ than fp16 round then bf16 round, and no fp16 overflow to inf). The int16 bitcast forces the fp16 store rounding+ inside the kernel; the ``& -1`` keeps inductor from folding the bitcast pair away. Eager it is a no-op. The+ result stays in the block layout: a reshape to the symbolic row width inside the graph costs a second copy."""+ import torch # noqa: PLC0415++ blocks = raw.reshape(-1, _Q8_0_TYPE_SIZE)+ d = blocks.view(torch.int16)[:, :1].view(torch.float16)+ x = blocks.view(torch.int8)[:, 2:]+ half_bits = (d * x).view(torch.int16) & -1+ return half_bits.view(torch.float16).to(dtype)+++def _q8_0_dequant_cast(+ raw: Any,+ dtype: Any,+ blocks_fn: Any = None,+) -> Any:+ """Q8_0 bytes ``(rows, n_blocks * 34)`` -> ``(rows, n_blocks * 32)`` weight in ``dtype``; ``blocks_fn`` is the+ (compiled) block dequant, eager by default. The final reshape is a free view of the contiguous block output."""+ return (blocks_fn or _q8_0_blocks_dequant_cast)(raw, dtype).reshape(raw.shape[0], -1)+++def is_fused_q8_0_installed() -> bool:+ return _fused_q8_0_installed+def is_compiled_dequant_installed() -> bool:return _compiled_dequant_installed@@ -67,6 +110,7 @@ def install_compiled_dequant(logger: Any = None) -> bool:if apply_patch(gguf_utils, _DEQUANT_ATTR, compiled, force = True):_compiled_dequant_installed = True_compile_error = None+ _install_fused_q8_0(gguf_utils, logger)return Truereturn Falseexcept Exception as exc: # noqa: BLE001 - optimisation only@@ -108,12 +152,63 @@ def _guard(compiled: Any, eager: Any, logger: Any) -> Any:return guarded+def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:+ """Patch ``GGUFLinear.forward_native`` so a Q8_0 weight is dequantised and cast in one compiled kernel; every+ other weight (other quant types, no compute dtype) takes the stock forward unchanged."""+ global _fused_q8_0_installed+ if _fused_q8_0_installed or not _enabled(_ENV_FUSED_Q8_0):+ return _fused_q8_0_installed+ linear_cls = getattr(gguf_utils, "GGUFLinear", None)+ if linear_cls is None or not callable(getattr(linear_cls, _FORWARD_ATTR, None)):+ return False+ try:+ import torch # noqa: PLC0415++ compiled = torch.compile(_q8_0_blocks_dequant_cast, dynamic = True)+ blocks_fn = _guard(compiled, _q8_0_blocks_dequant_cast, logger)+ stock_forward = getattr(linear_cls, _FORWARD_ATTR)+ F = torch.nn.functional++ @functools.wraps(stock_forward)+ def forward_native(self: Any, inputs: Any) -> Any:+ weight = self.weight+ dtype = self.compute_dtype+ if (+ getattr(weight, "quant_type", None) != _Q8_0+ or dtype is None+ or weight.dim() != 2+ or weight.shape[-1] % _Q8_0_TYPE_SIZE+ ):+ return stock_forward(self, inputs)+ dense = _q8_0_dequant_cast(weight.as_tensor(), dtype, blocks_fn)+ bias = self.bias.to(dtype) if self.bias is not None else None+ return F.linear(inputs, dense, bias)++ if apply_patch(linear_cls, _FORWARD_ATTR, forward_native, force = True):+ _fused_q8_0_installed = True+ return _fused_q8_0_installed+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "install fused Q8_0 dequant", exc)+ return False+++def _uninstall_fused_q8_0(gguf_utils: Any) -> None:+ global _fused_q8_0_installed+ if not _fused_q8_0_installed:+ return+ linear_cls = getattr(gguf_utils, "GGUFLinear", None) if gguf_utils is not None else None+ if linear_cls is not None:+ revert_patch(linear_cls, _FORWARD_ATTR)+ _fused_q8_0_installed = False++def uninstall_compiled_dequant() -> None:- """Restore the original ``dequantize_gguf_tensor``. Idempotent."""+ """Restore the original ``dequantize_gguf_tensor`` and ``GGUFLinear.forward_native``. Idempotent."""global _compiled_dequant_installedif not _compiled_dequant_installed:returngguf_utils = _gguf_utils()+ _uninstall_fused_q8_0(gguf_utils)if gguf_utils is not None:revert_patch(gguf_utils, _DEQUANT_ATTR)_compiled_dequant_installed = Falsediff --git a/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.pynew file mode 100644index 00000000000..273894f25ae--- /dev/null+++ b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py@@ -0,0 +1,185 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Fused Q8_0 dequant+cast for GGUFLinear: bit-identical to the stock dequant then cast, no graph breaks or+recompiles, other quant types untouched, reversible. Runs on CPU."""++from __future__ import annotations++import pytest++torch = pytest.importorskip("torch")+np = pytest.importorskip("numpy")+gguf = pytest.importorskip("gguf")+gguf_utils = pytest.importorskip("diffusers.quantizers.gguf.utils")++from core.inference import diffusion_gguf_compile as gc # noqa: E402++Q8_0 = gguf.GGMLQuantizationType.Q8_0+Q4_K = gguf.GGMLQuantizationType.Q4_K++[email protected](autouse = True)+def _clean():+ # Other Studio tests record inductor knobs (emulate_precision_casts among them) that the guarded compile+ # re-applies; start every test from torch's defaults so results do not depend on test order.+ from core.inference import diffusion_compile_config as cc++ cc._reset_for_tests()+ gc.uninstall_all()+ torch._dynamo.reset()+ with torch._inductor.config.patch(emulate_precision_casts = False):+ yield+ gc.uninstall_all()+ cc._reset_for_tests()+++def _q8_0_bytes(+ rows,+ cols,+ seed = 0,+ adversarial = False,+):+ """Real Q8_0 blocks from gguf's own quantiser, or (adversarial) random bytes with the scale drawn over the whole+ fp16 range, so fp16 overflow / subnormal / tie cases of the fp16 product are exercised."""+ rng = np.random.default_rng(seed)+ if not adversarial:+ w = (rng.standard_normal((rows, cols)) * 0.02).astype(np.float32)+ return torch.from_numpy(np.ascontiguousarray(gguf.quants.quantize(w, Q8_0)))+ raw = rng.integers(0, 256, size = (rows, cols // 32 * 34), dtype = np.uint8)+ return torch.from_numpy(raw)+++def _param(raw, qtype = Q8_0):+ return gguf_utils.GGUFParameter(raw, quant_type = qtype)+++def _stock_dense(+ raw,+ dtype,+ qtype = Q8_0,+):+ # diffusers' own eager path: GGUFLinear.forward_native's dequantize_gguf_tensor(weight).to(compute_dtype).+ return gguf_utils.dequantize_gguf_tensor(_param(raw, qtype)).to(dtype)+++def _bits(t):+ return t.view(+ {torch.bfloat16: torch.int16, torch.float16: torch.int16, torch.float32: torch.int32}[+ t.dtype+ ]+ )+++ "emulate", [False, True], ids = ["inductor_default", "emulate_precision_casts"]+)[email protected]("adversarial", [False, True], ids = ["gguf_quantized", "random_bytes"])[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])+def test_fused_q8_0_is_bit_identical_eager_and_compiled(adversarial, dtype, emulate):+ # Studio sets emulate_precision_casts on some renders, so the fused kernel must be exact with and without it.+ raw = _q8_0_bytes(96, 256, adversarial = adversarial)+ ref = _stock_dense(raw, dtype)+ eager = gc._q8_0_dequant_cast(raw, dtype)+ with torch._inductor.config.patch(emulate_precision_casts = emulate):+ compiled = gc._q8_0_dequant_cast(+ raw, dtype, torch.compile(gc._q8_0_blocks_dequant_cast, dynamic = True)+ )+ assert eager.shape == ref.shape == compiled.shape+ assert torch.equal(_bits(eager), _bits(ref))+ assert torch.equal(_bits(compiled), _bits(ref))+++def _linear(+ raw,+ qtype,+ out_f,+ in_f,+ dtype = torch.bfloat16,+):+ lin = gguf_utils.GGUFLinear(in_f, out_f, bias = True, compute_dtype = dtype)+ lin.weight = _param(raw, qtype)+ lin.bias = torch.nn.Parameter(torch.randn(out_f, generator = torch.Generator().manual_seed(1)))+ return lin+++def test_naive_fusion_would_not_be_bit_identical():+ """Guards the int16 bitcast in _q8_0_dequant_cast: without it inductor skips the fp16 rounding."""++ def naive(raw, dtype):+ blocks = raw.reshape(-1, 34)+ d = blocks.view(torch.int16)[:, :1].view(torch.float16)+ return (d * blocks.view(torch.int8)[:, 2:]).reshape(raw.shape[0], -1).to(dtype)++ raw = _q8_0_bytes(96, 256, adversarial = True)+ got = torch.compile(naive, dynamic = True)(raw, torch.bfloat16)+ assert not torch.equal(_bits(got), _bits(_stock_dense(raw, torch.bfloat16)))+++def test_patched_q8_0_linear_skips_the_unfused_dequant_and_matches_stock(monkeypatch):+ raw = _q8_0_bytes(64, 128, seed = 3)+ lin = _linear(raw, Q8_0, 64, 128)+ x = torch.randn(5, 128, generator = torch.Generator().manual_seed(2)).to(torch.bfloat16)+ expected = lin.forward_native(x) # stock, nothing installed++ assert gc.install_compiled_dequant() is True+ assert gc.is_fused_q8_0_installed() is True+ calls = []+ compiled_dequant = gguf_utils.dequantize_gguf_tensor+ monkeypatch.setattr(+ gguf_utils, "dequantize_gguf_tensor", lambda t: calls.append(1) or compiled_dequant(t)+ )+ got = lin(x)+ assert calls == [] # one fused kernel, not the compiled dequant plus an eager cast+ assert torch.equal(_bits(got), _bits(expected))+++def test_other_quant_types_take_the_stock_forward(monkeypatch):+ rng = np.random.default_rng(4)+ block = gguf.GGML_QUANT_SIZES[Q4_K][1]+ raw = rng.integers(0, 256, size = (16, 256 // 256 * block), dtype = np.uint8)+ raw[:, :4] = 0x10 # small finite fp16 d / dmin+ raw = torch.from_numpy(raw)+ lin = _linear(raw, Q4_K, 16, 256)+ x = torch.randn(3, 256, generator = torch.Generator().manual_seed(5)).to(torch.bfloat16)+ stock_forward = gguf_utils.GGUFLinear.forward_native+ assert gc.install_compiled_dequant() is True+ expected = stock_forward(lin, x) # the stock forward over the same compiled dequant+ calls = []+ compiled_dequant = gguf_utils.dequantize_gguf_tensor+ monkeypatch.setattr(+ gguf_utils, "dequantize_gguf_tensor", lambda t: calls.append(1) or compiled_dequant(t)+ )+ got = lin(x)+ assert calls == [1]+ assert torch.equal(_bits(got), _bits(expected))+++def test_fused_q8_0_has_no_graph_breaks_and_one_dynamic_graph():+ raw = _q8_0_bytes(32, 128)+ explain = torch._dynamo.explain(gc._q8_0_blocks_dequant_cast)(raw, torch.bfloat16)+ assert explain.graph_break_count == 0+ assert explain.graph_count == 1++ from torch._dynamo.testing import CompileCounter++ counter = CompileCounter()+ fn = torch.compile(gc._q8_0_blocks_dequant_cast, backend = counter, dynamic = True)+ for rows, cols in [(32, 128), (48, 256), (64, 4096), (17, 64)]:+ out = gc._q8_0_dequant_cast(_q8_0_bytes(rows, cols), torch.bfloat16, fn)+ assert out.shape == (rows, cols)+ assert counter.frame_count == 1 # new layer shapes reuse the graph: no recompiles+++def test_fused_q8_0_uninstall_and_kill_switch(monkeypatch):+ stock = gguf_utils.GGUFLinear.forward_native+ assert gc.install_compiled_dequant() is True+ assert gguf_utils.GGUFLinear.forward_native is not stock+ gc.uninstall_all()+ assert gguf_utils.GGUFLinear.forward_native is stock+ assert gc.is_fused_q8_0_installed() is False++ monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_FUSED_Q8_0", "0")+ assert gc.install_compiled_dequant() is True+ assert gc.is_fused_q8_0_installed() is False+ assert gguf_utils.GGUFLinear.forward_native is stock