Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio GGUF: cast K-quant and legacy weights to bf16 in their own buffer (#17080)

Daniel Han46c0c8f0d81b
diff --git a/studio/backend/core/inference/diffusion_gguf_compile.py b/studio/backend/core/inference/diffusion_gguf_compile.py
index 81841a21670..173ad4de45f 100644
--- a/studio/backend/core/inference/diffusion_gguf_compile.py
+++ b/studio/backend/core/inference/diffusion_gguf_compile.py
@@ -8,7 +8,10 @@ lazily; UNSLOTH_DIFFUSION_GGUF_COMPILE_DEQUANT=0 disables.
Q8_0 GGUFLinear weights also get one fused dequant+cast kernel (UNSLOTH_DIFFUSION_GGUF_FUSED_Q8_0=0 disables): the
stock forward runs the compiled dequant (fp16 out, several kernels) and then an eager ``.to(compute_dtype)`` copy per
linear per forward. The fused kernel keeps the fp16 rounding of ``d * q`` before the cast (see ``_q8_0_blocks_dequant_cast``),
-so the weight it produces is bit-identical to the stock one."""
+so the weight it produces is bit-identical to the stock one.
+
+Other fp16-dequantised types under a bf16 compute dtype are cast in the dequant's own buffer, bit-identical to the stock
+cast (UNSLOTH_DIFFUSION_GGUF_FUSED_CAST=0 disables); a compiled-in cast was not bit-identical for Q4_K / Q5_K."""
from __future__ import annotations
@@ -50,6 +53,10 @@ _Q8_0_TYPE_SIZE = 34 # one fp16 scale + 32 int8 codes per block
_FUSED_Q8_0_MIN_TORCH = (2, 7)
_fused_q8_0_installed = False
+_ENV_FUSED_CAST = "UNSLOTH_DIFFUSION_GGUF_FUSED_CAST"
+_fused_cast_installed = False
+_forward_patched = False
+
def _q8_0_blocks_dequant_cast(raw: Any, dtype: Any) -> Any:
"""Q8_0 bytes (any shape, ``n_blocks * 34`` bytes) -> ``(n_blocks, 32)`` weight in ``dtype``. Same ops and order
@@ -69,6 +76,17 @@ def _q8_0_blocks_dequant_cast(raw: Any, dtype: Any) -> Any:
return half_bits.view(torch.float16).to(dtype)
+def _cast_in_place(half: Any, dtype: Any) -> Any:
+ """``half.to(dtype)`` for an fp16 weight and a 2-byte ``dtype``, written over ``half`` (same conversion kernel)."""
+ dense = half.view(dtype)
+ dense.copy_(half)
+ return dense
+
+
+def is_fused_cast_installed() -> bool:
+ return _fused_cast_installed
+
+
def _q8_0_dequant_cast(
raw: Any,
dtype: Any,
@@ -161,10 +179,12 @@ def _guard(compiled: Any, eager: Any, logger: Any) -> Any:
def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:
- """Patch ``GGUFLinear.forward_native`` so a Q8_0 weight is dequantised and cast in one compiled kernel; every
- other weight (other quant types, no compute dtype) takes the stock forward unchanged."""
- global _fused_q8_0_installed
- if _fused_q8_0_installed or not _enabled(_ENV_FUSED_Q8_0):
+ """Patch ``GGUFLinear.forward_native``: Q8_0 dequant+cast in one kernel, an fp16 dequant cast to bf16 in place;
+ anything else takes the stock forward."""
+ global _fused_q8_0_installed, _fused_cast_installed, _forward_patched
+ q8_0 = _enabled(_ENV_FUSED_Q8_0)
+ fused_cast = _enabled(_ENV_FUSED_CAST)
+ if _forward_patched or not (q8_0 or fused_cast):
return _fused_q8_0_installed
linear_cls = getattr(gguf_utils, "GGUFLinear", None)
if linear_cls is None or not callable(getattr(linear_cls, _FORWARD_ATTR, None)):
@@ -183,19 +203,29 @@ def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:
def forward_native(self: Any, inputs: Any) -> Any:
weight = self.weight
dtype = self.compute_dtype
- if (
- getattr(weight, "quant_type", None) != _Q8_0
- or dtype is None
- or weight.dim() != 2
- or weight.shape[-1] % _Q8_0_TYPE_SIZE
- ):
+ quant_type = getattr(weight, "quant_type", None)
+ if dtype is None or weight.dim() != 2:
return stock_forward(self, inputs)
- dense = _q8_0_dequant_cast(weight.as_tensor(), dtype, blocks_fn)
+ if quant_type == _Q8_0:
+ if not q8_0 or weight.shape[-1] % _Q8_0_TYPE_SIZE:
+ return stock_forward(self, inputs)
+ dense = _q8_0_dequant_cast(weight.as_tensor(), dtype, blocks_fn)
+ else:
+ if not fused_cast or dtype != torch.bfloat16 or quant_type is None:
+ return stock_forward(self, inputs)
+ half = gguf_utils.dequantize_gguf_tensor(weight)
+ if half.dtype == torch.float16 and half.is_contiguous():
+ dense = _cast_in_place(half, dtype)
+ else:
+ dense = half.to(dtype)
+ del half # as in the stock forward, an fp32 dequant is freed before the matmul
bias = self.bias.to(dtype) if self.bias is not None else None
return F.linear(inputs, dense, bias)
if apply_patch(linear_cls, _FORWARD_ATTR, forward_native, force = True):
- _fused_q8_0_installed = True
+ _forward_patched = True
+ _fused_q8_0_installed = q8_0
+ _fused_cast_installed = fused_cast
return _fused_q8_0_installed
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "install fused Q8_0 dequant", exc)
@@ -203,13 +233,15 @@ def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:
def _uninstall_fused_q8_0(gguf_utils: Any) -> None:
- global _fused_q8_0_installed
- if not _fused_q8_0_installed:
+ global _fused_q8_0_installed, _fused_cast_installed, _forward_patched
+ if not _forward_patched:
return
linear_cls = getattr(gguf_utils, "GGUFLinear", None) if gguf_utils is not None else None
if linear_cls is not None:
revert_patch(linear_cls, _FORWARD_ATTR)
_fused_q8_0_installed = False
+ _fused_cast_installed = False
+ _forward_patched = False
def uninstall_compiled_dequant() -> None:
diff --git a/studio/backend/tests/test_diffusion_gguf_fused_cast.py b/studio/backend/tests/test_diffusion_gguf_fused_cast.py
new file mode 100644
index 00000000000..11be2c8567e
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_gguf_fused_cast.py
@@ -0,0 +1,192 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""GGUFLinear's fp16-dequantised types under a bf16 compute dtype: the compiled dequant's weight is cast in its own
+buffer, bit-identical to the stock ``.to``, and every other case keeps the stock forward. Runs on CPU."""
+
+from __future__ import annotations
+
+import pytest
+
+torch = pytest.importorskip("torch")
+np = pytest.importorskip("numpy")
+gguf = pytest.importorskip("gguf")
+gguf_utils = pytest.importorskip("diffusers.quantizers.gguf.utils")
+
+from core.inference import diffusion_gguf_compile as gc # noqa: E402
+
+T = gguf.GGMLQuantizationType
+needs_fused = pytest.mark.skipif(
+ not gc._torch_supports_fused_q8_0(torch.__version__),
+ reason = "the GGUF forward patch needs torch >= 2.7",
+)
+# byte offsets of each block's fp16 scales (diffusers' split_block_dims layouts): finite, realistic scales
+_F16_FIELDS = {
+ T.Q4_0: [0],
+ T.Q4_1: [0, 2],
+ T.Q5_0: [0],
+ T.Q5_1: [0, 2],
+ T.Q2_K: [80, 82],
+ T.Q3_K: [108],
+ T.Q4_K: [0, 2],
+ T.Q5_K: [0, 2],
+ T.Q6_K: [208],
+ T.IQ4_XS: [0],
+}
+FP16_TYPES = [T.Q4_K, T.Q5_K, T.Q6_K, T.Q3_K, T.Q2_K, T.Q4_0, T.Q4_1, T.Q5_0, T.Q5_1]
+
+
[email protected](autouse = True)
+def _clean(monkeypatch):
+ from core.inference import diffusion_compile_config as cc
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", raising = False)
+ cc._reset_for_tests()
+ gc.uninstall_all()
+ torch._dynamo.reset()
+ with torch._inductor.config.patch(emulate_precision_casts = False):
+ yield
+ gc.uninstall_all()
+ cc._reset_for_tests()
+
+
+def _raw(
+ qtype,
+ rows,
+ cols,
+ seed = 0,
+ adversarial = False,
+):
+ rng = np.random.default_rng(seed)
+ block, size = gguf.GGML_QUANT_SIZES[qtype]
+ if qtype == T.BF16:
+ w = (rng.standard_normal((rows, cols)) * 0.05).astype(np.float32).view(np.uint32)
+ return torch.from_numpy((w >> 16).astype(np.uint16).view(np.uint8).reshape(rows, -1).copy())
+ raw = rng.integers(0, 256, size = (rows, cols // block, size), dtype = np.uint8)
+ if not adversarial:
+ for off in _F16_FIELDS[qtype]:
+ scale = (np.abs(rng.standard_normal((rows, cols // block))) * 0.01 + 1e-4).astype(
+ np.float16
+ )
+ raw[:, :, off : off + 2] = scale.view(np.uint8).reshape(rows, cols // block, 2)
+ return torch.from_numpy(raw.reshape(rows, -1).copy())
+
+
+def _linear(
+ qtype,
+ rows,
+ cols,
+ dtype,
+ seed = 0,
+ adversarial = False,
+):
+ lin = gguf_utils.GGUFLinear(cols, rows, bias = True, compute_dtype = dtype)
+ lin.weight = gguf_utils.GGUFParameter(
+ _raw(qtype, rows, cols, seed, adversarial), quant_type = qtype
+ )
+ lin.bias = torch.nn.Parameter(torch.randn(rows, generator = torch.Generator().manual_seed(1)))
+ return lin
+
+
+def _bits(t):
+ return t.view(
+ {torch.bfloat16: torch.int16, torch.float16: torch.int16, torch.float32: torch.int32}[
+ t.dtype
+ ]
+ )
+
+
+def _watch(monkeypatch):
+ """Record the dequant's output buffer and the weight F.linear receives, per forward."""
+ import weakref
+
+ seen = {"dequant": [], "linear": [], "dequant_alive_in_linear": []}
+ dequant = gguf_utils.dequantize_gguf_tensor
+ linear = torch.nn.functional.linear
+ refs = []
+
+ def recorded_dequant(t):
+ out = dequant(t)
+ seen["dequant"].append(out.data_ptr())
+ refs.append(weakref.ref(out))
+ return out
+
+ def recorded_linear(
+ x,
+ w,
+ b = None,
+ ):
+ seen["linear"].append(w.data_ptr())
+ seen["dequant_alive_in_linear"].append([r() is not None for r in refs])
+ return linear(x, w, b)
+
+ monkeypatch.setattr(gguf_utils, "dequantize_gguf_tensor", recorded_dequant)
+ monkeypatch.setattr(torch.nn.functional, "linear", recorded_linear)
+ return seen
+
+
+@needs_fused
[email protected]("adversarial", [False, True], ids = ["scaled", "raw_bytes"])
[email protected]("qtype", FP16_TYPES, ids = lambda q: q.name)
+def test_an_fp16_dequant_is_cast_in_its_own_buffer_bit_for_bit(qtype, adversarial, monkeypatch):
+ # raw bytes: fp16 overflow, subnormals and NaN scales; 1023 rows: an odd count through the same dynamic graph
+ stock_forward = gguf_utils.GGUFLinear.forward_native
+ assert gc.install_compiled_dequant() is True
+ for rows in (256, 1023):
+ lin = _linear(qtype, rows, 512, torch.bfloat16, seed = rows, adversarial = adversarial)
+ x = torch.randn(3, 512, generator = torch.Generator().manual_seed(2)).to(torch.bfloat16)
+ expected = stock_forward(
+ lin, x
+ ) # today: the compiled dequant, then .to(bf16) into a second buffer
+ seen = _watch(monkeypatch)
+ got = lin(x)
+ assert torch.equal(_bits(got), _bits(expected))
+ # one dense weight: the linear reads the dequant's own buffer, not a copy of it
+ assert len(seen["dequant"]) == 1 and seen["linear"] == seen["dequant"]
+ monkeypatch.undo()
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", raising = False)
+
+
+@needs_fused
[email protected]("qtype", [T.IQ4_XS, T.BF16], ids = lambda q: q.name)
+def test_an_fp32_dequant_keeps_the_stock_cast(qtype, monkeypatch):
+ stock_forward = gguf_utils.GGUFLinear.forward_native
+ assert gc.install_compiled_dequant() is True
+ lin = _linear(qtype, 64, 256, torch.bfloat16)
+ x = torch.randn(2, 256).to(torch.bfloat16)
+ expected = stock_forward(lin, x)
+ seen = _watch(monkeypatch)
+ assert torch.equal(_bits(lin(x)), _bits(expected))
+ assert seen["linear"] != seen["dequant"] # a 4-byte weight cannot become a 2-byte one in place
+ # and, as in the stock forward, the fp32 dequant is gone before the matmul runs
+ assert seen["dequant_alive_in_linear"] == [[False]]
+ monkeypatch.undo()
+
+
+@needs_fused
+def test_fp16_compute_and_the_kill_switch_take_the_stock_forward(monkeypatch):
+ assert gc.install_compiled_dequant() is True
+ lin = _linear(T.Q4_K, 64, 256, torch.float16)
+ seen = _watch(monkeypatch)
+ lin(torch.randn(2, 256).to(torch.float16))
+ assert (
+ seen["linear"] == seen["dequant"]
+ ) # the stock forward: an fp16 weight's .to(fp16) is itself
+ monkeypatch.undo()
+
+ gc.uninstall_all()
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", "0")
+ assert gc.install_compiled_dequant() is True and not gc.is_fused_cast_installed()
+ lin = _linear(T.Q4_K, 64, 256, torch.bfloat16)
+ seen = _watch(monkeypatch)
+ lin(torch.randn(2, 256).to(torch.bfloat16))
+ assert seen["linear"] != seen["dequant"] # the stock cast into a second buffer
+ monkeypatch.undo()
+
+
+def test_cast_in_place_matches_to_bit_for_bit_including_nan():
+ bits = np.random.default_rng(3).integers(0, 1 << 16, size = 4096, dtype = np.uint16)
+ half = torch.from_numpy(bits.view(np.float16).copy())
+ want = half.to(torch.bfloat16)
+ got = gc._cast_in_place(half.clone(), torch.bfloat16)
+ assert torch.equal(_bits(got), _bits(want))
diff --git a/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py
index 2cfe4916430..18f3c072048 100644
--- a/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py
+++ b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py
@@ -142,6 +142,9 @@ def test_patched_q8_0_linear_skips_the_unfused_dequant_and_matches_stock(monkeyp
def test_other_quant_types_take_the_stock_forward(monkeypatch):
+ monkeypatch.setenv(
+ "UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", "0"
+ ) # their own fusion: test_diffusion_gguf_fused_cast
rng = np.random.default_rng(4)
block = gguf.GGML_QUANT_SIZES[Q4_K][1]
raw = rng.integers(0, 256, size = (16, 256 // 256 * block), dtype = np.uint8)
@@ -191,6 +194,11 @@ def test_fused_q8_0_uninstall_and_kill_switch(monkeypatch):
monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_FUSED_Q8_0", "0")
assert gc.install_compiled_dequant() is True
assert gc.is_fused_q8_0_installed() is False
+ assert gc.is_fused_cast_installed() is True # the other types still fuse their cast
+ gc.uninstall_all()
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", "0")
+ assert gc.install_compiled_dequant() is True
+ assert gc.is_fused_q8_0_installed() is False and gc.is_fused_cast_installed() is False
assert gguf_utils.GGUFLinear.forward_native is stock