Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio GGUF: cast K-quant and legacy weights to bf16 in their own buffer (#17080)
Daniel Han46c0c8f0d81b
diff --git a/studio/backend/core/inference/diffusion_gguf_compile.py b/studio/backend/core/inference/diffusion_gguf_compile.pyindex 81841a21670..173ad4de45f 100644--- a/studio/backend/core/inference/diffusion_gguf_compile.py+++ b/studio/backend/core/inference/diffusion_gguf_compile.py@@ -8,7 +8,10 @@ lazily; UNSLOTH_DIFFUSION_GGUF_COMPILE_DEQUANT=0 disables.Q8_0 GGUFLinear weights also get one fused dequant+cast kernel (UNSLOTH_DIFFUSION_GGUF_FUSED_Q8_0=0 disables): thestock forward runs the compiled dequant (fp16 out, several kernels) and then an eager ``.to(compute_dtype)`` copy perlinear per forward. The fused kernel keeps the fp16 rounding of ``d * q`` before the cast (see ``_q8_0_blocks_dequant_cast``),-so the weight it produces is bit-identical to the stock one."""+so the weight it produces is bit-identical to the stock one.++Other fp16-dequantised types under a bf16 compute dtype are cast in the dequant's own buffer, bit-identical to the stock+cast (UNSLOTH_DIFFUSION_GGUF_FUSED_CAST=0 disables); a compiled-in cast was not bit-identical for Q4_K / Q5_K."""from __future__ import annotations@@ -50,6 +53,10 @@ _Q8_0_TYPE_SIZE = 34 # one fp16 scale + 32 int8 codes per block_FUSED_Q8_0_MIN_TORCH = (2, 7)_fused_q8_0_installed = False+_ENV_FUSED_CAST = "UNSLOTH_DIFFUSION_GGUF_FUSED_CAST"+_fused_cast_installed = False+_forward_patched = False+def _q8_0_blocks_dequant_cast(raw: Any, dtype: Any) -> Any:"""Q8_0 bytes (any shape, ``n_blocks * 34`` bytes) -> ``(n_blocks, 32)`` weight in ``dtype``. Same ops and order@@ -69,6 +76,17 @@ def _q8_0_blocks_dequant_cast(raw: Any, dtype: Any) -> Any:return half_bits.view(torch.float16).to(dtype)+def _cast_in_place(half: Any, dtype: Any) -> Any:+ """``half.to(dtype)`` for an fp16 weight and a 2-byte ``dtype``, written over ``half`` (same conversion kernel)."""+ dense = half.view(dtype)+ dense.copy_(half)+ return dense+++def is_fused_cast_installed() -> bool:+ return _fused_cast_installed++def _q8_0_dequant_cast(raw: Any,dtype: Any,@@ -161,10 +179,12 @@ def _guard(compiled: Any, eager: Any, logger: Any) -> Any:def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:- """Patch ``GGUFLinear.forward_native`` so a Q8_0 weight is dequantised and cast in one compiled kernel; every- other weight (other quant types, no compute dtype) takes the stock forward unchanged."""- global _fused_q8_0_installed- if _fused_q8_0_installed or not _enabled(_ENV_FUSED_Q8_0):+ """Patch ``GGUFLinear.forward_native``: Q8_0 dequant+cast in one kernel, an fp16 dequant cast to bf16 in place;+ anything else takes the stock forward."""+ global _fused_q8_0_installed, _fused_cast_installed, _forward_patched+ q8_0 = _enabled(_ENV_FUSED_Q8_0)+ fused_cast = _enabled(_ENV_FUSED_CAST)+ if _forward_patched or not (q8_0 or fused_cast):return _fused_q8_0_installedlinear_cls = getattr(gguf_utils, "GGUFLinear", None)if linear_cls is None or not callable(getattr(linear_cls, _FORWARD_ATTR, None)):@@ -183,19 +203,29 @@ def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:def forward_native(self: Any, inputs: Any) -> Any:weight = self.weightdtype = self.compute_dtype- if (- getattr(weight, "quant_type", None) != _Q8_0- or dtype is None- or weight.dim() != 2- or weight.shape[-1] % _Q8_0_TYPE_SIZE- ):+ quant_type = getattr(weight, "quant_type", None)+ if dtype is None or weight.dim() != 2:return stock_forward(self, inputs)- dense = _q8_0_dequant_cast(weight.as_tensor(), dtype, blocks_fn)+ if quant_type == _Q8_0:+ if not q8_0 or weight.shape[-1] % _Q8_0_TYPE_SIZE:+ return stock_forward(self, inputs)+ dense = _q8_0_dequant_cast(weight.as_tensor(), dtype, blocks_fn)+ else:+ if not fused_cast or dtype != torch.bfloat16 or quant_type is None:+ return stock_forward(self, inputs)+ half = gguf_utils.dequantize_gguf_tensor(weight)+ if half.dtype == torch.float16 and half.is_contiguous():+ dense = _cast_in_place(half, dtype)+ else:+ dense = half.to(dtype)+ del half # as in the stock forward, an fp32 dequant is freed before the matmulbias = self.bias.to(dtype) if self.bias is not None else Nonereturn F.linear(inputs, dense, bias)if apply_patch(linear_cls, _FORWARD_ATTR, forward_native, force = True):- _fused_q8_0_installed = True+ _forward_patched = True+ _fused_q8_0_installed = q8_0+ _fused_cast_installed = fused_castreturn _fused_q8_0_installedexcept Exception as exc: # noqa: BLE001 - optimisation only_warn(logger, "install fused Q8_0 dequant", exc)@@ -203,13 +233,15 @@ def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:def _uninstall_fused_q8_0(gguf_utils: Any) -> None:- global _fused_q8_0_installed- if not _fused_q8_0_installed:+ global _fused_q8_0_installed, _fused_cast_installed, _forward_patched+ if not _forward_patched:returnlinear_cls = getattr(gguf_utils, "GGUFLinear", None) if gguf_utils is not None else Noneif linear_cls is not None:revert_patch(linear_cls, _FORWARD_ATTR)_fused_q8_0_installed = False+ _fused_cast_installed = False+ _forward_patched = Falsedef uninstall_compiled_dequant() -> None:diff --git a/studio/backend/tests/test_diffusion_gguf_fused_cast.py b/studio/backend/tests/test_diffusion_gguf_fused_cast.pynew file mode 100644index 00000000000..11be2c8567e--- /dev/null+++ b/studio/backend/tests/test_diffusion_gguf_fused_cast.py@@ -0,0 +1,192 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""GGUFLinear's fp16-dequantised types under a bf16 compute dtype: the compiled dequant's weight is cast in its own+buffer, bit-identical to the stock ``.to``, and every other case keeps the stock forward. Runs on CPU."""++from __future__ import annotations++import pytest++torch = pytest.importorskip("torch")+np = pytest.importorskip("numpy")+gguf = pytest.importorskip("gguf")+gguf_utils = pytest.importorskip("diffusers.quantizers.gguf.utils")++from core.inference import diffusion_gguf_compile as gc # noqa: E402++T = gguf.GGMLQuantizationType+needs_fused = pytest.mark.skipif(+ not gc._torch_supports_fused_q8_0(torch.__version__),+ reason = "the GGUF forward patch needs torch >= 2.7",+)+# byte offsets of each block's fp16 scales (diffusers' split_block_dims layouts): finite, realistic scales+_F16_FIELDS = {+ T.Q4_0: [0],+ T.Q4_1: [0, 2],+ T.Q5_0: [0],+ T.Q5_1: [0, 2],+ T.Q2_K: [80, 82],+ T.Q3_K: [108],+ T.Q4_K: [0, 2],+ T.Q5_K: [0, 2],+ T.Q6_K: [208],+ T.IQ4_XS: [0],+}+FP16_TYPES = [T.Q4_K, T.Q5_K, T.Q6_K, T.Q3_K, T.Q2_K, T.Q4_0, T.Q4_1, T.Q5_0, T.Q5_1]++[email protected](autouse = True)+def _clean(monkeypatch):+ from core.inference import diffusion_compile_config as cc++ monkeypatch.delenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", raising = False)+ cc._reset_for_tests()+ gc.uninstall_all()+ torch._dynamo.reset()+ with torch._inductor.config.patch(emulate_precision_casts = False):+ yield+ gc.uninstall_all()+ cc._reset_for_tests()+++def _raw(+ qtype,+ rows,+ cols,+ seed = 0,+ adversarial = False,+):+ rng = np.random.default_rng(seed)+ block, size = gguf.GGML_QUANT_SIZES[qtype]+ if qtype == T.BF16:+ w = (rng.standard_normal((rows, cols)) * 0.05).astype(np.float32).view(np.uint32)+ return torch.from_numpy((w >> 16).astype(np.uint16).view(np.uint8).reshape(rows, -1).copy())+ raw = rng.integers(0, 256, size = (rows, cols // block, size), dtype = np.uint8)+ if not adversarial:+ for off in _F16_FIELDS[qtype]:+ scale = (np.abs(rng.standard_normal((rows, cols // block))) * 0.01 + 1e-4).astype(+ np.float16+ )+ raw[:, :, off : off + 2] = scale.view(np.uint8).reshape(rows, cols // block, 2)+ return torch.from_numpy(raw.reshape(rows, -1).copy())+++def _linear(+ qtype,+ rows,+ cols,+ dtype,+ seed = 0,+ adversarial = False,+):+ lin = gguf_utils.GGUFLinear(cols, rows, bias = True, compute_dtype = dtype)+ lin.weight = gguf_utils.GGUFParameter(+ _raw(qtype, rows, cols, seed, adversarial), quant_type = qtype+ )+ lin.bias = torch.nn.Parameter(torch.randn(rows, generator = torch.Generator().manual_seed(1)))+ return lin+++def _bits(t):+ return t.view(+ {torch.bfloat16: torch.int16, torch.float16: torch.int16, torch.float32: torch.int32}[+ t.dtype+ ]+ )+++def _watch(monkeypatch):+ """Record the dequant's output buffer and the weight F.linear receives, per forward."""+ import weakref++ seen = {"dequant": [], "linear": [], "dequant_alive_in_linear": []}+ dequant = gguf_utils.dequantize_gguf_tensor+ linear = torch.nn.functional.linear+ refs = []++ def recorded_dequant(t):+ out = dequant(t)+ seen["dequant"].append(out.data_ptr())+ refs.append(weakref.ref(out))+ return out++ def recorded_linear(+ x,+ w,+ b = None,+ ):+ seen["linear"].append(w.data_ptr())+ seen["dequant_alive_in_linear"].append([r() is not None for r in refs])+ return linear(x, w, b)++ monkeypatch.setattr(gguf_utils, "dequantize_gguf_tensor", recorded_dequant)+ monkeypatch.setattr(torch.nn.functional, "linear", recorded_linear)+ return seen+++@needs_fused[email protected]("adversarial", [False, True], ids = ["scaled", "raw_bytes"])[email protected]("qtype", FP16_TYPES, ids = lambda q: q.name)+def test_an_fp16_dequant_is_cast_in_its_own_buffer_bit_for_bit(qtype, adversarial, monkeypatch):+ # raw bytes: fp16 overflow, subnormals and NaN scales; 1023 rows: an odd count through the same dynamic graph+ stock_forward = gguf_utils.GGUFLinear.forward_native+ assert gc.install_compiled_dequant() is True+ for rows in (256, 1023):+ lin = _linear(qtype, rows, 512, torch.bfloat16, seed = rows, adversarial = adversarial)+ x = torch.randn(3, 512, generator = torch.Generator().manual_seed(2)).to(torch.bfloat16)+ expected = stock_forward(+ lin, x+ ) # today: the compiled dequant, then .to(bf16) into a second buffer+ seen = _watch(monkeypatch)+ got = lin(x)+ assert torch.equal(_bits(got), _bits(expected))+ # one dense weight: the linear reads the dequant's own buffer, not a copy of it+ assert len(seen["dequant"]) == 1 and seen["linear"] == seen["dequant"]+ monkeypatch.undo()+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", raising = False)+++@needs_fused[email protected]("qtype", [T.IQ4_XS, T.BF16], ids = lambda q: q.name)+def test_an_fp32_dequant_keeps_the_stock_cast(qtype, monkeypatch):+ stock_forward = gguf_utils.GGUFLinear.forward_native+ assert gc.install_compiled_dequant() is True+ lin = _linear(qtype, 64, 256, torch.bfloat16)+ x = torch.randn(2, 256).to(torch.bfloat16)+ expected = stock_forward(lin, x)+ seen = _watch(monkeypatch)+ assert torch.equal(_bits(lin(x)), _bits(expected))+ assert seen["linear"] != seen["dequant"] # a 4-byte weight cannot become a 2-byte one in place+ # and, as in the stock forward, the fp32 dequant is gone before the matmul runs+ assert seen["dequant_alive_in_linear"] == [[False]]+ monkeypatch.undo()+++@needs_fused+def test_fp16_compute_and_the_kill_switch_take_the_stock_forward(monkeypatch):+ assert gc.install_compiled_dequant() is True+ lin = _linear(T.Q4_K, 64, 256, torch.float16)+ seen = _watch(monkeypatch)+ lin(torch.randn(2, 256).to(torch.float16))+ assert (+ seen["linear"] == seen["dequant"]+ ) # the stock forward: an fp16 weight's .to(fp16) is itself+ monkeypatch.undo()++ gc.uninstall_all()+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", "0")+ assert gc.install_compiled_dequant() is True and not gc.is_fused_cast_installed()+ lin = _linear(T.Q4_K, 64, 256, torch.bfloat16)+ seen = _watch(monkeypatch)+ lin(torch.randn(2, 256).to(torch.bfloat16))+ assert seen["linear"] != seen["dequant"] # the stock cast into a second buffer+ monkeypatch.undo()+++def test_cast_in_place_matches_to_bit_for_bit_including_nan():+ bits = np.random.default_rng(3).integers(0, 1 << 16, size = 4096, dtype = np.uint16)+ half = torch.from_numpy(bits.view(np.float16).copy())+ want = half.to(torch.bfloat16)+ got = gc._cast_in_place(half.clone(), torch.bfloat16)+ assert torch.equal(_bits(got), _bits(want))diff --git a/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.pyindex 2cfe4916430..18f3c072048 100644--- a/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py+++ b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py@@ -142,6 +142,9 @@ def test_patched_q8_0_linear_skips_the_unfused_dequant_and_matches_stock(monkeypdef test_other_quant_types_take_the_stock_forward(monkeypatch):+ monkeypatch.setenv(+ "UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", "0"+ ) # their own fusion: test_diffusion_gguf_fused_castrng = np.random.default_rng(4)block = gguf.GGML_QUANT_SIZES[Q4_K][1]raw = rng.integers(0, 256, size = (16, 256 // 256 * block), dtype = np.uint8)@@ -191,6 +194,11 @@ def test_fused_q8_0_uninstall_and_kill_switch(monkeypatch):monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_FUSED_Q8_0", "0")assert gc.install_compiled_dequant() is Trueassert gc.is_fused_q8_0_installed() is False+ assert gc.is_fused_cast_installed() is True # the other types still fuse their cast+ gc.uninstall_all()+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", "0")+ assert gc.install_compiled_dequant() is True+ assert gc.is_fused_q8_0_installed() is False and gc.is_fused_cast_installed() is Falseassert gguf_utils.GGUFLinear.forward_native is stock