Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio GGUF: dequantise bit-identical to llama.cpp, rounded once to the compute dtype (#17204)

Daniel Hanb67e5630656e
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 3d0c22fbb72..a8a1f0609d7 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -323,6 +323,7 @@ from .diffusion_offload_prefetch import cross_forward_render, cross_forward_step
from .diffusion_vae_fp16 import enable_fp16_vae_decode
from .diffusion_vae_upcast import pipeline_upcasts_keep_storage
from .diffusion_vae_tiling import install as install_wide_vae_tiles
+from .diffusion_gguf_exact import install_exact_dequant
from .diffusion_gguf_lowbit import install_lowbit_iq
from .diffusion_attention import (
apply_attention_backend,
@@ -9943,6 +9944,7 @@ class DiffusionBackend:
if kind == "gguf":
# IQ1 / IQ2 / IQ3 tensors diffusers' table lacks would refuse the file
install_lowbit_iq(logger)
+ install_exact_dequant(logger)
# Dequantise the GGUF transformer on-device at the compute dtype.
sf_kwargs["quantization_config"] = diffusers.GGUFQuantizationConfig(
compute_dtype = dtype
diff --git a/studio/backend/core/inference/diffusion_gguf_compile.py b/studio/backend/core/inference/diffusion_gguf_compile.py
index 173ad4de45f..5aedefed939 100644
--- a/studio/backend/core/inference/diffusion_gguf_compile.py
+++ b/studio/backend/core/inference/diffusion_gguf_compile.py
@@ -11,7 +11,8 @@ linear per forward. The fused kernel keeps the fp16 rounding of ``d * q`` before
so the weight it produces is bit-identical to the stock one.
Other fp16-dequantised types under a bf16 compute dtype are cast in the dequant's own buffer, bit-identical to the stock
-cast (UNSLOTH_DIFFUSION_GGUF_FUSED_CAST=0 disables); a compiled-in cast was not bit-identical for Q4_K / Q5_K."""
+cast (UNSLOTH_DIFFUSION_GGUF_FUSED_CAST=0 disables); a compiled-in cast was not bit-identical for Q4_K / Q5_K.
+diffusion_gguf_exact types compile its dequant per type, cast inside the kernel."""
from __future__ import annotations
@@ -136,6 +137,9 @@ def install_compiled_dequant(logger: Any = None) -> bool:
if apply_patch(gguf_utils, _DEQUANT_ATTR, compiled, force = True):
_compiled_dequant_installed = True
_compile_error = None
+ from . import diffusion_gguf_exact # noqa: PLC0415
+
+ diffusion_gguf_exact.set_compiled_dequant(True)
_install_fused_q8_0(gguf_utils, logger)
return True
return False
@@ -194,8 +198,12 @@ def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:
if not _torch_supports_fused_q8_0(torch.__version__):
return False
+ from . import diffusion_gguf_exact as gguf_exact # noqa: PLC0415
+
compiled = torch.compile(_q8_0_blocks_dequant_cast, dynamic = True)
blocks_fn = _guard(compiled, _q8_0_blocks_dequant_cast, logger)
+ # one compile per type, so no type evicts another
+ exact_fns = {fn: _guard(torch.compile(fn), fn, logger) for fn in gguf_exact.EXACT_FUNCTIONS}
stock_forward = getattr(linear_cls, _FORWARD_ATTR)
F = torch.nn.functional
@@ -206,7 +214,16 @@ def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:
quant_type = getattr(weight, "quant_type", None)
if dtype is None or weight.dim() != 2:
return stock_forward(self, inputs)
- if quant_type == _Q8_0:
+ exact_fn = gguf_exact.exact_function(quant_type) if quant_type is not None else None
+ if exact_fn is not None:
+ if not (q8_0 if quant_type == _Q8_0 else fused_cast) or not gguf_exact.row_aligned(
+ weight, quant_type
+ ):
+ return (gguf_exact.stock_forward() or stock_forward)(self, inputs)
+ dense = gguf_exact.dequantize_cast(
+ weight.as_tensor(), quant_type, dtype, exact_fns[exact_fn]
+ )
+ elif quant_type == _Q8_0:
if not q8_0 or weight.shape[-1] % _Q8_0_TYPE_SIZE:
return stock_forward(self, inputs)
dense = _q8_0_dequant_cast(weight.as_tensor(), dtype, blocks_fn)
@@ -254,6 +271,9 @@ def uninstall_compiled_dequant() -> None:
if gguf_utils is not None:
revert_patch(gguf_utils, _DEQUANT_ATTR)
_compiled_dequant_installed = False
+ from . import diffusion_gguf_exact # noqa: PLC0415
+
+ diffusion_gguf_exact.set_compiled_dequant(False)
def uninstall_all() -> None:
diff --git a/studio/backend/core/inference/diffusion_gguf_exact.py b/studio/backend/core/inference/diffusion_gguf_exact.py
new file mode 100644
index 00000000000..1f864317d43
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_gguf_exact.py
@@ -0,0 +1,446 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""GGUF dequant bit-identical to llama.cpp's ``dequantize_row_*`` (diffusers' rounds in fp16), rounded once to the
+compute dtype. Products are exact in fp32, so only ``+ m`` / ``- m`` rounds and FMA cannot change a bit.
+UNSLOTH_DIFFUSION_GGUF_EXACT=0 keeps diffusers'."""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+EXACT_ENV = "UNSLOTH_DIFFUSION_GGUF_EXACT"
+_QK_K = 256
+_CAST_ON_STORE = ("cuda", "cpu")
+_KVALUES_IQ4NL = (-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113)
+
+
+def exact_enabled() -> bool:
+ return os.environ.get(EXACT_ENV, "").strip().lower() not in ("0", "false", "no", "off")
+
+
+def _scale(blocks: Any, word: int) -> Any:
+ """fp16 at byte ``2 * word`` as fp32 ``(n, 1)``; view only 2 bytes: inductor copies a wider dtype view."""
+ import torch
+ return blocks[:, 2 * word : 2 * word + 2].view(torch.float16).to(torch.float32)
+
+
+def _half_math(blocks: Any, dtype: Any) -> Any:
+ import torch
+ if dtype != torch.float16 or torch.compiler.is_compiling():
+ return None
+ return blocks[:, :2].view(torch.float16)
+
+
+def _field(blocks: Any, start: int, stop: int, shape: tuple) -> Any:
+ return blocks[:, start:stop].unflatten(1, shape)
+
+
+def _shifts(like: Any, start: int, stop: int, step: int, shape: tuple) -> Any:
+ import torch
+ return torch.arange(start, stop, step, device = like.device, dtype = torch.uint8).reshape(shape)
+
+
+def _emit(scale: Any, q: Any, m: Any, op: str, n: int, block_size: int, dtype: Any) -> Any:
+ """``scale * q +/- m`` as ``(n, block_size)``, rounded once to ``dtype``. Only the contiguous result is reshaped
+ (a strided reshape is 1.7x slower Q6_K compiled). Traced, not ``addcmul(-m, ...)``: Triton's ``0 - m`` loses -0."""
+ import torch
+
+ if (
+ not torch.compiler.is_compiling()
+ and dtype is not None
+ and dtype != torch.float32
+ and q.device.type in _CAST_ON_STORE
+ ):
+ out = torch.empty(
+ torch.broadcast_shapes(scale.shape, q.shape), dtype = dtype, device = q.device
+ )
+ if op:
+ torch.addcmul(m if op == "+" else -m, scale, q, out = out)
+ else:
+ torch.mul(scale, q, out = out)
+ return out.reshape(n, block_size)
+ y = scale * q
+ if op == "+":
+ y = y.add_(m)
+ elif op == "-":
+ y = y.sub_(m)
+ y = y.reshape(n, block_size)
+ return y if dtype is None or dtype == torch.float32 else y.to(dtype)
+
+
+def _q4_codes(blocks: Any, at: int) -> Any:
+ """``(n, 2, 2, 8)``: element ``16h + 8a + b`` is nibble ``h`` of byte ``8a + b``."""
+ return (_field(blocks, at, at + 16, (1, 2, 8)) >> _shifts(blocks, 0, 8, 4, (2, 1, 1))) & 0x0F
+
+
+def dequantize_blocks_Q4_0(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ import torch
+
+ n = blocks.shape[0]
+ q = _q4_codes(blocks, 2).to(torch.int8) - 8
+ half = _half_math(blocks, dtype)
+ if half is not None:
+ return (half.reshape(n, 1, 1, 1) * q).reshape(n, 32)
+ return _emit(_scale(blocks, 0).reshape(n, 1, 1, 1), q, None, "", n, 32, dtype)
+
+
+def dequantize_blocks_Q4_1(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ n = blocks.shape[0]
+ d = _scale(blocks, 0).reshape(n, 1, 1, 1)
+ m = _scale(blocks, 1).reshape(n, 1, 1, 1)
+ return _emit(d, _q4_codes(blocks, 4), m, "+", n, 32, dtype)
+
+
+def _q5_codes(blocks: Any, qh_at: int) -> Any:
+ """Code ``k`` takes bit ``k`` of the little-endian 32-bit ``qh``."""
+ high = (_field(blocks, qh_at, qh_at + 4, (2, 2, 1)) >> _shifts(blocks, 0, 8, 1, (8,))) & 1
+ return _q4_codes(blocks, qh_at + 4) | (high << 4)
+
+
+def dequantize_blocks_Q5_0(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ import torch
+
+ n = blocks.shape[0]
+ q = _q5_codes(blocks, 2).to(torch.int8) - 16
+ half = _half_math(blocks, dtype)
+ if half is not None:
+ return (half.reshape(n, 1, 1, 1) * q).reshape(n, 32)
+ return _emit(_scale(blocks, 0).reshape(n, 1, 1, 1), q, None, "", n, 32, dtype)
+
+
+def dequantize_blocks_Q5_1(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ n = blocks.shape[0]
+ d = _scale(blocks, 0).reshape(n, 1, 1, 1)
+ m = _scale(blocks, 1).reshape(n, 1, 1, 1)
+ return _emit(d, _q5_codes(blocks, 4), m, "+", n, 32, dtype)
+
+
+def dequantize_blocks_Q8_0(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ import torch
+
+ n = blocks.shape[0]
+ q = blocks[:, 2:].view(torch.int8)
+ half = _half_math(blocks, dtype)
+ if half is not None:
+ return half * q
+ return _emit(_scale(blocks, 0), q, None, "", n, 32, dtype)
+
+
+def _k_codes(blocks: Any, at: int) -> Any:
+ """``(n, chunk c, shift j, half h, 16)`` = ``(q[32c + 16h + l] >> 2j) & 3``; scale group ``8c + 2j + h``."""
+ return (_field(blocks, at, at + 64, (2, 1, 2, 16)) >> _shifts(blocks, 0, 8, 2, (4, 1, 1))) & 3
+
+
+def dequantize_blocks_Q2_K(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ n = blocks.shape[0]
+ scales = blocks[:, :16]
+ d = _scale(blocks, 40)
+ dmin = _scale(blocks, 41)
+ dl = (d * (scales & 0x0F)).reshape(n, 2, 4, 2, 1)
+ ml = (dmin * (scales >> 4)).reshape(n, 2, 4, 2, 1)
+ return _emit(dl, _k_codes(blocks, 16), ml, "-", n, _QK_K, dtype)
+
+
+def dequantize_blocks_Q3_K(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ import torch
+
+ n = blocks.shape[0]
+ scales = blocks[:, 96:108]
+ d = _scale(blocks, 54)
+ # ggml's aux[] unpack: nibble of byte g % 8, 2 high bits from byte 8 + g % 4
+ low = (scales[:, :8].unsqueeze(1) >> _shifts(blocks, 0, 8, 4, (2, 1))) & 0x0F
+ high = (scales[:, 8:].unsqueeze(1) >> _shifts(blocks, 0, 8, 2, (4, 1))) & 3
+ sc = (low.reshape(n, 16) | (high.reshape(n, 16) << 4)).to(torch.int8) - 32
+ dl = (d * sc).reshape(n, 2, 4, 2, 1)
+ # hmask bit 4c + j clear -> subtract 4
+ hb = (_field(blocks, 0, 32, (1, 1, 2, 16)) >> _shifts(blocks, 0, 8, 1, (2, 4, 1, 1))) & 1
+ q = _k_codes(blocks, 32).to(torch.int8) - ((hb ^ 1) << 2).to(torch.int8)
+ return _emit(dl, q, None, "", n, _QK_K, dtype)
+
+
+def _scale_min_k4(scales: Any) -> tuple:
+ """ggml's get_scale_min_k4 for j = 0..7."""
+ import torch
+
+ a, b, c = scales[:, :4], scales[:, 4:8], scales[:, 8:12]
+ sc = torch.cat((a & 63, (c & 0x0F) | ((a >> 6) << 4)), dim = 1)
+ m = torch.cat((b & 63, (c >> 4) | ((b >> 6) << 4)), dim = 1)
+ return sc, m
+
+
+def _k4_scales(blocks: Any) -> tuple:
+ n = blocks.shape[0]
+ d = _scale(blocks, 0)
+ dmin = _scale(blocks, 1)
+ sc, m = _scale_min_k4(blocks[:, 4:16])
+ return (d * sc).reshape(n, 4, 2, 1), (dmin * m).reshape(n, 4, 2, 1)
+
+
+def _k4_codes(blocks: Any, at: int) -> Any:
+ return (_field(blocks, at, at + 128, (4, 1, 32)) >> _shifts(blocks, 0, 8, 4, (2, 1))) & 0x0F
+
+
+def dequantize_blocks_Q4_K(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ n = blocks.shape[0]
+ d1, m1 = _k4_scales(blocks)
+ return _emit(d1, _k4_codes(blocks, 16), m1, "-", n, _QK_K, dtype)
+
+
+def dequantize_blocks_Q5_K(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ n = blocks.shape[0]
+ d1, m1 = _k4_scales(blocks)
+ qh = (_field(blocks, 16, 48, (1, 1, 32)) >> _shifts(blocks, 0, 8, 1, (4, 2, 1))) & 1
+ q = _k4_codes(blocks, 48) | (qh << 4)
+ return _emit(d1, q, m1, "-", n, _QK_K, dtype)
+
+
+def dequantize_blocks_Q6_K(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ import torch
+
+ n = blocks.shape[0]
+ d = _scale(blocks, 104)
+ sc = blocks[:, 192:208].view(torch.int8)
+ # (n, chunk, nibble, half, 2, 16): quarter t = 2 * nibble + half, ggml's q1..q4
+ ql = (_field(blocks, 0, 128, (2, 1, 2, 2, 16)) >> _shifts(blocks, 0, 8, 4, (2, 1, 1, 1))) & 0x0F
+ qh = (_field(blocks, 128, 192, (2, 1, 1, 2, 16)) >> _shifts(blocks, 0, 8, 2, (2, 2, 1, 1))) & 3
+ q = (ql | (qh << 4)).to(torch.int8) - 32
+ return _emit((d * sc).reshape(n, 2, 2, 2, 2, 1), q, None, "", n, _QK_K, dtype)
+
+
+def dequantize_blocks_IQ4_XS(
+ blocks: Any,
+ block_size: Any,
+ type_size: Any,
+ dtype: Any = None,
+) -> Any:
+ import torch
+
+ n = blocks.shape[0]
+ d = _scale(blocks, 0)
+ scales_h = blocks[:, 2:3].to(torch.int32) | (blocks[:, 3:4].to(torch.int32) << 8)
+ shifts = torch.arange(0, 16, 2, device = blocks.device, dtype = torch.int32)
+ high = ((scales_h >> shifts) & 3).to(torch.uint8)
+ low = ((blocks[:, 4:8].unsqueeze(-1) >> _shifts(blocks, 0, 8, 4, (2,))) & 0x0F).reshape(n, 8)
+ ls = (low | (high << 4)).to(torch.int8) - 32
+ dl = (d * ls).reshape(n, 8, 1, 1)
+ kvalues = torch.tensor(_KVALUES_IQ4NL, device = blocks.device, dtype = torch.float32)
+ index = (_field(blocks, 8, 136, (8, 1, 16)) >> _shifts(blocks, 0, 8, 4, (2, 1))) & 0x0F
+ return _emit(dl, kvalues[index.long()], None, "", n, _QK_K, dtype)
+
+
+_FUNCTIONS = {
+ "Q4_0": dequantize_blocks_Q4_0,
+ "Q4_1": dequantize_blocks_Q4_1,
+ "Q5_0": dequantize_blocks_Q5_0,
+ "Q5_1": dequantize_blocks_Q5_1,
+ "Q8_0": dequantize_blocks_Q8_0,
+ "Q2_K": dequantize_blocks_Q2_K,
+ "Q3_K": dequantize_blocks_Q3_K,
+ "Q4_K": dequantize_blocks_Q4_K,
+ "Q5_K": dequantize_blocks_Q5_K,
+ "Q6_K": dequantize_blocks_Q6_K,
+ "IQ4_XS": dequantize_blocks_IQ4_XS,
+}
+EXACT_FUNCTIONS = tuple(_FUNCTIONS.values())
+_EXACT = frozenset(EXACT_FUNCTIONS)
+_TAG = "_unsloth_gguf_exact"
+for _fn in EXACT_FUNCTIONS:
+ # an attribute test traces inside a compiled block; set membership graph-breaks
+ setattr(_fn, _TAG, True)
+del _fn
+_replaced: dict = {}
+# "forward" (diffusers' forward_native) outlives uninstall: a later patch may fall back to _exact_forward
+_STATE: dict = {}
+
+
+def _table() -> Any:
+ table = _STATE.get("table")
+ if table is None:
+ try:
+ from diffusers.quantizers.gguf import utils as gguf_utils
+ except Exception: # noqa: BLE001
+ return None
+ table = _STATE["table"] = getattr(gguf_utils, "dequantize_functions", None)
+ return table
+
+
+def _sizes() -> Any:
+ sizes = _STATE.get("sizes")
+ if sizes is None:
+ from gguf import GGML_QUANT_SIZES
+ sizes = _STATE["sizes"] = GGML_QUANT_SIZES
+ return sizes
+
+
+def exact_function(quant_type: Any) -> Any:
+ table = _table()
+ fn = table.get(quant_type) if isinstance(table, dict) else None
+ return fn if getattr(fn, _TAG, False) else None
+
+
+def row_aligned(raw: Any, quant_type: Any) -> bool:
+ return raw.dim() >= 1 and raw.shape[-1] % _sizes()[quant_type][1] == 0
+
+
+def dequantize_cast(
+ raw: Any,
+ quant_type: Any,
+ dtype: Any,
+ blocks_fn: Any = None,
+) -> Any:
+ import torch
+
+ block_size, type_size = _sizes()[quant_type]
+ blocks = raw.view(torch.uint8).reshape(-1, type_size)
+ if blocks_fn is None:
+ blocks_fn = exact_function(quant_type)
+ elif not torch.compiler.is_compiling():
+ torch._dynamo.maybe_mark_dynamic(blocks, 0)
+ return blocks_fn(blocks, block_size, type_size, dtype).reshape(*raw.shape[:-1], -1)
+
+
+def _stock_forward(linear_cls: Any) -> Any:
+ """diffusers' own ``forward_native`` under any wrappers, by code file (``functools.wraps`` copies ``__module__``)."""
+ import sys
+
+ home = getattr(sys.modules.get(linear_cls.__module__), "__file__", None)
+ seen = []
+ stash = getattr(linear_cls, "_original_utils_GGUFLinear_forward_native", None)
+ for fn in (linear_cls.__dict__.get("forward_native"), stash):
+ while callable(fn) and fn not in seen:
+ seen.append(fn)
+ if (
+ home is not None
+ and getattr(getattr(fn, "__code__", None), "co_filename", None) == home
+ ):
+ return fn
+ fn = getattr(fn, "__wrapped__", None)
+ return None
+
+
+def set_compiled_dequant(on: bool) -> None:
+ _STATE["compiled_dequant"] = bool(on)
+
+
+def _exact_forward(self: Any, inputs: Any) -> Any:
+ weight = self.weight
+ dtype = self.compute_dtype
+ quant_type = getattr(weight, "quant_type", None)
+ if (
+ dtype is None
+ or quant_type is None
+ or weight.dim() != 2
+ or _STATE.get("compiled_dequant")
+ or exact_function(quant_type) is None
+ or not row_aligned(weight, quant_type)
+ ):
+ return _STATE["forward"](self, inputs)
+ import torch
+
+ dense = dequantize_cast(weight.as_tensor(), quant_type, dtype)
+ bias = self.bias.to(dtype) if self.bias is not None else None
+ return torch.nn.functional.linear(inputs, dense, bias)
+
+
+def stock_forward() -> Any:
+ return _STATE.get("forward")
+
+
+def install_exact_dequant(logger: Any = None) -> list:
+ if not exact_enabled():
+ return []
+ try:
+ import gguf
+ from diffusers.quantizers.gguf import utils as gguf_utils
+ except Exception: # noqa: BLE001 - no GGUF support in this diffusers
+ return []
+ table = getattr(gguf_utils, "dequantize_functions", None)
+ if not isinstance(table, dict):
+ return []
+ _STATE["table"] = table
+ _STATE["sizes"] = gguf.GGML_QUANT_SIZES
+ replaced = []
+ for name, fn in _FUNCTIONS.items():
+ qtype = getattr(gguf.GGMLQuantizationType, name, None)
+ if qtype is None or qtype not in table or table[qtype] is fn:
+ continue
+ _replaced.setdefault(qtype, table[qtype])
+ table[qtype] = fn
+ replaced.append(name)
+ linear_cls = getattr(gguf_utils, "GGUFLinear", None)
+ stock = _stock_forward(linear_cls) if linear_cls is not None else None
+ if stock is not None:
+ _STATE.setdefault("forward", stock)
+ if linear_cls.__dict__.get("forward_native") is stock:
+ linear_cls.forward_native = _exact_forward
+ if replaced and logger is not None:
+ logger.info("diffusion.gguf: llama.cpp-exact fp32 dequant for %s", ", ".join(replaced))
+ return replaced
+
+
+def uninstall_exact_dequant() -> None:
+ table = _STATE.get("table")
+ if isinstance(table, dict):
+ for qtype, original in _replaced.items():
+ if table.get(qtype) in _EXACT:
+ table[qtype] = original
+ _replaced.clear()
+ try:
+ from diffusers.quantizers.gguf import utils as gguf_utils
+ except Exception: # noqa: BLE001
+ return
+ linear_cls = getattr(gguf_utils, "GGUFLinear", None)
+ if linear_cls is not None and linear_cls.__dict__.get("forward_native") is _exact_forward:
+ linear_cls.forward_native = _STATE["forward"]
diff --git a/studio/backend/core/inference/diffusion_gguf_pipeline.py b/studio/backend/core/inference/diffusion_gguf_pipeline.py
index 7839975a955..fa760b6127b 100644
--- a/studio/backend/core/inference/diffusion_gguf_pipeline.py
+++ b/studio/backend/core/inference/diffusion_gguf_pipeline.py
@@ -66,9 +66,11 @@ def load_whole_pipeline_gguf(
import diffusers.loaders.single_file as single_file_mod
from diffusers.loaders.single_file_utils import load_single_file_checkpoint
+ from .diffusion_gguf_exact import install_exact_dequant
from .diffusion_gguf_lowbit import install_lowbit_iq
install_lowbit_iq() # diffusers' reader refuses unknown types, so before the read
+ install_exact_dequant()
checkpoint = load_single_file_checkpoint(
path,
local_files_only = pipe_kwargs.get("local_files_only"),
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index a77724fee9f..9662f9abaac 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -9904,8 +9904,11 @@ class VideoBackend:
"local_files_only": local_files_only,
}
if kind == "gguf":
+ from .diffusion_gguf_exact import install_exact_dequant
from .diffusion_gguf_lowbit import install_lowbit_iq
+
install_lowbit_iq(logger)
+ install_exact_dequant(logger)
sf_kwargs["quantization_config"] = diffusers.GGUFQuantizationConfig(
compute_dtype = dtype
)
diff --git a/studio/backend/core/inference/video_ltx2.py b/studio/backend/core/inference/video_ltx2.py
index 2c14cc8cb9f..d4b4c9d6c64 100644
--- a/studio/backend/core/inference/video_ltx2.py
+++ b/studio/backend/core/inference/video_ltx2.py
@@ -1330,8 +1330,11 @@ def load_ltx23_transformer(
# The load plan's commit for that hub-id config (absent for an unpinned load).
kwargs.update(pinned_revision_kwargs(base_repo))
if is_gguf:
+ from .diffusion_gguf_exact import install_exact_dequant
from .diffusion_gguf_lowbit import install_lowbit_iq
+
install_lowbit_iq()
+ install_exact_dequant()
kwargs["quantization_config"] = diffusers.GGUFQuantizationConfig(compute_dtype = torch_dtype)
elif device is not None:
# Else the meta load copies every tensor back to the host.
@@ -1635,8 +1638,11 @@ def _assemble_ltx23_pipeline(
state = _load_checkpoint_without_dit(checkpoint_path)
if state is None:
if is_gguf:
+ from .diffusion_gguf_exact import install_exact_dequant
from .diffusion_gguf_lowbit import install_lowbit_iq
+
install_lowbit_iq() # before the read: diffusers' reader refuses unknown types
+ install_exact_dequant()
state = load_single_file_checkpoint(str(checkpoint_path))
groups = _split_checkpoint(state)
del state
diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.py
index 080f9e0bbe2..f38726d4a0b 100644
--- a/studio/backend/core/inference/video_ltx25.py
+++ b/studio/backend/core/inference/video_ltx25.py
@@ -362,8 +362,11 @@ def load_ltx25_transformer(
from .video_ltx2 import _ltx23_pre_convert, ltx23_is_dit_key
if is_gguf:
+ from .diffusion_gguf_exact import install_exact_dequant
from .diffusion_gguf_lowbit import install_lowbit_iq
+
install_lowbit_iq() # before the read: diffusers' reader refuses unknown types
+ install_exact_dequant()
state = load_single_file_checkpoint(str(checkpoint_path))
dit_state = _ltx23_pre_convert({k: v for k, v in state.items() if ltx23_is_dit_key(k)})
del state
@@ -591,9 +594,11 @@ def _read_tensors(path: str, keep: Callable[[str], bool]) -> dict[str, Any]:
if not str(path).lower().endswith(".safetensors"):
from diffusers.loaders.single_file_utils import load_single_file_checkpoint
+ from .diffusion_gguf_exact import install_exact_dequant
from .diffusion_gguf_lowbit import install_lowbit_iq
install_lowbit_iq()
+ install_exact_dequant()
return {k: v for k, v in load_single_file_checkpoint(str(path)).items() if keep(k)}
from safetensors import safe_open
diff --git a/studio/backend/tests/test_diffusion_gguf_compile.py b/studio/backend/tests/test_diffusion_gguf_compile.py
index 6ce187642fd..d84656eb22e 100644
--- a/studio/backend/tests/test_diffusion_gguf_compile.py
+++ b/studio/backend/tests/test_diffusion_gguf_compile.py
@@ -18,9 +18,13 @@ from core.inference import diffusion_gguf_compile as gc # noqa: E402
@pytest.fixture(autouse = True)
def _clean():
# Always start and end from a clean, unpatched state so tests do not leak the process-wide patch into each other.
+ from core.inference import diffusion_gguf_exact as ex
+
gc.uninstall_all()
+ ex.uninstall_exact_dequant()
yield
gc.uninstall_all()
+ ex.uninstall_exact_dequant()
def test_compiled_dequant_install_uninstall_reversible():
diff --git a/studio/backend/tests/test_diffusion_gguf_exact.py b/studio/backend/tests/test_diffusion_gguf_exact.py
new file mode 100644
index 00000000000..b1983d0b90f
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_gguf_exact.py
@@ -0,0 +1,467 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""llama.cpp-exact GGUF dequant: fp32 bit-identical to ggml-quants.c (vendored digests of its output, no C compiler
+needed) and to gguf-py, NaN / inf / subnormal / -0 scale bits included, one rounding to the compute dtype, eager and
+compiled (no graph breaks, one dynamic graph per type), install / uninstall / kill switch, and the GGUFLinear
+forward. Runs on CPU."""
+
+from __future__ import annotations
+
+import hashlib
+
+import pytest
+
+torch = pytest.importorskip("torch")
+np = pytest.importorskip("numpy")
+gguf = pytest.importorskip("gguf")
+gguf_utils = pytest.importorskip("diffusers.quantizers.gguf.utils")
+
+from core.inference import diffusion_gguf_compile as gc # noqa: E402
+from core.inference import diffusion_gguf_exact as ex # noqa: E402
+
+T = gguf.GGMLQuantizationType
+TYPES = list(ex._FUNCTIONS)
+
+# byte offsets of each block's fp16 fields (ggml-common.h block structs)
+F16_OFFSETS = {
+ "Q4_0": (0,), "Q4_1": (0, 2), "Q5_0": (0,), "Q5_1": (0, 2), "Q8_0": (0,),
+ "Q2_K": (80, 82), "Q3_K": (108,), "Q4_K": (0, 2), "Q5_K": (0, 2), "Q6_K": (208,), "IQ4_XS": (0,),
+} # fmt: skip
+# +inf, -inf, qNaN, sNaN, -qNaN, -0, +0, smallest subnormal, largest subnormal, max, -max, smallest normal
+SPECIAL = (
+ 0x7C00,
+ 0xFC00,
+ 0x7E00,
+ 0x7C01,
+ 0xFE00,
+ 0x8000,
+ 0x0000,
+ 0x0001,
+ 0x03FF,
+ 0x7BFF,
+ 0xFBFF,
+ 0x0400,
+)
+ORACLE_BLOCKS = {name: 37 for name in F16_OFFSETS} # odd: no even-count assumption
+# ggml-org/llama.cpp ggml-quants.c dequantize_row_* (f477f79, -O0, no FMA) on oracle_blocks(name, 37): sha256 of the
+# fp32 output, NaN canonicalised. Regenerate with scripts/ggml_parity/make_oracle.py against a compiled ggml-quants.c.
+GGML_DIGESTS = {
+ "Q4_0": "11f2f408ad3ecc40298fc34e7d36e010e5a7f3338b2e49c3d67d1c1ea801c95f",
+ "Q4_1": "ea2dc4832791b6bb8c0a1800d75a153840f29b89064a6909109786ec0a6538b6",
+ "Q5_0": "84de0e848a5d8f7e62b9eadc0d9ee4f254e5f426643723da9ade8ed8a1e3ede9",
+ "Q5_1": "9cf75e735c74127787ffe0c1d5b02189c9aa5ac9339d3714693c7ed57e9e2e4b",
+ "Q8_0": "f46e3b0ed5965df250f13728201a942c5f06de257a2a7ae96d20c7fc1545d729",
+ "Q2_K": "d5437758841860ef8b258b2c7d4fc223bc8421b371b0e0c22f0da3b6a10e466f",
+ "Q3_K": "7a2776ee811eaa09ed093f1ebc856869783bffc16c4999ee95cf9735194edbe4",
+ "Q4_K": "5cae66ba58e1344b4c7a8ff706a5f8ce7e7d9bf346f6a3f20dc341e1e1f9d567",
+ "Q5_K": "28591e1ba7c6aa0f852e8107ad503b0e6b4c2e3a7b9e4718c800505301a9c03c",
+ "Q6_K": "94229b879c82509f6cdefa56bdd8852fefd787fe9946aa36f03997f1c57bb680",
+ "IQ4_XS": "510c1a39bb8c42cba718de1238d99df8ef6abaa6d8a84a0a6ed89f32f1fa221d",
+}
+
+
+def oracle_blocks(name: str, n: int):
+ """Deterministic blocks: sha256 counter bytes; the fp16 fields cycle through SPECIAL for the first 24 blocks,
+ then take normal values of either sign with exponents 2^-6 .. 2^-1."""
+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]
+ stream = b"".join(
+ hashlib.sha256(f"{name}:{i}".encode()).digest() for i in range((n * sz) // 32 + 1)
+ )
+ raw = np.frombuffer(stream[: n * sz], dtype = np.uint8).reshape(n, sz).copy()
+ for k, off in enumerate(F16_OFFSETS[name]):
+ u = raw[:, off].astype(np.uint16) | (raw[:, off + 1].astype(np.uint16) << 8)
+ normal = (u & 0x83FF) | ((9 + (u >> 10) % 6) << 10).astype(np.uint16)
+ for b in range(n):
+ bits = SPECIAL[(b + 5 * k) % len(SPECIAL)] if b < 2 * len(SPECIAL) else int(normal[b])
+ raw[b, off], raw[b, off + 1] = bits & 0xFF, bits >> 8
+ return raw
+
+
+def canonical_digest(values) -> str:
+ bits = np.ascontiguousarray(values, dtype = np.float32).reshape(-1).view(np.uint32).copy()
+ bits[np.isnan(np.asarray(values, dtype = np.float32).reshape(-1))] = 0x7FC00000
+ return hashlib.sha256(bits.tobytes()).hexdigest()
+
+
+def _same(got, ref) -> bool:
+ """Equal bits wherever not NaN, NaN at the same places (fp32 / bf16 / fp16)."""
+ got = got.reshape(-1)
+ ref = ref.reshape(-1)
+ int_dtype = torch.int32 if got.dtype == torch.float32 else torch.int16
+ nan = torch.isnan(ref)
+ return bool(
+ got.dtype == ref.dtype
+ and torch.equal(torch.isnan(got), nan)
+ and torch.equal(got[~nan].view(int_dtype), ref[~nan].view(int_dtype))
+ )
+
+
+def _random_blocks(
+ name: str,
+ n: int,
+ seed: int = 0,
+ special: bool = True,
+):
+ rng = np.random.default_rng(seed)
+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]
+ raw = rng.integers(0, 256, size = (n, sz), dtype = np.uint8)
+ for off in F16_OFFSETS[name]:
+ v = (rng.standard_normal(n) * 0.02).astype(np.float16).view(np.uint16)
+ if special:
+ k = min(n // 4, 48)
+ v[rng.choice(n, size = k, replace = False)] = rng.choice(SPECIAL, size = k)
+ raw[:, off : off + 2] = v.view(np.uint8).reshape(-1, 2)
+ return raw
+
+
[email protected](autouse = True)
+def _clean(monkeypatch):
+ from core.inference import diffusion_compile_config as cc
+
+ monkeypatch.delenv(ex.EXACT_ENV, raising = False)
+ cc._reset_for_tests()
+ gc.uninstall_all()
+ ex.uninstall_exact_dequant()
+ torch._dynamo.reset()
+ yield
+ gc.uninstall_all()
+ ex.uninstall_exact_dequant()
+ cc._reset_for_tests()
+
+
+def test_digest_table_covers_every_type():
+ assert set(GGML_DIGESTS) == set(TYPES) == set(F16_OFFSETS)
+
+
[email protected]("name", TYPES)
+def test_fp32_matches_the_vendored_ggml_output(name):
+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]
+ raw = oracle_blocks(name, ORACLE_BLOCKS[name])
+ out = ex._FUNCTIONS[name](torch.from_numpy(raw), bs, sz)
+ assert out.dtype == torch.float32 and out.shape == (raw.shape[0], bs)
+ assert canonical_digest(out.numpy()) == GGML_DIGESTS[name]
+ assert np.isnan(out.numpy()).any() and np.isinf(out.numpy()).any()
+
+
[email protected]("name", TYPES)
[email protected]("n", [1, 7, 513])
+def test_fp32_is_bit_identical_to_gguf_py(name, n):
+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]
+ raw = _random_blocks(name, n, seed = n)
+ with np.errstate(all = "ignore"):
+ ref = torch.from_numpy(
+ np.ascontiguousarray(gguf.quants.dequantize(raw, getattr(T, name)), dtype = np.float32)
+ )
+ assert _same(ex._FUNCTIONS[name](torch.from_numpy(raw), bs, sz), ref.reshape(n, bs))
+
+
[email protected]("name", TYPES)
[email protected]("dtype", [torch.bfloat16, torch.float16])
+def test_a_compute_dtype_is_one_rounding_of_the_fp32_result(name, dtype):
+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]
+ raw = torch.from_numpy(_random_blocks(name, 301, seed = 3))
+ fp32 = ex._FUNCTIONS[name](raw, bs, sz)
+ assert _same(ex._FUNCTIONS[name](raw, bs, sz, dtype), fp32.to(dtype))
+
+
+def test_diffusers_fp16_functions_are_not_exact():
+ # if diffusers' own function were already exact, this module would be redundant
+ bs, sz = gguf.GGML_QUANT_SIZES[T.Q4_K]
+ raw = torch.from_numpy(_random_blocks("Q4_K", 512, seed = 5))
+ stock = gguf_utils.dequantize_functions[T.Q4_K](raw, bs, sz)
+ ours = ex.dequantize_blocks_Q4_K(raw, bs, sz)
+ fin = torch.isfinite(ours) & torch.isfinite(stock.float())
+ assert not torch.equal(stock.to(torch.bfloat16)[fin], ours.to(torch.bfloat16)[fin])
+
+
+def _gguf_py(raw, qtype):
+ return torch.from_numpy(
+ np.ascontiguousarray(gguf.quants.dequantize(raw.numpy(), qtype), np.float32)
+ )
+
+
[email protected]("compiled", [False, True])
+def test_q8_0_bf16_is_not_rounded_through_fp16(compiled):
+ # d = 1371 / 4096, q = 3: fp16 d * q = 1 + 2**-8 (a bf16 tie, to 1); ggml's fp32 1 + 2**-8 + 2**-12 -> 1 + 2**-7
+ raw = torch.zeros(1, 34, dtype = torch.uint8)
+ raw[0, :2] = torch.tensor([np.float16(1371 / 4096)]).view(torch.uint8)
+ raw[0, 2:] = 3
+ fn = (
+ torch.compile(ex.dequantize_blocks_Q8_0, fullgraph = True)
+ if compiled
+ else ex.dequantize_blocks_Q8_0
+ )
+ got = fn(raw, 32, 34, torch.bfloat16)
+ assert torch.equal(got, torch.full_like(got, 1 + 2**-7))
+ assert torch.equal(
+ got.float(), _gguf_py(raw, T.Q8_0).to(torch.bfloat16).float().reshape(got.shape)
+ )
+ stock = gguf_utils.dequantize_functions[T.Q8_0]
+ if stock is not ex.dequantize_blocks_Q8_0:
+ assert float(stock(raw, 32, 34).to(torch.bfloat16)[0, 0]) == 1.0
+
+
[email protected]("compiled", [False, True])
+def test_iq4_xs_large_scale_does_not_overflow_in_fp16(compiled):
+ # d = 4096, ls - 32 = -32: ggml's fp32 dl = -131072 (fp16 would be -inf); * kvalue -127 is finite in fp32
+ bs, sz = gguf.GGML_QUANT_SIZES[T.IQ4_XS]
+ raw = torch.zeros(1, sz, dtype = torch.uint8)
+ raw[0, :2] = torch.tensor([np.float16(4096)]).view(torch.uint8)
+ fn = (
+ torch.compile(ex.dequantize_blocks_IQ4_XS, fullgraph = True)
+ if compiled
+ else ex.dequantize_blocks_IQ4_XS
+ )
+ got = fn(raw, bs, sz).reshape(-1)
+ assert torch.equal(got, _gguf_py(raw, T.IQ4_XS).reshape(-1))
+ assert torch.equal(got, torch.full_like(got, 4096.0 * 32 * 127))
+
+
[email protected]("name", TYPES)
[email protected]("rows", [1, 3, 5])
+def test_dequantize_cast_keeps_rows_of_whole_blocks(name, rows):
+ qtype = getattr(T, name)
+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]
+ raw = torch.from_numpy(_random_blocks(name, rows * 3, seed = rows)).reshape(rows, 3 * sz)
+ ex.install_exact_dequant()
+ dense = ex.dequantize_cast(raw, qtype, torch.bfloat16)
+ assert dense.shape == (rows, 3 * bs) and dense.dtype == torch.bfloat16
+ ref = gguf_utils.dequantize_gguf_tensor(gguf_utils.GGUFParameter(raw, quant_type = qtype)).to(
+ torch.bfloat16
+ )
+ assert _same(dense, ref)
+ assert ex.row_aligned(raw, qtype) and not ex.row_aligned(raw[:, 1:], qtype)
+
+
+def test_install_replaces_only_the_fp16_types_and_uninstall_restores_them():
+ table = gguf_utils.dequantize_functions
+ before = dict(table)
+ supported = list(gguf_utils.SUPPORTED_GGUF_QUANT_TYPES)
+ assert ex.install_exact_dequant() == TYPES
+ assert ex.install_exact_dequant() == []
+ for name in TYPES:
+ assert table[getattr(T, name)] is ex._FUNCTIONS[name]
+ assert ex.exact_function(getattr(T, name)) is ex._FUNCTIONS[name]
+ for qtype in (T.IQ4_NL, T.BF16):
+ assert table[qtype] is before[qtype] and ex.exact_function(qtype) is None
+ assert gguf_utils.SUPPORTED_GGUF_QUANT_TYPES == supported
+ assert gguf_utils.GGUFLinear.forward_native is ex._exact_forward
+ ex.uninstall_exact_dequant()
+ ex.uninstall_exact_dequant()
+ assert table == before
+ assert ex.exact_function(T.Q4_K) is None
+ assert gguf_utils.GGUFLinear.forward_native is ex._STATE["forward"]
+ assert ex._STATE["forward"].__module__ == gguf_utils.__name__
+
+
[email protected]("value", ["0", "false", "off", "no"])
+def test_kill_switch_keeps_diffusers_functions(monkeypatch, value):
+ monkeypatch.setenv(ex.EXACT_ENV, value)
+ before = dict(gguf_utils.dequantize_functions)
+ forward = gguf_utils.GGUFLinear.forward_native
+ assert ex.install_exact_dequant() == []
+ assert gguf_utils.dequantize_functions == before
+ assert gguf_utils.GGUFLinear.forward_native is forward
+
+
+def test_uninstall_leaves_a_later_replacement_alone():
+ ex.install_exact_dequant()
+ other = lambda *a, **k: None # noqa: E731
+ gguf_utils.dequantize_functions[T.Q8_0] = other
+ try:
+ ex.uninstall_exact_dequant()
+ assert gguf_utils.dequantize_functions[T.Q8_0] is other
+ finally:
+ gguf_utils.dequantize_functions[T.Q8_0] = gguf_utils.dequantize_blocks_Q8_0
+
+
[email protected]("name", TYPES)
+def test_compiled_has_no_graph_breaks_and_one_graph_for_every_weight(name):
+ # as the patched GGUFLinear forward runs it: one compile per type, only the block count dynamic
+ from torch._dynamo.testing import CompileCounter
+
+ qtype = getattr(T, name)
+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]
+ counter = CompileCounter()
+ fn = torch.compile(ex._FUNCTIONS[name], backend = counter, fullgraph = True)
+ for rows, n in ((2, 8), (13, 13), (4, 64), (7, 301)):
+ raw = torch.from_numpy(_random_blocks(name, rows * n, seed = n)).reshape(rows, n * sz)
+ got = ex.dequantize_cast(raw, qtype, torch.bfloat16, fn)
+ assert got.shape == (rows, n * bs)
+ assert _same(
+ got,
+ ex._FUNCTIONS[name](raw.reshape(-1, sz), bs, sz).to(torch.bfloat16).reshape(rows, -1),
+ )
+ assert counter.frame_count == 1
+
+
[email protected]("name", ["Q8_0", "Q4_K", "Q6_K", "Q5_1", "Q3_K", "Q2_K"])
+ "emulate", [False, True], ids = ["inductor_default", "emulate_precision_casts"]
+)
+def test_inductor_dequant_cast_is_bit_identical(name, emulate):
+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]
+ with torch._inductor.config.patch(emulate_precision_casts = emulate):
+ fn = torch.compile(ex._FUNCTIONS[name])
+ for n in (16, 37):
+ raw = torch.from_numpy(_random_blocks(name, n, seed = n))
+ fp32 = ex._FUNCTIONS[name](raw, bs, sz)
+ assert _same(ex.dequantize_cast(raw, getattr(T, name), torch.float32, fn), fp32)
+ assert _same(
+ ex.dequantize_cast(raw, getattr(T, name), torch.bfloat16, fn),
+ fp32.to(torch.bfloat16),
+ )
+
+
[email protected]("name", ["Q2_K", "Q4_K", "Q5_K"])
+def test_a_zero_code_under_a_negative_scale_and_zero_min_is_negative_zero(name):
+ # ggml: d1 * 0 - (+0) = -0 when d1 < 0. A lowering of ``-m + d1 * q`` as ``(0 - m) + ...`` gives +0.
+ qtype = getattr(T, name)
+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]
+ raw = np.zeros((4, sz), dtype = np.uint8)
+ d_off, dmin_off = F16_OFFSETS[name]
+ raw[:, d_off : d_off + 2] = np.array([0xBC00], np.uint16).view(np.uint8) # d = -1
+ raw[:, dmin_off : dmin_off + 2] = np.array([0x3C00], np.uint16).view(np.uint8) # dmin = +1
+ if name == "Q2_K":
+ raw[:, :16] = 0x01 # scale 1, min 0
+ else:
+ raw[:, 4:8] = 0x01 # sc = 1, m = 0 for the first four groups
+ with np.errstate(all = "ignore"):
+ ref = torch.from_numpy(
+ np.ascontiguousarray(gguf.quants.dequantize(raw, qtype), np.float32)
+ ).reshape(4, bs)
+ assert (torch.signbit(ref) & (ref == 0)).any()
+ t = torch.from_numpy(raw)
+ assert _same(ex._FUNCTIONS[name](t, bs, sz), ref)
+ assert _same(
+ ex.dequantize_cast(t, qtype, torch.float32, torch.compile(ex._FUNCTIONS[name])), ref
+ )
+
+
+needs_fused = pytest.mark.skipif(
+ not gc._torch_supports_fused_q8_0(torch.__version__),
+ reason = "the patched forward needs torch >= 2.7",
+)
+
+
+def _linear(qtype, rows, cols, dtype):
+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]
+ raw = torch.from_numpy(_random_blocks(qtype.name, rows * cols // bs, seed = 11, special = False))
+ lin = gguf_utils.GGUFLinear(cols, rows, bias = False, compute_dtype = dtype)
+ lin.weight = gguf_utils.GGUFParameter(raw.reshape(rows, cols // bs * sz), quant_type = qtype)
+ return lin
+
+
+@needs_fused
[email protected]("name", ["Q8_0", "Q4_K", "Q6_K"])
[email protected]("dtype", [torch.bfloat16, torch.float16])
+def test_patched_forward_uses_the_exact_fused_dequant(name, dtype, monkeypatch):
+ qtype = getattr(T, name)
+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]
+ ex.install_exact_dequant()
+ assert gc.install_compiled_dequant()
+ calls = []
+ real = ex.dequantize_cast
+ monkeypatch.setattr(ex, "dequantize_cast", lambda *a, **k: calls.append(a[1]) or real(*a, **k))
+ lin = _linear(qtype, 6, 512, dtype)
+ raw = lin.weight.as_tensor()
+ dense = ex._FUNCTIONS[name](raw.reshape(-1, sz), bs, sz).reshape(6, 512).to(dtype)
+ x = torch.randn(3, 512).to(dtype)
+ out = lin(x)
+ assert calls == [qtype]
+ assert torch.equal(out, torch.nn.functional.linear(x, dense))
+
+
+@needs_fused
+def test_kill_switch_restores_the_stock_q8_0_forward(monkeypatch):
+ monkeypatch.setenv(ex.EXACT_ENV, "0")
+ assert ex.install_exact_dequant() == []
+ assert gc.install_compiled_dequant()
+ calls = []
+ monkeypatch.setattr(ex, "dequantize_cast", lambda *a, **k: calls.append(1))
+ lin = _linear(T.Q8_0, 4, 64, torch.bfloat16)
+ lin(torch.ones(1, 64, dtype = torch.bfloat16))
+ assert calls == []
+
+
[email protected]("name", ["Q8_0", "Q4_K", "Q6_K", "Q2_K"])
[email protected]("dtype", [torch.bfloat16, torch.float16])
+def test_eager_forward_dequantises_straight_to_the_compute_dtype(name, dtype, monkeypatch):
+ qtype = getattr(T, name)
+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]
+ ex.install_exact_dequant()
+ calls = []
+ real = ex.dequantize_cast
+ monkeypatch.setattr(ex, "dequantize_cast", lambda *a, **k: calls.append(a[2]) or real(*a, **k))
+ lin = _linear(qtype, 6, 512, dtype)
+ raw = lin.weight.as_tensor()
+ x = torch.randn(3, 512).to(dtype)
+ out = lin(x)
+ assert calls == [dtype]
+ dense = ex._FUNCTIONS[name](raw.reshape(-1, sz), bs, sz).reshape(6, 512).to(dtype)
+ assert torch.equal(out, torch.nn.functional.linear(x, dense))
+
+
+def test_eager_forward_leaves_other_types_to_diffusers(monkeypatch):
+ ex.install_exact_dequant()
+ calls = []
+ monkeypatch.setattr(ex, "dequantize_cast", lambda *a, **k: calls.append(1))
+ raw = torch.randn(4, 64).to(torch.bfloat16).view(torch.uint8)
+ lin = gguf_utils.GGUFLinear(64, 4, bias = False, compute_dtype = torch.bfloat16)
+ lin.weight = gguf_utils.GGUFParameter(raw, quant_type = T.BF16)
+ x = torch.randn(2, 64).to(torch.bfloat16)
+ assert torch.equal(lin(x), torch.nn.functional.linear(x, raw.view(torch.bfloat16)))
+ assert calls == []
+
+
[email protected]("name", ["Q8_0", "Q4_K", "Q6_K"])
+def test_a_compiled_block_traces_the_eager_forward_without_graph_breaks(name):
+ # the max tier compiles whole blocks around GGUFLinear: the forward and dequant must trace in one graph
+ from torch._dynamo.testing import CompileCounter
+
+ ex.install_exact_dequant()
+ lin = _linear(getattr(T, name), 6, 512, torch.bfloat16)
+ x = torch.randn(3, 512).to(torch.bfloat16)
+ counter = CompileCounter()
+ out = torch.compile(lin, backend = counter, fullgraph = True)(x)
+ assert counter.frame_count == 1
+ assert torch.equal(out, lin(x))
+
+
+def test_install_order_with_the_compiled_forward_does_not_matter():
+ stock = gguf_utils.GGUFLinear.forward_native
+ assert ex._stock_forward(gguf_utils.GGUFLinear) is stock
+ # compiled forward first: its wrapper stays, diffusers' own forward is what the exact path falls back to
+ assert gc.install_compiled_dequant()
+ wrapper = gguf_utils.GGUFLinear.forward_native
+ assert ex.install_exact_dequant() == TYPES
+ assert gguf_utils.GGUFLinear.forward_native is wrapper
+ assert ex.stock_forward() is stock
+ gc.uninstall_all()
+ assert gguf_utils.GGUFLinear.forward_native is stock
+ # exact first, compiled on top, then out-of-order teardown: still back to diffusers' forward
+ ex.install_exact_dequant()
+ assert gguf_utils.GGUFLinear.forward_native is ex._exact_forward
+ assert gc.install_compiled_dequant()
+ ex.uninstall_exact_dequant()
+ gc.uninstall_all()
+ ex.uninstall_exact_dequant()
+ assert gguf_utils.GGUFLinear.forward_native is stock
+
+
+@needs_fused
+def test_fused_cast_switch_off_runs_the_compiled_stock_dequant_once_rounded(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", "0")
+ ex.install_exact_dequant()
+ assert gc.install_compiled_dequant()
+ calls = []
+ monkeypatch.setattr(ex, "dequantize_cast", lambda *a, **k: calls.append(1))
+ qtype = T.Q4_K
+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]
+ lin = _linear(qtype, 6, 512, torch.bfloat16)
+ raw = lin.weight.as_tensor()
+ x = torch.randn(3, 512).to(torch.bfloat16)
+ dense = ex._FUNCTIONS["Q4_K"](raw.reshape(-1, sz), bs, sz).reshape(6, 512).to(torch.bfloat16)
+ assert torch.equal(lin(x), torch.nn.functional.linear(x, dense))
+ assert calls == []
diff --git a/studio/backend/tests/test_diffusion_gguf_fused_cast.py b/studio/backend/tests/test_diffusion_gguf_fused_cast.py
index 11be2c8567e..604616c8185 100644
--- a/studio/backend/tests/test_diffusion_gguf_fused_cast.py
+++ b/studio/backend/tests/test_diffusion_gguf_fused_cast.py
@@ -40,13 +40,18 @@ FP16_TYPES = [T.Q4_K, T.Q5_K, T.Q6_K, T.Q3_K, T.Q2_K, T.Q4_0, T.Q4_1, T.Q5_0, T.
def _clean(monkeypatch):
from core.inference import diffusion_compile_config as cc
+ from core.inference import diffusion_gguf_exact as ex
+
monkeypatch.delenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", raising = False)
cc._reset_for_tests()
gc.uninstall_all()
+ # diffusers' fp16 functions: the path UNSLOTH_DIFFUSION_GGUF_EXACT=0 keeps
+ ex.uninstall_exact_dequant()
torch._dynamo.reset()
with torch._inductor.config.patch(emulate_precision_casts = False):
yield
gc.uninstall_all()
+ ex.uninstall_exact_dequant()
cc._reset_for_tests()
diff --git a/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py
index 18f3c072048..c4432de6774 100644
--- a/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py
+++ b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py
@@ -29,12 +29,17 @@ def _clean():
# re-applies; start every test from torch's defaults so results do not depend on test order.
from core.inference import diffusion_compile_config as cc
+ from core.inference import diffusion_gguf_exact as ex
+
cc._reset_for_tests()
gc.uninstall_all()
+ # diffusers' fp16 functions: the path UNSLOTH_DIFFUSION_GGUF_EXACT=0 keeps
+ ex.uninstall_exact_dequant()
torch._dynamo.reset()
with torch._inductor.config.patch(emulate_precision_casts = False):
yield
gc.uninstall_all()
+ ex.uninstall_exact_dequant()
cc._reset_for_tests()
diff --git a/studio/backend/tests/test_diffusion_gguf_lowbit.py b/studio/backend/tests/test_diffusion_gguf_lowbit.py
index 5df3dd99aad..ede15a7dd5e 100644
--- a/studio/backend/tests/test_diffusion_gguf_lowbit.py
+++ b/studio/backend/tests/test_diffusion_gguf_lowbit.py
@@ -20,10 +20,15 @@ TYPES = ["IQ1_S", "IQ1_M", "IQ2_XXS", "IQ2_XS", "IQ2_S", "IQ3_XXS", "IQ3_S"]
@pytest.fixture(autouse = True)
def _clean(monkeypatch):
+ from core.inference import diffusion_gguf_exact as ex
+
monkeypatch.delenv(lb.LOWBIT_IQ_ENV, raising = False)
lb.uninstall_lowbit_iq()
+ ex.uninstall_exact_dequant()
yield
lb.uninstall_lowbit_iq()
+ # the loaders under test also install the exact dequant
+ ex.uninstall_exact_dequant()
def _blocks(