Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio GGUF: dequantise bit-identical to llama.cpp, rounded once to the compute dtype (#17204)
Daniel Hanb67e5630656e
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 3d0c22fbb72..a8a1f0609d7 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -323,6 +323,7 @@ from .diffusion_offload_prefetch import cross_forward_render, cross_forward_stepfrom .diffusion_vae_fp16 import enable_fp16_vae_decodefrom .diffusion_vae_upcast import pipeline_upcasts_keep_storagefrom .diffusion_vae_tiling import install as install_wide_vae_tiles+from .diffusion_gguf_exact import install_exact_dequantfrom .diffusion_gguf_lowbit import install_lowbit_iqfrom .diffusion_attention import (apply_attention_backend,@@ -9943,6 +9944,7 @@ class DiffusionBackend:if kind == "gguf":# IQ1 / IQ2 / IQ3 tensors diffusers' table lacks would refuse the fileinstall_lowbit_iq(logger)+ install_exact_dequant(logger)# Dequantise the GGUF transformer on-device at the compute dtype.sf_kwargs["quantization_config"] = diffusers.GGUFQuantizationConfig(compute_dtype = dtypediff --git a/studio/backend/core/inference/diffusion_gguf_compile.py b/studio/backend/core/inference/diffusion_gguf_compile.pyindex 173ad4de45f..5aedefed939 100644--- a/studio/backend/core/inference/diffusion_gguf_compile.py+++ b/studio/backend/core/inference/diffusion_gguf_compile.py@@ -11,7 +11,8 @@ linear per forward. The fused kernel keeps the fp16 rounding of ``d * q`` beforeso the weight it produces is bit-identical to the stock one.Other fp16-dequantised types under a bf16 compute dtype are cast in the dequant's own buffer, bit-identical to the stock-cast (UNSLOTH_DIFFUSION_GGUF_FUSED_CAST=0 disables); a compiled-in cast was not bit-identical for Q4_K / Q5_K."""+cast (UNSLOTH_DIFFUSION_GGUF_FUSED_CAST=0 disables); a compiled-in cast was not bit-identical for Q4_K / Q5_K.+diffusion_gguf_exact types compile its dequant per type, cast inside the kernel."""from __future__ import annotations@@ -136,6 +137,9 @@ def install_compiled_dequant(logger: Any = None) -> bool:if apply_patch(gguf_utils, _DEQUANT_ATTR, compiled, force = True):_compiled_dequant_installed = True_compile_error = None+ from . import diffusion_gguf_exact # noqa: PLC0415++ diffusion_gguf_exact.set_compiled_dequant(True)_install_fused_q8_0(gguf_utils, logger)return Truereturn False@@ -194,8 +198,12 @@ def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:if not _torch_supports_fused_q8_0(torch.__version__):return False+ from . import diffusion_gguf_exact as gguf_exact # noqa: PLC0415+compiled = torch.compile(_q8_0_blocks_dequant_cast, dynamic = True)blocks_fn = _guard(compiled, _q8_0_blocks_dequant_cast, logger)+ # one compile per type, so no type evicts another+ exact_fns = {fn: _guard(torch.compile(fn), fn, logger) for fn in gguf_exact.EXACT_FUNCTIONS}stock_forward = getattr(linear_cls, _FORWARD_ATTR)F = torch.nn.functional@@ -206,7 +214,16 @@ def _install_fused_q8_0(gguf_utils: Any, logger: Any) -> bool:quant_type = getattr(weight, "quant_type", None)if dtype is None or weight.dim() != 2:return stock_forward(self, inputs)- if quant_type == _Q8_0:+ exact_fn = gguf_exact.exact_function(quant_type) if quant_type is not None else None+ if exact_fn is not None:+ if not (q8_0 if quant_type == _Q8_0 else fused_cast) or not gguf_exact.row_aligned(+ weight, quant_type+ ):+ return (gguf_exact.stock_forward() or stock_forward)(self, inputs)+ dense = gguf_exact.dequantize_cast(+ weight.as_tensor(), quant_type, dtype, exact_fns[exact_fn]+ )+ elif quant_type == _Q8_0:if not q8_0 or weight.shape[-1] % _Q8_0_TYPE_SIZE:return stock_forward(self, inputs)dense = _q8_0_dequant_cast(weight.as_tensor(), dtype, blocks_fn)@@ -254,6 +271,9 @@ def uninstall_compiled_dequant() -> None:if gguf_utils is not None:revert_patch(gguf_utils, _DEQUANT_ATTR)_compiled_dequant_installed = False+ from . import diffusion_gguf_exact # noqa: PLC0415++ diffusion_gguf_exact.set_compiled_dequant(False)def uninstall_all() -> None:diff --git a/studio/backend/core/inference/diffusion_gguf_exact.py b/studio/backend/core/inference/diffusion_gguf_exact.pynew file mode 100644index 00000000000..1f864317d43--- /dev/null+++ b/studio/backend/core/inference/diffusion_gguf_exact.py@@ -0,0 +1,446 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""GGUF dequant bit-identical to llama.cpp's ``dequantize_row_*`` (diffusers' rounds in fp16), rounded once to the+compute dtype. Products are exact in fp32, so only ``+ m`` / ``- m`` rounds and FMA cannot change a bit.+UNSLOTH_DIFFUSION_GGUF_EXACT=0 keeps diffusers'."""++from __future__ import annotations++import os+from typing import Any++EXACT_ENV = "UNSLOTH_DIFFUSION_GGUF_EXACT"+_QK_K = 256+_CAST_ON_STORE = ("cuda", "cpu")+_KVALUES_IQ4NL = (-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113)+++def exact_enabled() -> bool:+ return os.environ.get(EXACT_ENV, "").strip().lower() not in ("0", "false", "no", "off")+++def _scale(blocks: Any, word: int) -> Any:+ """fp16 at byte ``2 * word`` as fp32 ``(n, 1)``; view only 2 bytes: inductor copies a wider dtype view."""+ import torch+ return blocks[:, 2 * word : 2 * word + 2].view(torch.float16).to(torch.float32)+++def _half_math(blocks: Any, dtype: Any) -> Any:+ import torch+ if dtype != torch.float16 or torch.compiler.is_compiling():+ return None+ return blocks[:, :2].view(torch.float16)+++def _field(blocks: Any, start: int, stop: int, shape: tuple) -> Any:+ return blocks[:, start:stop].unflatten(1, shape)+++def _shifts(like: Any, start: int, stop: int, step: int, shape: tuple) -> Any:+ import torch+ return torch.arange(start, stop, step, device = like.device, dtype = torch.uint8).reshape(shape)+++def _emit(scale: Any, q: Any, m: Any, op: str, n: int, block_size: int, dtype: Any) -> Any:+ """``scale * q +/- m`` as ``(n, block_size)``, rounded once to ``dtype``. Only the contiguous result is reshaped+ (a strided reshape is 1.7x slower Q6_K compiled). Traced, not ``addcmul(-m, ...)``: Triton's ``0 - m`` loses -0."""+ import torch++ if (+ not torch.compiler.is_compiling()+ and dtype is not None+ and dtype != torch.float32+ and q.device.type in _CAST_ON_STORE+ ):+ out = torch.empty(+ torch.broadcast_shapes(scale.shape, q.shape), dtype = dtype, device = q.device+ )+ if op:+ torch.addcmul(m if op == "+" else -m, scale, q, out = out)+ else:+ torch.mul(scale, q, out = out)+ return out.reshape(n, block_size)+ y = scale * q+ if op == "+":+ y = y.add_(m)+ elif op == "-":+ y = y.sub_(m)+ y = y.reshape(n, block_size)+ return y if dtype is None or dtype == torch.float32 else y.to(dtype)+++def _q4_codes(blocks: Any, at: int) -> Any:+ """``(n, 2, 2, 8)``: element ``16h + 8a + b`` is nibble ``h`` of byte ``8a + b``."""+ return (_field(blocks, at, at + 16, (1, 2, 8)) >> _shifts(blocks, 0, 8, 4, (2, 1, 1))) & 0x0F+++def dequantize_blocks_Q4_0(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ import torch++ n = blocks.shape[0]+ q = _q4_codes(blocks, 2).to(torch.int8) - 8+ half = _half_math(blocks, dtype)+ if half is not None:+ return (half.reshape(n, 1, 1, 1) * q).reshape(n, 32)+ return _emit(_scale(blocks, 0).reshape(n, 1, 1, 1), q, None, "", n, 32, dtype)+++def dequantize_blocks_Q4_1(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ n = blocks.shape[0]+ d = _scale(blocks, 0).reshape(n, 1, 1, 1)+ m = _scale(blocks, 1).reshape(n, 1, 1, 1)+ return _emit(d, _q4_codes(blocks, 4), m, "+", n, 32, dtype)+++def _q5_codes(blocks: Any, qh_at: int) -> Any:+ """Code ``k`` takes bit ``k`` of the little-endian 32-bit ``qh``."""+ high = (_field(blocks, qh_at, qh_at + 4, (2, 2, 1)) >> _shifts(blocks, 0, 8, 1, (8,))) & 1+ return _q4_codes(blocks, qh_at + 4) | (high << 4)+++def dequantize_blocks_Q5_0(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ import torch++ n = blocks.shape[0]+ q = _q5_codes(blocks, 2).to(torch.int8) - 16+ half = _half_math(blocks, dtype)+ if half is not None:+ return (half.reshape(n, 1, 1, 1) * q).reshape(n, 32)+ return _emit(_scale(blocks, 0).reshape(n, 1, 1, 1), q, None, "", n, 32, dtype)+++def dequantize_blocks_Q5_1(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ n = blocks.shape[0]+ d = _scale(blocks, 0).reshape(n, 1, 1, 1)+ m = _scale(blocks, 1).reshape(n, 1, 1, 1)+ return _emit(d, _q5_codes(blocks, 4), m, "+", n, 32, dtype)+++def dequantize_blocks_Q8_0(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ import torch++ n = blocks.shape[0]+ q = blocks[:, 2:].view(torch.int8)+ half = _half_math(blocks, dtype)+ if half is not None:+ return half * q+ return _emit(_scale(blocks, 0), q, None, "", n, 32, dtype)+++def _k_codes(blocks: Any, at: int) -> Any:+ """``(n, chunk c, shift j, half h, 16)`` = ``(q[32c + 16h + l] >> 2j) & 3``; scale group ``8c + 2j + h``."""+ return (_field(blocks, at, at + 64, (2, 1, 2, 16)) >> _shifts(blocks, 0, 8, 2, (4, 1, 1))) & 3+++def dequantize_blocks_Q2_K(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ n = blocks.shape[0]+ scales = blocks[:, :16]+ d = _scale(blocks, 40)+ dmin = _scale(blocks, 41)+ dl = (d * (scales & 0x0F)).reshape(n, 2, 4, 2, 1)+ ml = (dmin * (scales >> 4)).reshape(n, 2, 4, 2, 1)+ return _emit(dl, _k_codes(blocks, 16), ml, "-", n, _QK_K, dtype)+++def dequantize_blocks_Q3_K(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ import torch++ n = blocks.shape[0]+ scales = blocks[:, 96:108]+ d = _scale(blocks, 54)+ # ggml's aux[] unpack: nibble of byte g % 8, 2 high bits from byte 8 + g % 4+ low = (scales[:, :8].unsqueeze(1) >> _shifts(blocks, 0, 8, 4, (2, 1))) & 0x0F+ high = (scales[:, 8:].unsqueeze(1) >> _shifts(blocks, 0, 8, 2, (4, 1))) & 3+ sc = (low.reshape(n, 16) | (high.reshape(n, 16) << 4)).to(torch.int8) - 32+ dl = (d * sc).reshape(n, 2, 4, 2, 1)+ # hmask bit 4c + j clear -> subtract 4+ hb = (_field(blocks, 0, 32, (1, 1, 2, 16)) >> _shifts(blocks, 0, 8, 1, (2, 4, 1, 1))) & 1+ q = _k_codes(blocks, 32).to(torch.int8) - ((hb ^ 1) << 2).to(torch.int8)+ return _emit(dl, q, None, "", n, _QK_K, dtype)+++def _scale_min_k4(scales: Any) -> tuple:+ """ggml's get_scale_min_k4 for j = 0..7."""+ import torch++ a, b, c = scales[:, :4], scales[:, 4:8], scales[:, 8:12]+ sc = torch.cat((a & 63, (c & 0x0F) | ((a >> 6) << 4)), dim = 1)+ m = torch.cat((b & 63, (c >> 4) | ((b >> 6) << 4)), dim = 1)+ return sc, m+++def _k4_scales(blocks: Any) -> tuple:+ n = blocks.shape[0]+ d = _scale(blocks, 0)+ dmin = _scale(blocks, 1)+ sc, m = _scale_min_k4(blocks[:, 4:16])+ return (d * sc).reshape(n, 4, 2, 1), (dmin * m).reshape(n, 4, 2, 1)+++def _k4_codes(blocks: Any, at: int) -> Any:+ return (_field(blocks, at, at + 128, (4, 1, 32)) >> _shifts(blocks, 0, 8, 4, (2, 1))) & 0x0F+++def dequantize_blocks_Q4_K(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ n = blocks.shape[0]+ d1, m1 = _k4_scales(blocks)+ return _emit(d1, _k4_codes(blocks, 16), m1, "-", n, _QK_K, dtype)+++def dequantize_blocks_Q5_K(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ n = blocks.shape[0]+ d1, m1 = _k4_scales(blocks)+ qh = (_field(blocks, 16, 48, (1, 1, 32)) >> _shifts(blocks, 0, 8, 1, (4, 2, 1))) & 1+ q = _k4_codes(blocks, 48) | (qh << 4)+ return _emit(d1, q, m1, "-", n, _QK_K, dtype)+++def dequantize_blocks_Q6_K(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ import torch++ n = blocks.shape[0]+ d = _scale(blocks, 104)+ sc = blocks[:, 192:208].view(torch.int8)+ # (n, chunk, nibble, half, 2, 16): quarter t = 2 * nibble + half, ggml's q1..q4+ ql = (_field(blocks, 0, 128, (2, 1, 2, 2, 16)) >> _shifts(blocks, 0, 8, 4, (2, 1, 1, 1))) & 0x0F+ qh = (_field(blocks, 128, 192, (2, 1, 1, 2, 16)) >> _shifts(blocks, 0, 8, 2, (2, 2, 1, 1))) & 3+ q = (ql | (qh << 4)).to(torch.int8) - 32+ return _emit((d * sc).reshape(n, 2, 2, 2, 2, 1), q, None, "", n, _QK_K, dtype)+++def dequantize_blocks_IQ4_XS(+ blocks: Any,+ block_size: Any,+ type_size: Any,+ dtype: Any = None,+) -> Any:+ import torch++ n = blocks.shape[0]+ d = _scale(blocks, 0)+ scales_h = blocks[:, 2:3].to(torch.int32) | (blocks[:, 3:4].to(torch.int32) << 8)+ shifts = torch.arange(0, 16, 2, device = blocks.device, dtype = torch.int32)+ high = ((scales_h >> shifts) & 3).to(torch.uint8)+ low = ((blocks[:, 4:8].unsqueeze(-1) >> _shifts(blocks, 0, 8, 4, (2,))) & 0x0F).reshape(n, 8)+ ls = (low | (high << 4)).to(torch.int8) - 32+ dl = (d * ls).reshape(n, 8, 1, 1)+ kvalues = torch.tensor(_KVALUES_IQ4NL, device = blocks.device, dtype = torch.float32)+ index = (_field(blocks, 8, 136, (8, 1, 16)) >> _shifts(blocks, 0, 8, 4, (2, 1))) & 0x0F+ return _emit(dl, kvalues[index.long()], None, "", n, _QK_K, dtype)+++_FUNCTIONS = {+ "Q4_0": dequantize_blocks_Q4_0,+ "Q4_1": dequantize_blocks_Q4_1,+ "Q5_0": dequantize_blocks_Q5_0,+ "Q5_1": dequantize_blocks_Q5_1,+ "Q8_0": dequantize_blocks_Q8_0,+ "Q2_K": dequantize_blocks_Q2_K,+ "Q3_K": dequantize_blocks_Q3_K,+ "Q4_K": dequantize_blocks_Q4_K,+ "Q5_K": dequantize_blocks_Q5_K,+ "Q6_K": dequantize_blocks_Q6_K,+ "IQ4_XS": dequantize_blocks_IQ4_XS,+}+EXACT_FUNCTIONS = tuple(_FUNCTIONS.values())+_EXACT = frozenset(EXACT_FUNCTIONS)+_TAG = "_unsloth_gguf_exact"+for _fn in EXACT_FUNCTIONS:+ # an attribute test traces inside a compiled block; set membership graph-breaks+ setattr(_fn, _TAG, True)+del _fn+_replaced: dict = {}+# "forward" (diffusers' forward_native) outlives uninstall: a later patch may fall back to _exact_forward+_STATE: dict = {}+++def _table() -> Any:+ table = _STATE.get("table")+ if table is None:+ try:+ from diffusers.quantizers.gguf import utils as gguf_utils+ except Exception: # noqa: BLE001+ return None+ table = _STATE["table"] = getattr(gguf_utils, "dequantize_functions", None)+ return table+++def _sizes() -> Any:+ sizes = _STATE.get("sizes")+ if sizes is None:+ from gguf import GGML_QUANT_SIZES+ sizes = _STATE["sizes"] = GGML_QUANT_SIZES+ return sizes+++def exact_function(quant_type: Any) -> Any:+ table = _table()+ fn = table.get(quant_type) if isinstance(table, dict) else None+ return fn if getattr(fn, _TAG, False) else None+++def row_aligned(raw: Any, quant_type: Any) -> bool:+ return raw.dim() >= 1 and raw.shape[-1] % _sizes()[quant_type][1] == 0+++def dequantize_cast(+ raw: Any,+ quant_type: Any,+ dtype: Any,+ blocks_fn: Any = None,+) -> Any:+ import torch++ block_size, type_size = _sizes()[quant_type]+ blocks = raw.view(torch.uint8).reshape(-1, type_size)+ if blocks_fn is None:+ blocks_fn = exact_function(quant_type)+ elif not torch.compiler.is_compiling():+ torch._dynamo.maybe_mark_dynamic(blocks, 0)+ return blocks_fn(blocks, block_size, type_size, dtype).reshape(*raw.shape[:-1], -1)+++def _stock_forward(linear_cls: Any) -> Any:+ """diffusers' own ``forward_native`` under any wrappers, by code file (``functools.wraps`` copies ``__module__``)."""+ import sys++ home = getattr(sys.modules.get(linear_cls.__module__), "__file__", None)+ seen = []+ stash = getattr(linear_cls, "_original_utils_GGUFLinear_forward_native", None)+ for fn in (linear_cls.__dict__.get("forward_native"), stash):+ while callable(fn) and fn not in seen:+ seen.append(fn)+ if (+ home is not None+ and getattr(getattr(fn, "__code__", None), "co_filename", None) == home+ ):+ return fn+ fn = getattr(fn, "__wrapped__", None)+ return None+++def set_compiled_dequant(on: bool) -> None:+ _STATE["compiled_dequant"] = bool(on)+++def _exact_forward(self: Any, inputs: Any) -> Any:+ weight = self.weight+ dtype = self.compute_dtype+ quant_type = getattr(weight, "quant_type", None)+ if (+ dtype is None+ or quant_type is None+ or weight.dim() != 2+ or _STATE.get("compiled_dequant")+ or exact_function(quant_type) is None+ or not row_aligned(weight, quant_type)+ ):+ return _STATE["forward"](self, inputs)+ import torch++ dense = dequantize_cast(weight.as_tensor(), quant_type, dtype)+ bias = self.bias.to(dtype) if self.bias is not None else None+ return torch.nn.functional.linear(inputs, dense, bias)+++def stock_forward() -> Any:+ return _STATE.get("forward")+++def install_exact_dequant(logger: Any = None) -> list:+ if not exact_enabled():+ return []+ try:+ import gguf+ from diffusers.quantizers.gguf import utils as gguf_utils+ except Exception: # noqa: BLE001 - no GGUF support in this diffusers+ return []+ table = getattr(gguf_utils, "dequantize_functions", None)+ if not isinstance(table, dict):+ return []+ _STATE["table"] = table+ _STATE["sizes"] = gguf.GGML_QUANT_SIZES+ replaced = []+ for name, fn in _FUNCTIONS.items():+ qtype = getattr(gguf.GGMLQuantizationType, name, None)+ if qtype is None or qtype not in table or table[qtype] is fn:+ continue+ _replaced.setdefault(qtype, table[qtype])+ table[qtype] = fn+ replaced.append(name)+ linear_cls = getattr(gguf_utils, "GGUFLinear", None)+ stock = _stock_forward(linear_cls) if linear_cls is not None else None+ if stock is not None:+ _STATE.setdefault("forward", stock)+ if linear_cls.__dict__.get("forward_native") is stock:+ linear_cls.forward_native = _exact_forward+ if replaced and logger is not None:+ logger.info("diffusion.gguf: llama.cpp-exact fp32 dequant for %s", ", ".join(replaced))+ return replaced+++def uninstall_exact_dequant() -> None:+ table = _STATE.get("table")+ if isinstance(table, dict):+ for qtype, original in _replaced.items():+ if table.get(qtype) in _EXACT:+ table[qtype] = original+ _replaced.clear()+ try:+ from diffusers.quantizers.gguf import utils as gguf_utils+ except Exception: # noqa: BLE001+ return+ linear_cls = getattr(gguf_utils, "GGUFLinear", None)+ if linear_cls is not None and linear_cls.__dict__.get("forward_native") is _exact_forward:+ linear_cls.forward_native = _STATE["forward"]diff --git a/studio/backend/core/inference/diffusion_gguf_pipeline.py b/studio/backend/core/inference/diffusion_gguf_pipeline.pyindex 7839975a955..fa760b6127b 100644--- a/studio/backend/core/inference/diffusion_gguf_pipeline.py+++ b/studio/backend/core/inference/diffusion_gguf_pipeline.py@@ -66,9 +66,11 @@ def load_whole_pipeline_gguf(import diffusers.loaders.single_file as single_file_modfrom diffusers.loaders.single_file_utils import load_single_file_checkpoint+ from .diffusion_gguf_exact import install_exact_dequantfrom .diffusion_gguf_lowbit import install_lowbit_iqinstall_lowbit_iq() # diffusers' reader refuses unknown types, so before the read+ install_exact_dequant()checkpoint = load_single_file_checkpoint(path,local_files_only = pipe_kwargs.get("local_files_only"),diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex a77724fee9f..9662f9abaac 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -9904,8 +9904,11 @@ class VideoBackend:"local_files_only": local_files_only,}if kind == "gguf":+ from .diffusion_gguf_exact import install_exact_dequantfrom .diffusion_gguf_lowbit import install_lowbit_iq+install_lowbit_iq(logger)+ install_exact_dequant(logger)sf_kwargs["quantization_config"] = diffusers.GGUFQuantizationConfig(compute_dtype = dtype)diff --git a/studio/backend/core/inference/video_ltx2.py b/studio/backend/core/inference/video_ltx2.pyindex 2c14cc8cb9f..d4b4c9d6c64 100644--- a/studio/backend/core/inference/video_ltx2.py+++ b/studio/backend/core/inference/video_ltx2.py@@ -1330,8 +1330,11 @@ def load_ltx23_transformer(# The load plan's commit for that hub-id config (absent for an unpinned load).kwargs.update(pinned_revision_kwargs(base_repo))if is_gguf:+ from .diffusion_gguf_exact import install_exact_dequantfrom .diffusion_gguf_lowbit import install_lowbit_iq+install_lowbit_iq()+ install_exact_dequant()kwargs["quantization_config"] = diffusers.GGUFQuantizationConfig(compute_dtype = torch_dtype)elif device is not None:# Else the meta load copies every tensor back to the host.@@ -1635,8 +1638,11 @@ def _assemble_ltx23_pipeline(state = _load_checkpoint_without_dit(checkpoint_path)if state is None:if is_gguf:+ from .diffusion_gguf_exact import install_exact_dequantfrom .diffusion_gguf_lowbit import install_lowbit_iq+install_lowbit_iq() # before the read: diffusers' reader refuses unknown types+ install_exact_dequant()state = load_single_file_checkpoint(str(checkpoint_path))groups = _split_checkpoint(state)del statediff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.pyindex 080f9e0bbe2..f38726d4a0b 100644--- a/studio/backend/core/inference/video_ltx25.py+++ b/studio/backend/core/inference/video_ltx25.py@@ -362,8 +362,11 @@ def load_ltx25_transformer(from .video_ltx2 import _ltx23_pre_convert, ltx23_is_dit_keyif is_gguf:+ from .diffusion_gguf_exact import install_exact_dequantfrom .diffusion_gguf_lowbit import install_lowbit_iq+install_lowbit_iq() # before the read: diffusers' reader refuses unknown types+ install_exact_dequant()state = load_single_file_checkpoint(str(checkpoint_path))dit_state = _ltx23_pre_convert({k: v for k, v in state.items() if ltx23_is_dit_key(k)})del state@@ -591,9 +594,11 @@ def _read_tensors(path: str, keep: Callable[[str], bool]) -> dict[str, Any]:if not str(path).lower().endswith(".safetensors"):from diffusers.loaders.single_file_utils import load_single_file_checkpoint+ from .diffusion_gguf_exact import install_exact_dequantfrom .diffusion_gguf_lowbit import install_lowbit_iqinstall_lowbit_iq()+ install_exact_dequant()return {k: v for k, v in load_single_file_checkpoint(str(path)).items() if keep(k)}from safetensors import safe_opendiff --git a/studio/backend/tests/test_diffusion_gguf_compile.py b/studio/backend/tests/test_diffusion_gguf_compile.pyindex 6ce187642fd..d84656eb22e 100644--- a/studio/backend/tests/test_diffusion_gguf_compile.py+++ b/studio/backend/tests/test_diffusion_gguf_compile.py@@ -18,9 +18,13 @@ from core.inference import diffusion_gguf_compile as gc # noqa: E402@pytest.fixture(autouse = True)def _clean():# Always start and end from a clean, unpatched state so tests do not leak the process-wide patch into each other.+ from core.inference import diffusion_gguf_exact as ex+gc.uninstall_all()+ ex.uninstall_exact_dequant()yieldgc.uninstall_all()+ ex.uninstall_exact_dequant()def test_compiled_dequant_install_uninstall_reversible():diff --git a/studio/backend/tests/test_diffusion_gguf_exact.py b/studio/backend/tests/test_diffusion_gguf_exact.pynew file mode 100644index 00000000000..b1983d0b90f--- /dev/null+++ b/studio/backend/tests/test_diffusion_gguf_exact.py@@ -0,0 +1,467 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""llama.cpp-exact GGUF dequant: fp32 bit-identical to ggml-quants.c (vendored digests of its output, no C compiler+needed) and to gguf-py, NaN / inf / subnormal / -0 scale bits included, one rounding to the compute dtype, eager and+compiled (no graph breaks, one dynamic graph per type), install / uninstall / kill switch, and the GGUFLinear+forward. Runs on CPU."""++from __future__ import annotations++import hashlib++import pytest++torch = pytest.importorskip("torch")+np = pytest.importorskip("numpy")+gguf = pytest.importorskip("gguf")+gguf_utils = pytest.importorskip("diffusers.quantizers.gguf.utils")++from core.inference import diffusion_gguf_compile as gc # noqa: E402+from core.inference import diffusion_gguf_exact as ex # noqa: E402++T = gguf.GGMLQuantizationType+TYPES = list(ex._FUNCTIONS)++# byte offsets of each block's fp16 fields (ggml-common.h block structs)+F16_OFFSETS = {+ "Q4_0": (0,), "Q4_1": (0, 2), "Q5_0": (0,), "Q5_1": (0, 2), "Q8_0": (0,),+ "Q2_K": (80, 82), "Q3_K": (108,), "Q4_K": (0, 2), "Q5_K": (0, 2), "Q6_K": (208,), "IQ4_XS": (0,),+} # fmt: skip+# +inf, -inf, qNaN, sNaN, -qNaN, -0, +0, smallest subnormal, largest subnormal, max, -max, smallest normal+SPECIAL = (+ 0x7C00,+ 0xFC00,+ 0x7E00,+ 0x7C01,+ 0xFE00,+ 0x8000,+ 0x0000,+ 0x0001,+ 0x03FF,+ 0x7BFF,+ 0xFBFF,+ 0x0400,+)+ORACLE_BLOCKS = {name: 37 for name in F16_OFFSETS} # odd: no even-count assumption+# ggml-org/llama.cpp ggml-quants.c dequantize_row_* (f477f79, -O0, no FMA) on oracle_blocks(name, 37): sha256 of the+# fp32 output, NaN canonicalised. Regenerate with scripts/ggml_parity/make_oracle.py against a compiled ggml-quants.c.+GGML_DIGESTS = {+ "Q4_0": "11f2f408ad3ecc40298fc34e7d36e010e5a7f3338b2e49c3d67d1c1ea801c95f",+ "Q4_1": "ea2dc4832791b6bb8c0a1800d75a153840f29b89064a6909109786ec0a6538b6",+ "Q5_0": "84de0e848a5d8f7e62b9eadc0d9ee4f254e5f426643723da9ade8ed8a1e3ede9",+ "Q5_1": "9cf75e735c74127787ffe0c1d5b02189c9aa5ac9339d3714693c7ed57e9e2e4b",+ "Q8_0": "f46e3b0ed5965df250f13728201a942c5f06de257a2a7ae96d20c7fc1545d729",+ "Q2_K": "d5437758841860ef8b258b2c7d4fc223bc8421b371b0e0c22f0da3b6a10e466f",+ "Q3_K": "7a2776ee811eaa09ed093f1ebc856869783bffc16c4999ee95cf9735194edbe4",+ "Q4_K": "5cae66ba58e1344b4c7a8ff706a5f8ce7e7d9bf346f6a3f20dc341e1e1f9d567",+ "Q5_K": "28591e1ba7c6aa0f852e8107ad503b0e6b4c2e3a7b9e4718c800505301a9c03c",+ "Q6_K": "94229b879c82509f6cdefa56bdd8852fefd787fe9946aa36f03997f1c57bb680",+ "IQ4_XS": "510c1a39bb8c42cba718de1238d99df8ef6abaa6d8a84a0a6ed89f32f1fa221d",+}+++def oracle_blocks(name: str, n: int):+ """Deterministic blocks: sha256 counter bytes; the fp16 fields cycle through SPECIAL for the first 24 blocks,+ then take normal values of either sign with exponents 2^-6 .. 2^-1."""+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]+ stream = b"".join(+ hashlib.sha256(f"{name}:{i}".encode()).digest() for i in range((n * sz) // 32 + 1)+ )+ raw = np.frombuffer(stream[: n * sz], dtype = np.uint8).reshape(n, sz).copy()+ for k, off in enumerate(F16_OFFSETS[name]):+ u = raw[:, off].astype(np.uint16) | (raw[:, off + 1].astype(np.uint16) << 8)+ normal = (u & 0x83FF) | ((9 + (u >> 10) % 6) << 10).astype(np.uint16)+ for b in range(n):+ bits = SPECIAL[(b + 5 * k) % len(SPECIAL)] if b < 2 * len(SPECIAL) else int(normal[b])+ raw[b, off], raw[b, off + 1] = bits & 0xFF, bits >> 8+ return raw+++def canonical_digest(values) -> str:+ bits = np.ascontiguousarray(values, dtype = np.float32).reshape(-1).view(np.uint32).copy()+ bits[np.isnan(np.asarray(values, dtype = np.float32).reshape(-1))] = 0x7FC00000+ return hashlib.sha256(bits.tobytes()).hexdigest()+++def _same(got, ref) -> bool:+ """Equal bits wherever not NaN, NaN at the same places (fp32 / bf16 / fp16)."""+ got = got.reshape(-1)+ ref = ref.reshape(-1)+ int_dtype = torch.int32 if got.dtype == torch.float32 else torch.int16+ nan = torch.isnan(ref)+ return bool(+ got.dtype == ref.dtype+ and torch.equal(torch.isnan(got), nan)+ and torch.equal(got[~nan].view(int_dtype), ref[~nan].view(int_dtype))+ )+++def _random_blocks(+ name: str,+ n: int,+ seed: int = 0,+ special: bool = True,+):+ rng = np.random.default_rng(seed)+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]+ raw = rng.integers(0, 256, size = (n, sz), dtype = np.uint8)+ for off in F16_OFFSETS[name]:+ v = (rng.standard_normal(n) * 0.02).astype(np.float16).view(np.uint16)+ if special:+ k = min(n // 4, 48)+ v[rng.choice(n, size = k, replace = False)] = rng.choice(SPECIAL, size = k)+ raw[:, off : off + 2] = v.view(np.uint8).reshape(-1, 2)+ return raw++[email protected](autouse = True)+def _clean(monkeypatch):+ from core.inference import diffusion_compile_config as cc++ monkeypatch.delenv(ex.EXACT_ENV, raising = False)+ cc._reset_for_tests()+ gc.uninstall_all()+ ex.uninstall_exact_dequant()+ torch._dynamo.reset()+ yield+ gc.uninstall_all()+ ex.uninstall_exact_dequant()+ cc._reset_for_tests()+++def test_digest_table_covers_every_type():+ assert set(GGML_DIGESTS) == set(TYPES) == set(F16_OFFSETS)++[email protected]("name", TYPES)+def test_fp32_matches_the_vendored_ggml_output(name):+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]+ raw = oracle_blocks(name, ORACLE_BLOCKS[name])+ out = ex._FUNCTIONS[name](torch.from_numpy(raw), bs, sz)+ assert out.dtype == torch.float32 and out.shape == (raw.shape[0], bs)+ assert canonical_digest(out.numpy()) == GGML_DIGESTS[name]+ assert np.isnan(out.numpy()).any() and np.isinf(out.numpy()).any()++[email protected]("name", TYPES)[email protected]("n", [1, 7, 513])+def test_fp32_is_bit_identical_to_gguf_py(name, n):+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]+ raw = _random_blocks(name, n, seed = n)+ with np.errstate(all = "ignore"):+ ref = torch.from_numpy(+ np.ascontiguousarray(gguf.quants.dequantize(raw, getattr(T, name)), dtype = np.float32)+ )+ assert _same(ex._FUNCTIONS[name](torch.from_numpy(raw), bs, sz), ref.reshape(n, bs))++[email protected]("name", TYPES)[email protected]("dtype", [torch.bfloat16, torch.float16])+def test_a_compute_dtype_is_one_rounding_of_the_fp32_result(name, dtype):+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]+ raw = torch.from_numpy(_random_blocks(name, 301, seed = 3))+ fp32 = ex._FUNCTIONS[name](raw, bs, sz)+ assert _same(ex._FUNCTIONS[name](raw, bs, sz, dtype), fp32.to(dtype))+++def test_diffusers_fp16_functions_are_not_exact():+ # if diffusers' own function were already exact, this module would be redundant+ bs, sz = gguf.GGML_QUANT_SIZES[T.Q4_K]+ raw = torch.from_numpy(_random_blocks("Q4_K", 512, seed = 5))+ stock = gguf_utils.dequantize_functions[T.Q4_K](raw, bs, sz)+ ours = ex.dequantize_blocks_Q4_K(raw, bs, sz)+ fin = torch.isfinite(ours) & torch.isfinite(stock.float())+ assert not torch.equal(stock.to(torch.bfloat16)[fin], ours.to(torch.bfloat16)[fin])+++def _gguf_py(raw, qtype):+ return torch.from_numpy(+ np.ascontiguousarray(gguf.quants.dequantize(raw.numpy(), qtype), np.float32)+ )++[email protected]("compiled", [False, True])+def test_q8_0_bf16_is_not_rounded_through_fp16(compiled):+ # d = 1371 / 4096, q = 3: fp16 d * q = 1 + 2**-8 (a bf16 tie, to 1); ggml's fp32 1 + 2**-8 + 2**-12 -> 1 + 2**-7+ raw = torch.zeros(1, 34, dtype = torch.uint8)+ raw[0, :2] = torch.tensor([np.float16(1371 / 4096)]).view(torch.uint8)+ raw[0, 2:] = 3+ fn = (+ torch.compile(ex.dequantize_blocks_Q8_0, fullgraph = True)+ if compiled+ else ex.dequantize_blocks_Q8_0+ )+ got = fn(raw, 32, 34, torch.bfloat16)+ assert torch.equal(got, torch.full_like(got, 1 + 2**-7))+ assert torch.equal(+ got.float(), _gguf_py(raw, T.Q8_0).to(torch.bfloat16).float().reshape(got.shape)+ )+ stock = gguf_utils.dequantize_functions[T.Q8_0]+ if stock is not ex.dequantize_blocks_Q8_0:+ assert float(stock(raw, 32, 34).to(torch.bfloat16)[0, 0]) == 1.0++[email protected]("compiled", [False, True])+def test_iq4_xs_large_scale_does_not_overflow_in_fp16(compiled):+ # d = 4096, ls - 32 = -32: ggml's fp32 dl = -131072 (fp16 would be -inf); * kvalue -127 is finite in fp32+ bs, sz = gguf.GGML_QUANT_SIZES[T.IQ4_XS]+ raw = torch.zeros(1, sz, dtype = torch.uint8)+ raw[0, :2] = torch.tensor([np.float16(4096)]).view(torch.uint8)+ fn = (+ torch.compile(ex.dequantize_blocks_IQ4_XS, fullgraph = True)+ if compiled+ else ex.dequantize_blocks_IQ4_XS+ )+ got = fn(raw, bs, sz).reshape(-1)+ assert torch.equal(got, _gguf_py(raw, T.IQ4_XS).reshape(-1))+ assert torch.equal(got, torch.full_like(got, 4096.0 * 32 * 127))++[email protected]("name", TYPES)[email protected]("rows", [1, 3, 5])+def test_dequantize_cast_keeps_rows_of_whole_blocks(name, rows):+ qtype = getattr(T, name)+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]+ raw = torch.from_numpy(_random_blocks(name, rows * 3, seed = rows)).reshape(rows, 3 * sz)+ ex.install_exact_dequant()+ dense = ex.dequantize_cast(raw, qtype, torch.bfloat16)+ assert dense.shape == (rows, 3 * bs) and dense.dtype == torch.bfloat16+ ref = gguf_utils.dequantize_gguf_tensor(gguf_utils.GGUFParameter(raw, quant_type = qtype)).to(+ torch.bfloat16+ )+ assert _same(dense, ref)+ assert ex.row_aligned(raw, qtype) and not ex.row_aligned(raw[:, 1:], qtype)+++def test_install_replaces_only_the_fp16_types_and_uninstall_restores_them():+ table = gguf_utils.dequantize_functions+ before = dict(table)+ supported = list(gguf_utils.SUPPORTED_GGUF_QUANT_TYPES)+ assert ex.install_exact_dequant() == TYPES+ assert ex.install_exact_dequant() == []+ for name in TYPES:+ assert table[getattr(T, name)] is ex._FUNCTIONS[name]+ assert ex.exact_function(getattr(T, name)) is ex._FUNCTIONS[name]+ for qtype in (T.IQ4_NL, T.BF16):+ assert table[qtype] is before[qtype] and ex.exact_function(qtype) is None+ assert gguf_utils.SUPPORTED_GGUF_QUANT_TYPES == supported+ assert gguf_utils.GGUFLinear.forward_native is ex._exact_forward+ ex.uninstall_exact_dequant()+ ex.uninstall_exact_dequant()+ assert table == before+ assert ex.exact_function(T.Q4_K) is None+ assert gguf_utils.GGUFLinear.forward_native is ex._STATE["forward"]+ assert ex._STATE["forward"].__module__ == gguf_utils.__name__++[email protected]("value", ["0", "false", "off", "no"])+def test_kill_switch_keeps_diffusers_functions(monkeypatch, value):+ monkeypatch.setenv(ex.EXACT_ENV, value)+ before = dict(gguf_utils.dequantize_functions)+ forward = gguf_utils.GGUFLinear.forward_native+ assert ex.install_exact_dequant() == []+ assert gguf_utils.dequantize_functions == before+ assert gguf_utils.GGUFLinear.forward_native is forward+++def test_uninstall_leaves_a_later_replacement_alone():+ ex.install_exact_dequant()+ other = lambda *a, **k: None # noqa: E731+ gguf_utils.dequantize_functions[T.Q8_0] = other+ try:+ ex.uninstall_exact_dequant()+ assert gguf_utils.dequantize_functions[T.Q8_0] is other+ finally:+ gguf_utils.dequantize_functions[T.Q8_0] = gguf_utils.dequantize_blocks_Q8_0++[email protected]("name", TYPES)+def test_compiled_has_no_graph_breaks_and_one_graph_for_every_weight(name):+ # as the patched GGUFLinear forward runs it: one compile per type, only the block count dynamic+ from torch._dynamo.testing import CompileCounter++ qtype = getattr(T, name)+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]+ counter = CompileCounter()+ fn = torch.compile(ex._FUNCTIONS[name], backend = counter, fullgraph = True)+ for rows, n in ((2, 8), (13, 13), (4, 64), (7, 301)):+ raw = torch.from_numpy(_random_blocks(name, rows * n, seed = n)).reshape(rows, n * sz)+ got = ex.dequantize_cast(raw, qtype, torch.bfloat16, fn)+ assert got.shape == (rows, n * bs)+ assert _same(+ got,+ ex._FUNCTIONS[name](raw.reshape(-1, sz), bs, sz).to(torch.bfloat16).reshape(rows, -1),+ )+ assert counter.frame_count == 1++[email protected]("name", ["Q8_0", "Q4_K", "Q6_K", "Q5_1", "Q3_K", "Q2_K"])+ "emulate", [False, True], ids = ["inductor_default", "emulate_precision_casts"]+)+def test_inductor_dequant_cast_is_bit_identical(name, emulate):+ bs, sz = gguf.GGML_QUANT_SIZES[getattr(T, name)]+ with torch._inductor.config.patch(emulate_precision_casts = emulate):+ fn = torch.compile(ex._FUNCTIONS[name])+ for n in (16, 37):+ raw = torch.from_numpy(_random_blocks(name, n, seed = n))+ fp32 = ex._FUNCTIONS[name](raw, bs, sz)+ assert _same(ex.dequantize_cast(raw, getattr(T, name), torch.float32, fn), fp32)+ assert _same(+ ex.dequantize_cast(raw, getattr(T, name), torch.bfloat16, fn),+ fp32.to(torch.bfloat16),+ )++[email protected]("name", ["Q2_K", "Q4_K", "Q5_K"])+def test_a_zero_code_under_a_negative_scale_and_zero_min_is_negative_zero(name):+ # ggml: d1 * 0 - (+0) = -0 when d1 < 0. A lowering of ``-m + d1 * q`` as ``(0 - m) + ...`` gives +0.+ qtype = getattr(T, name)+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]+ raw = np.zeros((4, sz), dtype = np.uint8)+ d_off, dmin_off = F16_OFFSETS[name]+ raw[:, d_off : d_off + 2] = np.array([0xBC00], np.uint16).view(np.uint8) # d = -1+ raw[:, dmin_off : dmin_off + 2] = np.array([0x3C00], np.uint16).view(np.uint8) # dmin = +1+ if name == "Q2_K":+ raw[:, :16] = 0x01 # scale 1, min 0+ else:+ raw[:, 4:8] = 0x01 # sc = 1, m = 0 for the first four groups+ with np.errstate(all = "ignore"):+ ref = torch.from_numpy(+ np.ascontiguousarray(gguf.quants.dequantize(raw, qtype), np.float32)+ ).reshape(4, bs)+ assert (torch.signbit(ref) & (ref == 0)).any()+ t = torch.from_numpy(raw)+ assert _same(ex._FUNCTIONS[name](t, bs, sz), ref)+ assert _same(+ ex.dequantize_cast(t, qtype, torch.float32, torch.compile(ex._FUNCTIONS[name])), ref+ )+++needs_fused = pytest.mark.skipif(+ not gc._torch_supports_fused_q8_0(torch.__version__),+ reason = "the patched forward needs torch >= 2.7",+)+++def _linear(qtype, rows, cols, dtype):+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]+ raw = torch.from_numpy(_random_blocks(qtype.name, rows * cols // bs, seed = 11, special = False))+ lin = gguf_utils.GGUFLinear(cols, rows, bias = False, compute_dtype = dtype)+ lin.weight = gguf_utils.GGUFParameter(raw.reshape(rows, cols // bs * sz), quant_type = qtype)+ return lin+++@needs_fused[email protected]("name", ["Q8_0", "Q4_K", "Q6_K"])[email protected]("dtype", [torch.bfloat16, torch.float16])+def test_patched_forward_uses_the_exact_fused_dequant(name, dtype, monkeypatch):+ qtype = getattr(T, name)+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]+ ex.install_exact_dequant()+ assert gc.install_compiled_dequant()+ calls = []+ real = ex.dequantize_cast+ monkeypatch.setattr(ex, "dequantize_cast", lambda *a, **k: calls.append(a[1]) or real(*a, **k))+ lin = _linear(qtype, 6, 512, dtype)+ raw = lin.weight.as_tensor()+ dense = ex._FUNCTIONS[name](raw.reshape(-1, sz), bs, sz).reshape(6, 512).to(dtype)+ x = torch.randn(3, 512).to(dtype)+ out = lin(x)+ assert calls == [qtype]+ assert torch.equal(out, torch.nn.functional.linear(x, dense))+++@needs_fused+def test_kill_switch_restores_the_stock_q8_0_forward(monkeypatch):+ monkeypatch.setenv(ex.EXACT_ENV, "0")+ assert ex.install_exact_dequant() == []+ assert gc.install_compiled_dequant()+ calls = []+ monkeypatch.setattr(ex, "dequantize_cast", lambda *a, **k: calls.append(1))+ lin = _linear(T.Q8_0, 4, 64, torch.bfloat16)+ lin(torch.ones(1, 64, dtype = torch.bfloat16))+ assert calls == []++[email protected]("name", ["Q8_0", "Q4_K", "Q6_K", "Q2_K"])[email protected]("dtype", [torch.bfloat16, torch.float16])+def test_eager_forward_dequantises_straight_to_the_compute_dtype(name, dtype, monkeypatch):+ qtype = getattr(T, name)+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]+ ex.install_exact_dequant()+ calls = []+ real = ex.dequantize_cast+ monkeypatch.setattr(ex, "dequantize_cast", lambda *a, **k: calls.append(a[2]) or real(*a, **k))+ lin = _linear(qtype, 6, 512, dtype)+ raw = lin.weight.as_tensor()+ x = torch.randn(3, 512).to(dtype)+ out = lin(x)+ assert calls == [dtype]+ dense = ex._FUNCTIONS[name](raw.reshape(-1, sz), bs, sz).reshape(6, 512).to(dtype)+ assert torch.equal(out, torch.nn.functional.linear(x, dense))+++def test_eager_forward_leaves_other_types_to_diffusers(monkeypatch):+ ex.install_exact_dequant()+ calls = []+ monkeypatch.setattr(ex, "dequantize_cast", lambda *a, **k: calls.append(1))+ raw = torch.randn(4, 64).to(torch.bfloat16).view(torch.uint8)+ lin = gguf_utils.GGUFLinear(64, 4, bias = False, compute_dtype = torch.bfloat16)+ lin.weight = gguf_utils.GGUFParameter(raw, quant_type = T.BF16)+ x = torch.randn(2, 64).to(torch.bfloat16)+ assert torch.equal(lin(x), torch.nn.functional.linear(x, raw.view(torch.bfloat16)))+ assert calls == []++[email protected]("name", ["Q8_0", "Q4_K", "Q6_K"])+def test_a_compiled_block_traces_the_eager_forward_without_graph_breaks(name):+ # the max tier compiles whole blocks around GGUFLinear: the forward and dequant must trace in one graph+ from torch._dynamo.testing import CompileCounter++ ex.install_exact_dequant()+ lin = _linear(getattr(T, name), 6, 512, torch.bfloat16)+ x = torch.randn(3, 512).to(torch.bfloat16)+ counter = CompileCounter()+ out = torch.compile(lin, backend = counter, fullgraph = True)(x)+ assert counter.frame_count == 1+ assert torch.equal(out, lin(x))+++def test_install_order_with_the_compiled_forward_does_not_matter():+ stock = gguf_utils.GGUFLinear.forward_native+ assert ex._stock_forward(gguf_utils.GGUFLinear) is stock+ # compiled forward first: its wrapper stays, diffusers' own forward is what the exact path falls back to+ assert gc.install_compiled_dequant()+ wrapper = gguf_utils.GGUFLinear.forward_native+ assert ex.install_exact_dequant() == TYPES+ assert gguf_utils.GGUFLinear.forward_native is wrapper+ assert ex.stock_forward() is stock+ gc.uninstall_all()+ assert gguf_utils.GGUFLinear.forward_native is stock+ # exact first, compiled on top, then out-of-order teardown: still back to diffusers' forward+ ex.install_exact_dequant()+ assert gguf_utils.GGUFLinear.forward_native is ex._exact_forward+ assert gc.install_compiled_dequant()+ ex.uninstall_exact_dequant()+ gc.uninstall_all()+ ex.uninstall_exact_dequant()+ assert gguf_utils.GGUFLinear.forward_native is stock+++@needs_fused+def test_fused_cast_switch_off_runs_the_compiled_stock_dequant_once_rounded(monkeypatch):+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", "0")+ ex.install_exact_dequant()+ assert gc.install_compiled_dequant()+ calls = []+ monkeypatch.setattr(ex, "dequantize_cast", lambda *a, **k: calls.append(1))+ qtype = T.Q4_K+ bs, sz = gguf.GGML_QUANT_SIZES[qtype]+ lin = _linear(qtype, 6, 512, torch.bfloat16)+ raw = lin.weight.as_tensor()+ x = torch.randn(3, 512).to(torch.bfloat16)+ dense = ex._FUNCTIONS["Q4_K"](raw.reshape(-1, sz), bs, sz).reshape(6, 512).to(torch.bfloat16)+ assert torch.equal(lin(x), torch.nn.functional.linear(x, dense))+ assert calls == []diff --git a/studio/backend/tests/test_diffusion_gguf_fused_cast.py b/studio/backend/tests/test_diffusion_gguf_fused_cast.pyindex 11be2c8567e..604616c8185 100644--- a/studio/backend/tests/test_diffusion_gguf_fused_cast.py+++ b/studio/backend/tests/test_diffusion_gguf_fused_cast.py@@ -40,13 +40,18 @@ FP16_TYPES = [T.Q4_K, T.Q5_K, T.Q6_K, T.Q3_K, T.Q2_K, T.Q4_0, T.Q4_1, T.Q5_0, T.def _clean(monkeypatch):from core.inference import diffusion_compile_config as cc+ from core.inference import diffusion_gguf_exact as ex+monkeypatch.delenv("UNSLOTH_DIFFUSION_GGUF_FUSED_CAST", raising = False)cc._reset_for_tests()gc.uninstall_all()+ # diffusers' fp16 functions: the path UNSLOTH_DIFFUSION_GGUF_EXACT=0 keeps+ ex.uninstall_exact_dequant()torch._dynamo.reset()with torch._inductor.config.patch(emulate_precision_casts = False):yieldgc.uninstall_all()+ ex.uninstall_exact_dequant()cc._reset_for_tests()diff --git a/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.pyindex 18f3c072048..c4432de6774 100644--- a/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py+++ b/studio/backend/tests/test_diffusion_gguf_fused_q8_0.py@@ -29,12 +29,17 @@ def _clean():# re-applies; start every test from torch's defaults so results do not depend on test order.from core.inference import diffusion_compile_config as cc+ from core.inference import diffusion_gguf_exact as ex+cc._reset_for_tests()gc.uninstall_all()+ # diffusers' fp16 functions: the path UNSLOTH_DIFFUSION_GGUF_EXACT=0 keeps+ ex.uninstall_exact_dequant()torch._dynamo.reset()with torch._inductor.config.patch(emulate_precision_casts = False):yieldgc.uninstall_all()+ ex.uninstall_exact_dequant()cc._reset_for_tests()diff --git a/studio/backend/tests/test_diffusion_gguf_lowbit.py b/studio/backend/tests/test_diffusion_gguf_lowbit.pyindex 5df3dd99aad..ede15a7dd5e 100644--- a/studio/backend/tests/test_diffusion_gguf_lowbit.py+++ b/studio/backend/tests/test_diffusion_gguf_lowbit.py@@ -20,10 +20,15 @@ TYPES = ["IQ1_S", "IQ1_M", "IQ2_XXS", "IQ2_XS", "IQ2_S", "IQ3_XXS", "IQ3_S"]@pytest.fixture(autouse = True)def _clean(monkeypatch):+ from core.inference import diffusion_gguf_exact as ex+monkeypatch.delenv(lb.LOWBIT_IQ_ENV, raising = False)lb.uninstall_lowbit_iq()+ ex.uninstall_exact_dequant()yieldlb.uninstall_lowbit_iq()+ # the loaders under test also install the exact dequant+ ex.uninstall_exact_dequant()def _blocks(