Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Compiled GGUF dequant accelerator: install, reversibility and kill-switch, run on CPU."""
from __future__ import annotations
import logging
import pytest
torch = pytest.importorskip("torch")
gguf_utils = pytest.importorskip("diffusers.quantizers.gguf.utils")
from core.inference import diffusion_gguf_compile as gc # noqa: E402
@pytest.fixture(autouse = True)
def _clean():
# Always start and end from a clean, unpatched state so tests do not leak the process-wide patch into each other.
from core.inference import diffusion_gguf_exact as ex
gc.uninstall_all()
ex.uninstall_exact_dequant()
yield
gc.uninstall_all()
ex.uninstall_exact_dequant()
def test_compiled_dequant_install_uninstall_reversible():
orig = gguf_utils.dequantize_gguf_tensor
assert gc.is_compiled_dequant_installed() is False
assert gc.install_compiled_dequant() is True
assert gc.is_compiled_dequant_installed() is True
assert gguf_utils.dequantize_gguf_tensor is not orig
patched = gguf_utils.dequantize_gguf_tensor
assert gc.install_compiled_dequant() is True
assert gguf_utils.dequantize_gguf_tensor is patched
gc.uninstall_compiled_dequant()
assert gc.is_compiled_dequant_installed() is False
assert gguf_utils.dequantize_gguf_tensor is orig
gc.uninstall_compiled_dequant()
assert gguf_utils.dequantize_gguf_tensor is orig
def test_compiled_dequant_kill_switch(monkeypatch):
monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_COMPILE_DEQUANT", "0")
orig = gguf_utils.dequantize_gguf_tensor
assert gc.install_compiled_dequant() is False
assert gc.is_compiled_dequant_installed() is False
assert gguf_utils.dequantize_gguf_tensor is orig
def test_compiled_dequant_on_by_default(monkeypatch):
monkeypatch.delenv("UNSLOTH_DIFFUSION_GGUF_COMPILE_DEQUANT", raising = False)
assert gc.install_compiled_dequant() is True
assert gc.is_compiled_dequant_installed() is True
def test_uninstall_all(monkeypatch):
orig = gguf_utils.dequantize_gguf_tensor
gc.install_compiled_dequant()
assert gc.is_installed() is True
gc.uninstall_all()
assert gc.is_installed() is False
assert gguf_utils.dequantize_gguf_tensor is orig
def _inductor_error():
from torch._inductor.exc import InductorError
# What #9897 raised: Triton's driver build (clang-cl on hip_utils.c) failed inside inductor's codegen.
import subprocess
return InductorError(subprocess.CalledProcessError(1, ["clang-cl.exe", "hip_utils.c"]), None)
def test_compile_failure_falls_back_to_eager():
calls: list = []
def compiled(t):
calls.append("compiled")
raise _inductor_error()
def eager(t):
calls.append("eager")
return t * 2
guarded = gc._guard(compiled, eager, logging.getLogger("t"))
x = torch.ones(3)
assert torch.equal(guarded(x), x * 2)
assert gc._compile_error.startswith("InductorError")
assert torch.equal(guarded(x), x * 2)
assert calls == ["compiled", "eager", "eager"]
def test_the_lazy_compile_sees_the_recorded_compile_config():
import threading
from core.inference import diffusion_compile_config as cc
cc._reset_for_tests()
seen = {}
def compiled(t):
seen["memory_planning"] = torch._inductor.config.memory_planning
return t
try:
assert cc.set_knob("torch._inductor.config", "memory_planning", False)
guarded = gc._guard(compiled, lambda t: t, None)
def render():
# A render context still carrying the training config (torch 2.12+ keeps config writes per context).
torch._inductor.config.memory_planning = True
guarded(torch.ones(1))
worker = threading.Thread(target = render)
worker.start()
worker.join()
assert seen["memory_planning"] is False
finally:
cc._reset_for_tests()
@pytest.mark.parametrize(
"exc",
[RuntimeError("bad weight"), torch.OutOfMemoryError("CUDA out of memory")],
ids = ["kernel_error", "oom"],
)
def test_non_compile_errors_still_raise(exc):
def compiled(t):
raise exc
guarded = gc._guard(compiled, lambda t: t, None)
with pytest.raises(type(exc)):
guarded(torch.ones(1))
def test_installed_dequant_survives_a_failing_compile(monkeypatch):
def failing_compile(fn, **kwargs):
def compiled(*a, **k):
raise _inductor_error()
return compiled
monkeypatch.setattr(torch, "compile", failing_compile)
orig = gguf_utils.dequantize_gguf_tensor
assert gc.install_compiled_dequant() is True
plain = torch.arange(4, dtype = torch.float32)
# A non-GGUF tensor goes through the stock function unchanged, so this compares to the original exactly.
assert torch.equal(gguf_utils.dequantize_gguf_tensor(plain), orig(plain))
assert gc.compile_error().startswith("InductorError")
gc.uninstall_all()
assert gc.compile_error() is None
class TestGgufTrimmedDimsAreRestored:
"""GGUF drops size-1 axes, so trimmed dims must be restored or diffusers refuses the shape check."""
@staticmethod
def _model():
import torch
class _M(torch.nn.Module):
def __init__(self):
super().__init__()
self.cap_pad_token = torch.nn.Parameter(torch.zeros((1, 8)))
self.proj = torch.nn.Linear(8, 4)
return _M()
def test_a_trimmed_leading_axis_is_put_back(self):
import torch
from core.inference.diffusion import _restore_gguf_trimmed_dims
model = self._model()
sd = {"cap_pad_token": torch.arange(8, dtype = torch.float32)}
out = _restore_gguf_trimmed_dims(model, sd)
assert tuple(out["cap_pad_token"].shape) == (1, 8)
assert torch.equal(out["cap_pad_token"].flatten(), torch.arange(8, dtype = torch.float32))
def test_a_tensor_of_the_wrong_size_is_left_alone(self):
"""The guard must not rescue a genuinely wrong tensor: element count has to match."""
import torch
from core.inference.diffusion import _restore_gguf_trimmed_dims
model = self._model()
sd = {"cap_pad_token": torch.zeros(7)}
out = _restore_gguf_trimmed_dims(model, sd)
assert tuple(out["cap_pad_token"].shape) == (7,)
def test_a_transposed_tensor_is_left_alone(self):
"""Same element count, but the expected shape is not the stored shape with 1s in front, so
this is a real mismatch and must still reach diffusers' error rather than be reshaped."""
import torch
from core.inference.diffusion import _restore_gguf_trimmed_dims
model = self._model()
sd = {"proj.weight": torch.zeros(8, 4)} # model wants (4, 8)
out = _restore_gguf_trimmed_dims(model, sd)
assert tuple(out["proj.weight"].shape) == (8, 4)
def test_the_shim_reaches_the_name_single_file_model_actually_calls(self):
"""``single_file_model`` imports the loader at module level, so it holds its own reference.
Patching only the defining module leaves the real call site bound to the original, which is
exactly the way the first version of this fix silently did nothing on a live load."""
import torch
from diffusers.loaders import single_file_model as sfm
from diffusers.models import model_loading_utils as mlu
from core.inference.diffusion import _install_gguf_dim_restore
originals = (mlu.load_model_dict_into_meta, sfm.load_model_dict_into_meta)
seen: dict = {}
def _fake(model, state_dict, *args, **kwargs):
seen["shape"] = tuple(state_dict["cap_pad_token"].shape)
return []
mlu.load_model_dict_into_meta = _fake
sfm.load_model_dict_into_meta = _fake
try:
_install_gguf_dim_restore(logging.getLogger("t"))
first = sfm.load_model_dict_into_meta
_install_gguf_dim_restore(logging.getLogger("t"))
assert sfm.load_model_dict_into_meta is first, "shim wrapped itself twice"
# Call through the module the loader really uses, not the one that defines it.
sfm.load_model_dict_into_meta(self._model(), {"cap_pad_token": torch.zeros(8)})
assert seen["shape"] == (1, 8)
finally:
mlu.load_model_dict_into_meta, sfm.load_model_dict_into_meta = originals