Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Load Qwen-Image-2.1 GGUFs through diffusers on GPU hosts (#11604)
Daniel Han5c535375fa4c
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex caeda3ee7d0..5eda9f1a0e0 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -1073,6 +1073,98 @@ def _install_gguf_prefix_strip(transformer_cls: Any, logger: Any) -> None:logger.warning("diffusion.gguf: prefix-strip shim not installed: %s", exc)+def _qwen_image_21_checkpoint_to_diffusers(checkpoint = None, **kwargs):+ """sd.cpp / ComfyUI Qwen-Image-2.1 single file -> diffusers ``QwenImage21Transformer2DModel``.++ The two layouts differ in exactly one way once the ``model.diffusion_model.`` container prefix+ is gone: each block stores its MLP input projection FUSED, ``img_mlp.gate_up`` [24576, 4096],+ where diffusers keeps ``img_mlp.gate_layer`` and ``img_mlp.proj`` [12288, 4096] apart. That is+ 265 tensors against 297, 32 blocks times one fused pair, and every other name and shape+ matches. The order is gate first, then proj, checked against the upstream bf16 weights rather+ than read off a reference implementation: both halves are bit-identical to their diffusers+ tensors, and the swapped assignment is off by up to 1.08.++ Splitting a GGUF tensor by rows is safe at any quant type: GGML packs blocks along the last+ (input) dimension, so a row is always a whole number of blocks and ``GGUFParameter`` keeps its+ quant type through the slice.++ One-dimensional tensors are dequantised here, because diffusers only dequantises inside the+ ``nn.Linear`` layers its GGUF quantizer swaps in. Everything 1-D in this model is a norm weight,+ read straight off ``self.weight``, so a norm stored in any wrapped type reaches the forward as+ raw bytes: the public Q4_K_M keeps ``txt_in.text_norm`` in BF16 and dies on the first step with+ "size of tensor a (4096) must match ... b (8192)", and a build without the norm=f32 pin puts all+ 64 attention norms in Q8_0 the same way. They are a few KB in total, so there is nothing to save+ by keeping them packed.+ """+ prefix = "model.diffusion_model."+ fused = "img_mlp.gate_up.weight"+ converted = {}+ for key, value in (checkpoint or {}).items():+ if key.startswith(prefix):+ key = key[len(prefix) :]+ quant_shape = getattr(value, "quant_shape", None)+ if quant_shape is not None and len(quant_shape) == 1:+ from diffusers.quantizers.gguf.utils import dequantize_gguf_tensor+ value = dequantize_gguf_tensor(value)+ if key.endswith("." + fused):+ rows = int(value.shape[0])+ if rows % 2:+ raise ValueError(+ f"{key}: a fused gate_up with an odd row count ({rows}) cannot be split into "+ "gate_layer and proj; this is not a Qwen-Image-2.1 checkpoint"+ )+ stem = key[: -len("gate_up.weight")]+ converted[stem + "gate_layer.weight"] = value[: rows // 2]+ converted[stem + "proj.weight"] = value[rows // 2 :]+ continue+ converted[key] = value+ return converted+++# Transformer classes the installed diffusers cannot load from a single file, with the converter+# that lets it. Without an entry ``from_single_file`` refuses before reading a byte ("FromOriginalModelMixin is+# currently only compatible with ..."), and on a CUDA, ROCm or XPU host that is every GGUF load of the+# family, because a GPU backend routes GGUFs to diffusers. The native engine is no way round it+# there: the pinned sd.cpp prebuilt carries no Windows GPU build at all.+_UNREGISTERED_SINGLE_FILE_CLASSES: dict = {+ "QwenImage21Transformer2DModel": _qwen_image_21_checkpoint_to_diffusers,+}+++def _register_unregistered_single_file_classes(logger: Any = None) -> tuple:+ """Add single-file support for the classes above that the installed diffusers lacks.++ Never overwrites: a class diffusers registers itself wins, since upstream's mapping is+ authoritative and ours only fills the gap until it lands. A class this diffusers does not ship+ is skipped rather than registered, and that is load-bearing, not tidiness: diffusers resolves+ EVERY registry entry with ``getattr(diffusers, name)`` on each ``from_single_file`` call, so one+ entry naming a missing class would break single-file loads for every other family too.+ Idempotent and best-effort; returns the names it added.+ """+ added: list = []+ try:+ import diffusers+ from diffusers.loaders import single_file_model as sfm+ except Exception: # noqa: BLE001 - no diffusers, nothing to register into+ return ()+ for name, mapping_fn in _UNREGISTERED_SINGLE_FILE_CLASSES.items():+ if name in sfm.SINGLE_FILE_LOADABLE_CLASSES:+ continue+ try:+ if getattr(diffusers, name, None) is None:+ continue+ except Exception: # noqa: BLE001 - a class whose module fails to import is not loadable+ continue+ sfm.SINGLE_FILE_LOADABLE_CLASSES[name] = {+ "checkpoint_mapping_fn": mapping_fn,+ "default_subfolder": "transformer",+ }+ added.append(name)+ if added and logger is not None:+ logger.info("diffusion.single_file: registered %s", ", ".join(added))+ return tuple(added)++def _restore_gguf_trimmed_dims(model: Any, state_dict: Any) -> Any:"""Put back the leading size-1 dimensions GGUF drops when it stores a tensor.@@ -5077,6 +5169,9 @@ class DiffusionBackend:# pipeline assembly below was already guarded; this call was not."local_files_only": local_files_only,}+ # Before the prefix shim below, which wraps whatever entry it finds and+ # finds nothing for a class that is not registered yet.+ _register_unregistered_single_file_classes(logger)if kind == "gguf":# Dequantise the GGUF transformer on-device at the compute dtype.sf_kwargs["quantization_config"] = diffusers.GGUFQuantizationConfig(diff --git a/studio/backend/tests/test_diffusion_single_file_registry.py b/studio/backend/tests/test_diffusion_single_file_registry.pynew file mode 100644index 00000000000..cc86704596c--- /dev/null+++ b/studio/backend/tests/test_diffusion_single_file_registry.py@@ -0,0 +1,168 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Single-file support Unsloth registers for transformer classes diffusers has not.++Without it, every GGUF load of Qwen-Image-2.1 on a CUDA, ROCm or XPU host died before reading a+byte with "FromOriginalModelMixin is currently only compatible with ...", because a GPU backend+routes GGUFs to diffusers and diffusers had no single-file entry for the class.+"""++from __future__ import annotations++import pytest+import torch++from core.inference import diffusion as studio++CLASS = "QwenImage21Transformer2DModel"+PREFIX = "model.diffusion_model."+++def _sfm():+ try:+ import diffusers+ from diffusers.loaders import single_file_model as sfm+ except Exception as exc: # noqa: BLE001 - an optional integration failing is an environment fact+ pytest.skip(f"diffusers is not importable here: {type(exc).__name__}")+ return diffusers, sfm+++def test_the_fused_mlp_splits_gate_first_and_the_prefix_goes():+ """The one layout difference between an sd.cpp 2.1 file and diffusers, and its order.++ Gate first, proj second is what the upstream bf16 weights say: both halves of the fused tensor+ are bit-identical to their diffusers tensors that way round, and off by up to 1.08 the other.+ """+ gate = torch.arange(0, 6 * 4, dtype = torch.float32).reshape(6, 4)+ proj = -torch.arange(0, 6 * 4, dtype = torch.float32).reshape(6, 4)+ checkpoint = {+ PREFIX + "transformer_blocks.3.img_mlp.gate_up.weight": torch.cat([gate, proj]),+ PREFIX + "transformer_blocks.3.img_mlp.out.weight": torch.ones(4, 6),+ "img_in.weight": torch.zeros(2, 2),+ }+ out = studio._qwen_image_21_checkpoint_to_diffusers(checkpoint = checkpoint, config = {})++ assert set(out) == {+ "transformer_blocks.3.img_mlp.gate_layer.weight",+ "transformer_blocks.3.img_mlp.proj.weight",+ "transformer_blocks.3.img_mlp.out.weight",+ "img_in.weight",+ }+ assert torch.equal(out["transformer_blocks.3.img_mlp.gate_layer.weight"], gate)+ assert torch.equal(out["transformer_blocks.3.img_mlp.proj.weight"], proj)+++def test_an_odd_fused_tensor_is_refused_rather_than_split_wrong():+ with pytest.raises(ValueError, match = "odd row count"):+ studio._qwen_image_21_checkpoint_to_diffusers(+ checkpoint = {"transformer_blocks.0.img_mlp.gate_up.weight": torch.zeros(5, 4)}+ )+++def test_a_row_split_keeps_a_gguf_tensor_quantised():+ """GGML packs blocks along the input dimension, so a row slice is whole blocks and has to stay+ a ``GGUFParameter`` of the same quant type, with the logical shape halved."""+ try:+ import gguf+ from diffusers.quantizers.gguf.utils import GGUFParameter+ except Exception as exc: # noqa: BLE001+ pytest.skip(f"gguf support is not importable here: {type(exc).__name__}")+ qtype = gguf.GGMLQuantizationType.Q4_K+ block_size, type_size = gguf.GGML_QUANT_SIZES[qtype]+ cols = 2 * block_size+ raw = torch.randint(0, 255, (8, cols // block_size * type_size), dtype = torch.uint8)+ fused = GGUFParameter(raw, quant_type = qtype)++ out = studio._qwen_image_21_checkpoint_to_diffusers(+ checkpoint = {"transformer_blocks.0.img_mlp.gate_up.weight": fused}+ )+ for name, rows in (("gate_layer", slice(0, 4)), ("proj", slice(4, 8))):+ half = out[f"transformer_blocks.0.img_mlp.{name}.weight"]+ assert isinstance(half, GGUFParameter), name+ assert half.quant_type == qtype, name+ assert tuple(half.quant_shape) == (4, cols), name+ assert torch.equal(half.as_tensor(), raw[rows]), name++[email protected]("qtype_name", ["BF16", "Q8_0"])+def test_a_packed_norm_weight_comes_out_as_real_values(qtype_name):+ """diffusers dequantises only inside the Linear layers it swaps, and a norm reads self.weight+ directly. Measured: the public Q4_K_M keeps txt_in.text_norm in BF16 and the first step died on+ "size of tensor a (4096) must match ... b (8192)" -- the raw bytes, not the values."""+ try:+ import gguf+ from diffusers.quantizers.gguf.utils import GGUFParameter, dequantize_gguf_tensor+ except Exception as exc: # noqa: BLE001+ pytest.skip(f"gguf support is not importable here: {type(exc).__name__}")+ qtype = getattr(gguf.GGMLQuantizationType, qtype_name)+ block_size, type_size = gguf.GGML_QUANT_SIZES[qtype]+ n = 4 * block_size+ raw = torch.randint(0, 255, (n // block_size * type_size,), dtype = torch.uint8)+ packed = GGUFParameter(raw, quant_type = qtype)+ expected = dequantize_gguf_tensor(GGUFParameter(raw.clone(), quant_type = qtype))++ out = studio._qwen_image_21_checkpoint_to_diffusers(+ checkpoint = {PREFIX + "txt_in.text_norm.weight": packed}+ )["txt_in.text_norm.weight"]+ assert not hasattr(out, "quant_type")+ assert tuple(out.shape) == (n,)+ assert torch.equal(out, expected)+++def test_registration_fills_the_gap_and_the_real_call_stops_refusing(monkeypatch):+ diffusers, sfm = _sfm()+ if getattr(diffusers, CLASS, None) is None:+ pytest.skip(f"this diffusers does not ship {CLASS}")+ monkeypatch.setattr(sfm, "SINGLE_FILE_LOADABLE_CLASSES", dict(sfm.SINGLE_FILE_LOADABLE_CLASSES))+ sfm.SINGLE_FILE_LOADABLE_CLASSES.pop(CLASS, None)++ assert studio._register_unregistered_single_file_classes() == (CLASS,)+ entry = sfm.SINGLE_FILE_LOADABLE_CLASSES[CLASS]+ assert entry["checkpoint_mapping_fn"] is studio._qwen_image_21_checkpoint_to_diffusers+ assert entry["default_subfolder"] == "transformer"+ # And diffusers' own lookup now resolves the class, which is the check that raised.+ assert sfm._get_single_file_loadable_mapping_class(getattr(diffusers, CLASS)) == CLASS+ # Idempotent.+ assert studio._register_unregistered_single_file_classes() == ()+++def test_registration_never_overwrites_diffusers_own_entry(monkeypatch):+ diffusers, sfm = _sfm()+ upstream = {"checkpoint_mapping_fn": lambda checkpoint, **kwargs: checkpoint}+ monkeypatch.setattr(+ sfm, "SINGLE_FILE_LOADABLE_CLASSES", {**sfm.SINGLE_FILE_LOADABLE_CLASSES, CLASS: upstream}+ )+ monkeypatch.setattr(diffusers, CLASS, type(CLASS, (), {}), raising = False)++ assert studio._register_unregistered_single_file_classes() == ()+ assert sfm.SINGLE_FILE_LOADABLE_CLASSES[CLASS] is upstream+++def test_a_class_this_diffusers_lacks_is_not_registered(monkeypatch):+ """Not tidiness: diffusers resolves EVERY registry entry with getattr(diffusers, name) on each+ from_single_file call, so one entry for a missing class would break every other family's+ single-file load along with it."""+ diffusers, sfm = _sfm()+ monkeypatch.setattr(sfm, "SINGLE_FILE_LOADABLE_CLASSES", dict(sfm.SINGLE_FILE_LOADABLE_CLASSES))+ assert getattr(diffusers, "NotAClassThisDiffusersShips2DModel", None) is None+ monkeypatch.setattr(+ studio,+ "_UNREGISTERED_SINGLE_FILE_CLASSES",+ {"NotAClassThisDiffusersShips2DModel": studio._qwen_image_21_checkpoint_to_diffusers},+ )++ assert studio._register_unregistered_single_file_classes() == ()+ assert "NotAClassThisDiffusersShips2DModel" not in sfm.SINGLE_FILE_LOADABLE_CLASSES+++def test_the_transformer_only_single_file_branch_registers_before_loading():+ """The call site, not just the helper: registration has to precede both the prefix shim (which+ wraps whatever entry it finds) and the from_single_file it exists for."""+ import inspect++ source = inspect.getsource(studio)+ register = source.index("_register_unregistered_single_file_classes(logger)")+ shim = source.index("_install_gguf_prefix_strip(transformer_cls, logger)")+ load = source.index("transformer = transformer_cls.from_single_file(")+ assert register < shim < load