Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: load the cached int8 checkpoint for a 5-bit-or-below GGUF pick that has to offload (Qwen-Image-2.1 about 2x faster at 12 / 8 GB) (#12455)

Daniel Hanf2f09ec2e4cd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 3d8ca806021..3921b44d057 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -5556,6 +5556,10 @@ class DiffusionBackend:
# repo's transformer/ shards, since the fallback would pull them HERE, inside the load lock, after
# eviction, where unload cannot preempt it and progress already reported 100%.
dense_fallback_allowed = bool(_transformer_prefetched)
+ # Set when an offloading GGUF pick loads the pre-quantised checkpoint instead (diffusion_gguf_route).
+ gguf_offload_placement: Optional[Any] = None
+ gguf_offload_scheme: Optional[str] = None
+ gguf_offload_swap = False
# A GGUF pick with the scheme left to us: the hosted pre-quant is free only while already cached,
# since fetching it means a SECOND multi-GB denoiser and the GGUF never runs. An explicit scheme, a
# LoRA bake and every non-GGUF kind keep today's behaviour. An all-zero-weight list is not a bake.
@@ -5747,6 +5751,23 @@ class DiffusionBackend:
transformer_quant_decline = _torchao_offload_decline(
"the quantised build still needs", replanned
)
+ # Held until the resident retry below had its turn.
+ gguf_offload_placement, gguf_offload_reason = (
+ self._gguf_offload_prequant_placement(
+ replanned,
+ candidate,
+ fam = fam,
+ target = target,
+ auto = transformer_quant_auto,
+ gguf_filename = gguf_filename,
+ base = base,
+ prequant_path = transformer_prequant_path,
+ )
+ )
+ # The retry below may rebind ``candidate``; the placement is this one's.
+ gguf_offload_scheme = getattr(candidate, "scheme", None)
+ if gguf_offload_reason is not None:
+ transformer_quant_decline = gguf_offload_reason
if plan_keeps_transformer_resident(replanned):
quant_plan = replanned
# The GGUF plan declined resident; a prequant-sized replan says nothing about the
@@ -5799,6 +5820,18 @@ class DiffusionBackend:
quant_plan = retry_plan
if candidate.prequant:
dense_fallback_allowed = False
+ if quant_plan is None and gguf_offload_placement is not None:
+ logger.info(
+ "diffusion.transformer_quant: the GGUF pick offloads ('%s'); loading the "
+ "pre-quantised %s checkpoint under the same placement instead",
+ gguf_offload_placement.offload_policy,
+ gguf_offload_scheme,
+ )
+ quant_plan = gguf_offload_placement
+ gguf_offload_swap = True
+ transformer_quant_decline = None
+ # A dense bf16 fallback would land the whole denoiser on a card it does not fit.
+ dense_fallback_allowed = False
else:
# This materialises the dense bf16 transformer, so re-check the fit rather than OOMing after
# eviction (skipped for a prequant).
@@ -5992,15 +6025,42 @@ class DiffusionBackend:
),
text_encoder_quant = text_encoder_quant,
)
+ sized_placement, sized_reason = (
+ (None, None)
+ if plan_keeps_transformer_resident(sized_plan)
+ else self._gguf_offload_prequant_placement(
+ sized_plan,
+ sized_candidate,
+ fam = fam,
+ target = target,
+ auto = transformer_quant_auto,
+ gguf_filename = gguf_filename,
+ base = base,
+ prequant_path = transformer_prequant_path,
+ )
+ )
if plan_keeps_transformer_resident(sized_plan):
quant_plan = sized_plan
if sized_plan.offload_policy != OFFLOAD_NONE:
dense_fallback_allowed = False
+ elif sized_placement is not None:
+ logger.info(
+ "diffusion.transformer_quant: the pre-quantised %s checkpoint "
+ "offloads ('%s'); loading it under that placement instead of the GGUF",
+ sized_candidate.scheme,
+ sized_placement.offload_policy,
+ )
+ quant_plan = sized_placement
+ gguf_offload_swap = True
+ dense_fallback_allowed = False
else:
dense_declined = True
dense_fallback_allowed = False
- transformer_quant_decline = _torchao_offload_decline(
- "the pre-quantised transformer needs", sized_plan
+ transformer_quant_decline = (
+ sized_reason
+ or _torchao_offload_decline(
+ "the pre-quantised transformer needs", sized_plan
+ )
)
logger.info(
"diffusion.transformer_quant_declined: %s",
@@ -6132,6 +6192,14 @@ class DiffusionBackend:
fetch_base = fetch_base,
local_files_only = local_files_only,
_load_token = _load_token,
+ seed_plan = quant_plan if quant_plan is not None else plan,
+ )
+ transformer_quant_artifact = self._gguf_route_artifact(
+ pipe,
+ fam,
+ transformer_quant_engaged,
+ base = base,
+ prequant_path = transformer_prequant_path,
)
except Exception as exc: # noqa: BLE001 - fall back to the GGUF build
self._raise_if_load_cancelled(_load_token)
@@ -7189,6 +7257,15 @@ class DiffusionBackend:
if transformer_quant_artifact is not None:
# Beside ``source``, never in it: the frontend branches on "auto"/"explicit".
resolved["transformer_quant"]["artifact"] = transformer_quant_artifact
+ if gguf_offload_swap and transformer_quant_engaged is not None:
+ from .diffusion_gguf_route import gguf_offload_swap_reason
+ resolved["transformer_quant"]["reason"] = gguf_offload_swap_reason(
+ transformer_quant_engaged,
+ transformer_quant_artifact,
+ plan,
+ gguf_filename,
+ )
+ resolved["transformer_quant"]["replaced"] = f"gguf:{gguf_filename}"
# Opt-in (UNSLOTH_DIFFUSION_BG_COMPILE=1): off by default so the same seed twice repeats.
bg_module = (
@@ -7301,6 +7378,66 @@ class DiffusionBackend:
)
return self.status()
+ @staticmethod
+ def _gguf_route_artifact(
+ pipe: Any,
+ fam: Any,
+ scheme: Optional[str],
+ *,
+ base: Optional[str],
+ prequant_path: Optional[str],
+ ) -> Optional[str]:
+ """``prequant:<repo>/<file>`` when the build loaded a pre-quantized checkpoint, else None."""
+ try:
+ transformer = getattr(pipe, "transformer", None)
+ if scheme is None or getattr(transformer, "_unsloth_prequant_path", None) is None:
+ return None
+ from .diffusion_denoiser_prequant import prequant_artifact_label
+
+ return prequant_artifact_label(
+ resolve_prequant_source(fam, scheme, path_override = prequant_path, base_repo = base),
+ transformer,
+ )
+ except Exception: # noqa: BLE001 - status only
+ return None
+
+ @staticmethod
+ def _gguf_offload_prequant_placement(
+ replanned: Any,
+ candidate: Any,
+ *,
+ fam: Any,
+ target: Any,
+ auto: bool,
+ gguf_filename: Optional[str],
+ base: Optional[str],
+ prequant_path: Optional[str],
+ ) -> tuple[Optional[Any], Optional[str]]:
+ """``diffusion_gguf_route.gguf_offload_prequant_placement`` with the cache state resolved."""
+ from .diffusion_gguf_route import gguf_offload_prequant_placement
+
+ scheme = getattr(candidate, "scheme", None)
+ if candidate is None or not scheme:
+ return None, None
+ try:
+ prequant_cached = (
+ _uncached_prequant_repo(
+ fam, target, scheme, base_repo = base, prequant_path = prequant_path
+ )
+ is None
+ )
+ except Exception: # noqa: BLE001 - unknown cache state: treat as uncached
+ prequant_cached = False
+ return gguf_offload_prequant_placement(
+ replanned,
+ candidate,
+ scheme = scheme,
+ auto = auto,
+ gguf_filename = gguf_filename,
+ prequant_cached = prequant_cached,
+ torchao_offload_plan = torchao_offload_plan,
+ )
+
def _load_dense_quant_pipeline(
self,
transformer_cls: Any,
@@ -7322,9 +7459,13 @@ class DiffusionBackend:
fetch_base: Optional[str] = None,
local_files_only: bool = False,
_load_token: Optional[int] = None,
+ seed_plan: Any = None,
) -> tuple[Any, str]:
"""Build the opt-in fast pipeline and return ``(pipe, engaged_scheme)``.
+ ``seed_plan``: the placement the build runs under; an offloading one seeds a pre-quantized checkpoint on the
+ host (``prequant_seed_device``).
+
Two ways to get the quantized transformer, in order: (1) Pre-quantized, when a checkpoint is
configured for the chosen scheme (an explicit ``prequant_path`` or the family's hosted
repo), loaded onto the meta device and assigned in, so the dense bf16 never lands on the GPU
@@ -7370,6 +7511,12 @@ class DiffusionBackend:
)
check_cancelled()
if source is not None:
+ # A GPU seed under offload OOMed 8 GB Qwen-Image-2.1 in the streaming hooks.
+ seed_device = (
+ prequant_seed_device(seed_plan, device, scheme)
+ if seed_plan is not None
+ else device
+ )
transformer = load_prequantized_transformer(
transformer_cls,
fetch_base,
@@ -7391,6 +7538,7 @@ class DiffusionBackend:
# re-downloaded under the import-time constant.
cache_dir = hub_cache_dir(),
logger = logger,
+ placement_device = None if seed_device == device else seed_device,
)
check_cancelled()
if transformer is not None:
@@ -7403,7 +7551,7 @@ class DiffusionBackend:
transformer,
dtype,
hf_token,
- device,
+ seed_device,
base_local_dir,
fam = fam,
te_quant_mode = text_encoder_quant,
diff --git a/studio/backend/core/inference/diffusion_gguf_route.py b/studio/backend/core/inference/diffusion_gguf_route.py
new file mode 100644
index 00000000000..90b9d9e52bf
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_gguf_route.py
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Let a GGUF pick whose memory plan offloads load the cached hosted int8 / fp8 checkpoint under that offload.
+
+Only a pre-quantised checkpoint, never a dense bf16 build; ``auto`` takes it only when cached and only for GGUFs less
+accurate than it. Device eligibility (CUDA bf16) is the caller's.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Optional
+
+# "0" restores the resident-only swap.
+GGUF_OFFLOAD_PREQUANT_ENV = "UNSLOTH_DIFFUSION_GGUF_OFFLOAD_PREQUANT"
+
+# Widest GGUF auto replaces (Qwen-Image-2.1 LPIPS vs bf16: Q5_K_M 0.077, int8 0.059, Q6_K 0.042, fp8 0.10-0.11).
+_MAX_REPLACED_GGUF_BITS = {"int8": 5}
+_DEFAULT_MAX_REPLACED_GGUF_BITS = 4
+# Tokens led by their bit width; any other (BF16, F16, F32, unknown) is kept.
+_GGUF_BITS_RE = re.compile(r"^(?:MXFP|IQ|P?TQ|P?Q)([0-9]+)")
+
+
+def gguf_offload_prequant_enabled() -> bool:
+ return str(os.environ.get(GGUF_OFFLOAD_PREQUANT_ENV, "")).strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def gguf_quant_token(gguf_filename: Optional[str]) -> Optional[str]:
+ if not gguf_filename:
+ return None
+ try:
+ from hub.utils.gguf import extract_quant_token
+ token = extract_quant_token(str(gguf_filename))
+ except Exception: # noqa: BLE001 - an unparseable name is treated as unknown precision
+ token = None
+ return str(token).upper() if token else None
+
+
+def gguf_outranks_scheme(gguf_filename: Optional[str], scheme: Optional[str]) -> bool:
+ """Whether the GGUF is at least as accurate as ``scheme``'s checkpoint; an unknown quant counts as outranking."""
+ token = gguf_quant_token(gguf_filename)
+ if token is None:
+ return True
+ match = _GGUF_BITS_RE.match(token[3:] if token.startswith("UD-") else token)
+ if match is None:
+ return True
+ return int(match.group(1)) > _MAX_REPLACED_GGUF_BITS.get(
+ str(scheme), _DEFAULT_MAX_REPLACED_GGUF_BITS
+ )
+
+
+def gguf_offload_prequant_placement(
+ replanned: Any,
+ candidate: Any,
+ *,
+ scheme: Optional[str],
+ auto: bool,
+ gguf_filename: Optional[str],
+ prequant_cached: bool,
+ torchao_offload_plan: Any,
+) -> tuple[Optional[Any], Optional[str]]:
+ """``(placement, None)`` to swap, ``(None, reason)`` to keep the GGUF by this rule, ``(None, None)`` when it
+ does not apply."""
+ if not gguf_offload_prequant_enabled():
+ return None, None
+ if candidate is None or not bool(getattr(candidate, "prequant", False)) or not scheme:
+ return None, None
+ if auto and not prequant_cached:
+ return None, (
+ f"the plan offloads the denoiser and the hosted {scheme} checkpoint is not cached; an auto quant never "
+ "downloads a second transformer for a GGUF pick"
+ )
+ if auto and gguf_outranks_scheme(gguf_filename, scheme):
+ return None, (
+ f"the plan offloads the denoiser, and the picked GGUF ({gguf_quant_token(gguf_filename) or 'unknown quant'}) "
+ f"is at least as accurate as the hosted {scheme} checkpoint, so auto keeps the GGUF"
+ )
+ try:
+ placement = torchao_offload_plan(replanned, scheme)
+ except Exception: # noqa: BLE001 - an unanswerable placement keeps the GGUF
+ placement = None
+ return placement, None
+
+
+def gguf_offload_swap_reason(
+ scheme: str, artifact: Optional[str], placement: Any, gguf_filename: Optional[str]
+) -> str:
+ what = (
+ artifact.split(":", 1)[1]
+ if artifact and ":" in artifact
+ else (artifact or f"the hosted {scheme} checkpoint")
+ )
+ token = gguf_quant_token(gguf_filename) or "GGUF"
+ return (
+ f"the {token} GGUF pick was replaced by {what}: the memory plan offloads "
+ f"('{getattr(placement, 'offload_policy', 'offload')}'), where the {scheme} checkpoint runs faster than "
+ "dequantising the GGUF every step. Set Precision to Off to run the GGUF itself"
+ )
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 49f56a40a6a..79783323cb9 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -4662,6 +4662,11 @@ class DiffusionResolvedControl(BaseModel):
"weights being quantised in memory. Declared here or pydantic drops it and no API client "
"ever sees the provenance. Null on every other control and on a runtime quantise.",
)
+ replaced: Optional[str] = Field(
+ None,
+ description = 'The picked checkpoint that did NOT run, as "gguf:<file>", when a GGUF pick whose memory plan '
+ "offloads loaded the hosted pre-quantized checkpoint (``artifact``) instead. Null otherwise.",
+ )
class DiffusionDownloadPlanEntry(BaseModel):
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index f60f1eb8556..2d41b06e38e 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -6796,6 +6796,7 @@ def test_diffusion_status_response_carries_resolved():
"status": "applied",
"reason": "blackwell",
"artifact": None,
+ "replaced": None,
}
}
# Absent by default (nothing resolved / native engine).
@@ -6818,7 +6819,7 @@ def test_diffusion_status_response_carries_requested_precision():
}
resp = DiffusionStatusResponse(loaded = True, resolved = rec)
assert resp.model_dump()["resolved"] == {
- "transformer_quant": {**rec["transformer_quant"], "artifact": None}
+ "transformer_quant": {**rec["transformer_quant"], "artifact": None, "replaced": None}
}
@@ -14194,3 +14195,228 @@ def test_candidate_overrides_price_the_encoder_the_load_opens(monkeypatch):
]
== 0
)
+
+
+_Q4 = "m-Q4_K_M.gguf"
+
+
+def _gguf_offload_swap_setup(
+ monkeypatch,
+ tmp_path,
+ *,
+ scheme = "int8",
+ policy = "model",
+):
+ """A CUDA GGUF pick whose plan and every quantised re-plan offload, with a cached hosted checkpoint."""
+ import dataclasses
+
+ from core.inference import diffusion as dmod
+
+ (tmp_path / _Q4).write_bytes(b"x")
+ (tmp_path / "m-Q8_0.gguf").write_bytes(b"x")
+ backend = DiffusionBackend()
+ _force_cuda_target(backend, monkeypatch)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GGUF_OFFLOAD_PREQUANT", raising = False)
+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(
+ dmod, "select_transformer_quant_scheme", lambda target, mode, family = None, **_kw: scheme
+ )
+ monkeypatch.setattr(dmod, "_uncached_prequant_repo", lambda *a, **k: None)
+ monkeypatch.setattr(
+ DiffusionBackend, "_auto_prequant_retry_scheme", staticmethod(lambda *a, **k: None)
+ )
+ monkeypatch.setattr(
+ dmod,
+ "resolve_dense_quant_candidate",
+ lambda **kw: types.SimpleNamespace(
+ scheme = scheme, transient_transformer_mib = 7_000, companions_mib = 8_000, prequant = True
+ ),
+ )
+ source = types.SimpleNamespace(
+ kind = "repo", location = "unsloth/Z-FP8", filename = "Z-INT8.safetensors"
+ )
+ monkeypatch.setattr(dmod, "resolve_prequant_source", lambda *a, **k: source)
+ monkeypatch.setattr(dmod, "torchao_offload_plan", lambda plan, s, **k: plan)
+ orig_plan = DiffusionBackend._plan_memory
+
+ def spy_plan(self, *a, **k):
+ return dataclasses.replace(orig_plan(self, *a, **k), offload_policy = policy)
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", spy_plan)
+ calls: list = []
+
+ def fake_dense_load(self, *a, **k):
+ calls.append(k)
+ pipe = _FakePipe()
+ pipe.transformer = types.SimpleNamespace(_unsloth_prequant_path = "/hub/Z-INT8.safetensors")
+ return pipe, scheme
+
+ monkeypatch.setattr(DiffusionBackend, "_load_dense_quant_pipeline", fake_dense_load)
+ return backend, calls
+
+
+def _load_gguf(
+ backend,
+ tmp_path,
+ name = _Q4,
+ **kwargs,
+):
+ return backend.load_pipeline(
+ str(tmp_path), gguf_filename = name, family_override = "z-image", **kwargs
+ )
+
+
+def test_offloading_gguf_pick_loads_the_cached_int8_checkpoint(fake_runtime, tmp_path, monkeypatch):
+ backend, calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)
+ status = _load_gguf(backend, tmp_path)
+ assert len(calls) == 1
+ assert calls[0]["allow_dense_fallback"] is False
+ assert calls[0]["seed_plan"].offload_policy == "model"
+ assert status["transformer_quant"] == "int8"
+ assert status["offload_policy"] == "model"
+ tq = status["resolved"]["transformer_quant"]
+ assert tq["value"] == "int8" and tq["source"] == "auto"
+ assert tq["artifact"] == "prequant:unsloth/Z-FP8/Z-INT8.safetensors"
+ assert tq["replaced"] == f"gguf:{_Q4}"
+ assert "Q4_K_M GGUF pick was replaced by unsloth/Z-FP8/Z-INT8.safetensors" in tq["reason"]
+
+
+def test_gguf_offload_swap_kill_switch_keeps_the_gguf(fake_runtime, tmp_path, monkeypatch):
+ backend, calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_OFFLOAD_PREQUANT", "0")
+ status = _load_gguf(backend, tmp_path)
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert _FakeTransformer.last["path"].endswith(_Q4)
+ assert "replaced" not in status["resolved"]["transformer_quant"]
+
+
+def test_gguf_offload_swap_keeps_a_q8_pick_on_auto(fake_runtime, tmp_path, monkeypatch):
+ backend, calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)
+ status = _load_gguf(backend, tmp_path, name = "m-Q8_0.gguf")
+ assert calls == []
+ assert status["transformer_quant"] is None
+ reason = status["resolved"]["transformer_quant"]["reason"]
+ assert "Q8_0" in reason and "at least as accurate" in reason
+
+
[email protected]("request_", ["off", "none"])
+def test_gguf_offload_swap_never_overrides_precision_off(
+ fake_runtime, tmp_path, monkeypatch, request_
+):
+ backend, calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)
+ status = _load_gguf(backend, tmp_path, transformer_quant = request_)
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert _FakeTransformer.last["path"].endswith(_Q4)
+
+
+def test_gguf_offload_swap_falls_back_to_the_gguf_when_the_build_fails(
+ fake_runtime, tmp_path, monkeypatch
+):
+ backend, _calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)
+
+ def boom(self, *a, **k):
+ raise RuntimeError("checkpoint unreadable")
+
+ monkeypatch.setattr(DiffusionBackend, "_load_dense_quant_pipeline", boom)
+ status = _load_gguf(backend, tmp_path)
+ assert status["transformer_quant"] is None
+ assert _FakeTransformer.last["path"].endswith(_Q4)
+ assert "replaced" not in status["resolved"]["transformer_quant"]
+
+
+def test_gguf_offload_swap_on_mps_keeps_the_gguf(fake_runtime, tmp_path, monkeypatch):
+ from core.inference import diffusion as dmod
+
+ backend, calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)
+ torch = sys.modules["torch"]
+ monkeypatch.setattr(
+ backend, "_target_for_ordinal", lambda fam, ordinal = None: _mps_target(torch)
+ )
+ # The real device gate, not the test's CUDA override.
+ from core.inference.diffusion_transformer_quant import dense_transformer_supported
+
+ monkeypatch.setattr(dmod, "dense_transformer_supported", dense_transformer_supported)
+ status = _load_gguf(backend, tmp_path)
+ assert calls == []
+ assert status["transformer_quant"] is None
+
+
+ "policy, stream, env, placed_on_host",
+ [
+ ("streaming", True, None, True),
+ ("group", True, None, True),
+ ("model", True, None, True),
+ ("none", True, None, False),
+ ("group", False, None, False), # denoiser stays resident: seed where it runs
+ ("streaming", True, "0", False), # UNSLOTH_DIFFUSION_PREQUANT_SEED_ON_HOST=0
+ ],
+)
+def test_gguf_route_prequant_seed_lands_on_the_host_when_the_plan_offloads(
+ fake_runtime, monkeypatch, policy, stream, env, placed_on_host
+):
+ from core.inference import diffusion as dmod
+
+ if env is None:
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PREQUANT_SEED_ON_HOST", raising = False)
+ else:
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PREQUANT_SEED_ON_HOST", env)
+ monkeypatch.setattr(dmod, "_planned_quant_scheme", lambda *a, **k: "int8")
+ monkeypatch.setattr(
+ dmod,
+ "resolve_prequant_source",
+ lambda *a, **k: types.SimpleNamespace(kind = "repo", location = "o/r", filename = "f"),
+ )
+ loaded: dict = {}
+
+ def fake_load(*a, **k):
+ loaded.update(k)
+ return object()
+
+ monkeypatch.setattr(dmod, "load_prequantized_transformer", fake_load)
+ assembled: list = []
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_assemble_pipe",
+ staticmethod(
+ lambda pcls, base, tr, dtype, tok, device, *a, **k: assembled.append(device)
+ or _FakePipe()
+ ),
+ )
+ _pipe, scheme = DiffusionBackend()._load_dense_quant_pipeline(
+ object,
+ object,
+ "base/repo",
+ "cuda:0",
+ "bf16",
+ None,
+ types.SimpleNamespace(device = "cuda", dtype = "bf16"),
+ "int8",
+ fam = types.SimpleNamespace(name = "qwen-image-2.1"),
+ seed_plan = types.SimpleNamespace(offload_policy = policy, stream_transformer = stream),
+ )
+ assert scheme == "int8"
+ assert loaded["placement_device"] == ("cpu" if placed_on_host else None)
+ assert assembled == ["cpu" if placed_on_host else "cuda:0"]
+
+
+def test_diffusion_status_response_keeps_the_gguf_a_swap_replaced():
+ # Pydantic drops undeclared keys.
+ from models.inference import DiffusionStatusResponse
+
+ rec = {
+ "transformer_quant": {
+ "value": "int8",
+ "source": "auto",
+ "reason": "the Q4_K_M GGUF pick was replaced",
+ "artifact": "prequant:o/r/f.safetensors",
+ "replaced": "gguf:m-Q4_K_M.gguf",
+ }
+ }
+ dumped = DiffusionStatusResponse(loaded = True, resolved = rec).model_dump()["resolved"][
+ "transformer_quant"
+ ]
+ assert dumped["replaced"] == "gguf:m-Q4_K_M.gguf"
+ assert dumped["artifact"] == "prequant:o/r/f.safetensors"
diff --git a/studio/backend/tests/test_diffusion_gguf_offload_route.py b/studio/backend/tests/test_diffusion_gguf_offload_route.py
new file mode 100644
index 00000000000..d6be197ca68
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_gguf_offload_route.py
@@ -0,0 +1,179 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A GGUF pick whose memory plan offloads loads the cached hosted int8 / fp8 checkpoint under the same placement."""
+
+from __future__ import annotations
+
+import types
+from dataclasses import replace
+
+import pytest
+
+from core.inference import diffusion_gguf_route as route
+from core.inference import diffusion_memory as mem
+from core.inference.diffusion_memory import (
+ OFFLOAD_GROUP,
+ OFFLOAD_NONE,
+ OFFLOAD_SEQUENTIAL,
+ OFFLOAD_STREAMING,
+ DeviceMemory,
+ plan_diffusion_memory,
+)
+
+
[email protected](autouse = True)
+def _roomy_host(monkeypatch):
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 10**7)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: (0, 40))
+ monkeypatch.delenv(route.GGUF_OFFLOAD_PREQUANT_ENV, raising = False)
+
+
+def _plan(policy, *, stream_transformer = True):
+ device = DeviceMemory("cuda", "cuda:0", "discrete_vram", 11_000, 12_288)
+ plan = plan_diffusion_memory(
+ target = types.SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = device,
+ model_dense_mib = 15_000,
+ runtime_headroom_mib = 2_000,
+ companion_dense_mib = 8_000,
+ text_encoder_dense_mib = 7_800,
+ )
+ return replace(plan, offload_policy = policy, stream_transformer = stream_transformer)
+
+
+def _ao(version = (0, 18)):
+ return lambda plan, scheme: mem.torchao_offload_plan(plan, scheme, torchao_version = version)
+
+
+INT8 = types.SimpleNamespace(scheme = "int8", prequant = True, transient_transformer_mib = 7_000)
+DENSE = types.SimpleNamespace(scheme = "int8", prequant = False, transient_transformer_mib = 20_000)
+
+
+def _decide(
+ plan,
+ candidate = INT8,
+ *,
+ auto = True,
+ gguf = "qwen-image-2.1-Q4_K_M.gguf",
+ cached = True,
+ version = (0, 18),
+):
+ return route.gguf_offload_prequant_placement(
+ plan,
+ candidate,
+ scheme = getattr(candidate, "scheme", None),
+ auto = auto,
+ gguf_filename = gguf,
+ prequant_cached = cached,
+ torchao_offload_plan = _ao(version),
+ )
+
+
[email protected]("policy", [OFFLOAD_STREAMING, OFFLOAD_GROUP])
+def test_offloading_q4_pick_takes_the_cached_int8_checkpoint(policy):
+ placement, reason = _decide(_plan(policy))
+ assert reason is None
+ assert placement is not None and placement.offload_policy == policy
+
+
+def test_sequential_offload_keeps_the_gguf():
+ # torchao never measured under sequential offload: the caller's decline stands.
+ assert _decide(_plan(OFFLOAD_SEQUENTIAL)) == (None, None)
+
+
+def test_torchao_too_old_to_stream_int8_keeps_the_gguf():
+ assert _decide(_plan(OFFLOAD_STREAMING), version = (0, 16)) == (None, None)
+
+
[email protected]("value", ["0", "off", "false", "no"])
+def test_kill_switch_restores_the_resident_only_rule(monkeypatch, value):
+ monkeypatch.setenv(route.GGUF_OFFLOAD_PREQUANT_ENV, value)
+ assert _decide(_plan(OFFLOAD_STREAMING)) == (None, None)
+
+
+def test_never_builds_dense_under_offload():
+ assert _decide(_plan(OFFLOAD_STREAMING), DENSE) == (None, None)
+
+
+def test_auto_never_downloads_a_second_denoiser():
+ placement, reason = _decide(_plan(OFFLOAD_STREAMING), cached = False)
+ assert placement is None and "not cached" in reason
+
+
+def test_explicit_scheme_may_fetch_its_checkpoint():
+ placement, reason = _decide(_plan(OFFLOAD_STREAMING), cached = False, auto = False)
+ assert reason is None and placement is not None
+
+
+ "gguf, outranks",
+ [
+ ("qwen-image-2.1-Q2_K.gguf", False),
+ ("qwen-image-2.1-Q3_K_M.gguf", False),
+ ("qwen-image-2.1-Q4_K_M.gguf", False),
+ ("qwen-image-2.1-Q5_K_S.gguf", False),
+ ("qwen-image-2.1-Q5_K_M.gguf", False),
+ ("model-UD-Q4_K_XL.gguf", False),
+ ("model-IQ4_XS.gguf", False),
+ ("model-MXFP4.gguf", False),
+ ("qwen-image-2.1-Q6_K.gguf", True),
+ ("qwen-image-2.1-Q6_K_XL.gguf", True),
+ ("model-UD-Q6_K_XL.gguf", True),
+ ("qwen-image-2.1-Q8_0.gguf", True),
+ ("qwen-image-2.1-F16.gguf", True),
+ ("qwen-image-2.1-BF16.gguf", True),
+ ("model.gguf", True),
+ ("model-MXFP8.gguf", True),
+ ("model-IQ6_XS.gguf", True),
+ ("model-PQ8_0.gguf", True),
+ ("model-TQ1_0.gguf", False),
+ ("model-Q5_0.gguf", False),
+ (None, True),
+ ],
+)
+def test_accuracy_gate_by_quant(gguf, outranks):
+ assert route.gguf_outranks_scheme(gguf, "int8") is outranks
+
+
+ "gguf, outranks",
+ [
+ ("x-Q4_K_M.gguf", False),
+ ("x-Q5_K_M.gguf", True),
+ ("x-Q6_K.gguf", True),
+ ("x-MXFP8.gguf", True),
+ ],
+)
+def test_fp8_only_replaces_4_bit_and_narrower(gguf, outranks):
+ assert route.gguf_outranks_scheme(gguf, "fp8") is outranks
+
+
+def test_auto_keeps_a_q8_pick_explicit_int8_still_swaps():
+ placement, reason = _decide(_plan(OFFLOAD_STREAMING), gguf = "qwen-image-2.1-Q8_0.gguf")
+ assert placement is None and "Q8_0" in reason and "at least as accurate" in reason
+ placement, reason = _decide(
+ _plan(OFFLOAD_STREAMING), gguf = "qwen-image-2.1-Q8_0.gguf", auto = False
+ )
+ assert placement is not None and reason is None
+
+
+def test_swap_reason_names_what_ran_and_how_to_keep_the_gguf():
+ text = route.gguf_offload_swap_reason(
+ "int8",
+ "prequant:unsloth/Qwen-Image-2.1-FP8/Qwen-Image-2.1-INT8.safetensors",
+ types.SimpleNamespace(offload_policy = OFFLOAD_STREAMING),
+ "qwen-image-2.1-Q4_K_M.gguf",
+ )
+ assert (
+ "Q4_K_M GGUF pick was replaced by unsloth/Qwen-Image-2.1-FP8/Qwen-Image-2.1-INT8.safetensors"
+ in text
+ )
+ assert "'streaming'" in text and "Precision to Off" in text
+
+
+def test_resident_plans_are_not_this_rules_business():
+ placement, _ = _decide(_plan(OFFLOAD_NONE))
+ assert placement.offload_policy == OFFLOAD_NONE
diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py
index 59688f7f167..48f527f3e26 100644
--- a/studio/backend/tests/test_diffusion_routes.py
+++ b/studio/backend/tests/test_diffusion_routes.py
@@ -1471,6 +1471,7 @@ def test_status_passes_through_resolved(client, monkeypatch):
assert body["resolved"]["transformer_quant"] == {
**resolved["transformer_quant"],
"artifact": None,
+ "replaced": None,
}
# Entries from an older backend (no requested/status) still parse, defaulted to "applied".
assert body["resolved"]["speed_mode"]["requested"] is None
diff --git a/studio/backend/tests/test_video_routes.py b/studio/backend/tests/test_video_routes.py
index fe9fc83eb4e..e056979c180 100644
--- a/studio/backend/tests/test_video_routes.py
+++ b/studio/backend/tests/test_video_routes.py
@@ -888,14 +888,16 @@ def test_status_passthrough(client, monkeypatch):
)
body = client.get("/api/inference/video/status").json()
assert body["loaded"] is True and body["family"] == "ltx-2"
- # ``artifact`` is additive on the response model: a record naming no hosted checkpoint is null.
+ # ``artifact`` / ``replaced`` are additive on the response model: a record naming neither is null.
assert body["resolved"]["transformer_quant"] == {
**resolved["transformer_quant"],
"artifact": None,
+ "replaced": None,
}
assert body["resolved"]["text_encoder_quant"] == {
**resolved["text_encoder_quant"],
"artifact": None,
+ "replaced": None,
}
# Entries from an older backend (no requested/status) still parse, defaulted to "applied".
assert body["resolved"]["speed_mode"]["requested"] is None
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts
index 3cbf95c4b8a..cd98f9b499c 100644
--- a/studio/frontend/src/features/images/api.ts
+++ b/studio/frontend/src/features/images/api.ts
@@ -23,6 +23,8 @@ export interface DiffusionResolvedControl {
reason: string;
// "prequant:<repo>/<file>" when a hosted checkpoint was seeded; absent on a runtime quantise.
artifact?: string | null;
+ // "gguf:<file>" the `artifact` replaced.
+ replaced?: string | null;
}
export interface DiffusionStatus {