Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: load the cached int8 checkpoint for a 5-bit-or-below GGUF pick that has to offload (Qwen-Image-2.1 about 2x faster at 12 / 8 GB) (#12455)
Daniel Hanf2f09ec2e4cd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 3d8ca806021..3921b44d057 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -5556,6 +5556,10 @@ class DiffusionBackend:# repo's transformer/ shards, since the fallback would pull them HERE, inside the load lock, after# eviction, where unload cannot preempt it and progress already reported 100%.dense_fallback_allowed = bool(_transformer_prefetched)+ # Set when an offloading GGUF pick loads the pre-quantised checkpoint instead (diffusion_gguf_route).+ gguf_offload_placement: Optional[Any] = None+ gguf_offload_scheme: Optional[str] = None+ gguf_offload_swap = False# A GGUF pick with the scheme left to us: the hosted pre-quant is free only while already cached,# since fetching it means a SECOND multi-GB denoiser and the GGUF never runs. An explicit scheme, a# LoRA bake and every non-GGUF kind keep today's behaviour. An all-zero-weight list is not a bake.@@ -5747,6 +5751,23 @@ class DiffusionBackend:transformer_quant_decline = _torchao_offload_decline("the quantised build still needs", replanned)+ # Held until the resident retry below had its turn.+ gguf_offload_placement, gguf_offload_reason = (+ self._gguf_offload_prequant_placement(+ replanned,+ candidate,+ fam = fam,+ target = target,+ auto = transformer_quant_auto,+ gguf_filename = gguf_filename,+ base = base,+ prequant_path = transformer_prequant_path,+ )+ )+ # The retry below may rebind ``candidate``; the placement is this one's.+ gguf_offload_scheme = getattr(candidate, "scheme", None)+ if gguf_offload_reason is not None:+ transformer_quant_decline = gguf_offload_reasonif plan_keeps_transformer_resident(replanned):quant_plan = replanned# The GGUF plan declined resident; a prequant-sized replan says nothing about the@@ -5799,6 +5820,18 @@ class DiffusionBackend:quant_plan = retry_planif candidate.prequant:dense_fallback_allowed = False+ if quant_plan is None and gguf_offload_placement is not None:+ logger.info(+ "diffusion.transformer_quant: the GGUF pick offloads ('%s'); loading the "+ "pre-quantised %s checkpoint under the same placement instead",+ gguf_offload_placement.offload_policy,+ gguf_offload_scheme,+ )+ quant_plan = gguf_offload_placement+ gguf_offload_swap = True+ transformer_quant_decline = None+ # A dense bf16 fallback would land the whole denoiser on a card it does not fit.+ dense_fallback_allowed = Falseelse:# This materialises the dense bf16 transformer, so re-check the fit rather than OOMing after# eviction (skipped for a prequant).@@ -5992,15 +6025,42 @@ class DiffusionBackend:),text_encoder_quant = text_encoder_quant,)+ sized_placement, sized_reason = (+ (None, None)+ if plan_keeps_transformer_resident(sized_plan)+ else self._gguf_offload_prequant_placement(+ sized_plan,+ sized_candidate,+ fam = fam,+ target = target,+ auto = transformer_quant_auto,+ gguf_filename = gguf_filename,+ base = base,+ prequant_path = transformer_prequant_path,+ )+ )if plan_keeps_transformer_resident(sized_plan):quant_plan = sized_planif sized_plan.offload_policy != OFFLOAD_NONE:dense_fallback_allowed = False+ elif sized_placement is not None:+ logger.info(+ "diffusion.transformer_quant: the pre-quantised %s checkpoint "+ "offloads ('%s'); loading it under that placement instead of the GGUF",+ sized_candidate.scheme,+ sized_placement.offload_policy,+ )+ quant_plan = sized_placement+ gguf_offload_swap = True+ dense_fallback_allowed = Falseelse:dense_declined = Truedense_fallback_allowed = False- transformer_quant_decline = _torchao_offload_decline(- "the pre-quantised transformer needs", sized_plan+ transformer_quant_decline = (+ sized_reason+ or _torchao_offload_decline(+ "the pre-quantised transformer needs", sized_plan+ ))logger.info("diffusion.transformer_quant_declined: %s",@@ -6132,6 +6192,14 @@ class DiffusionBackend:fetch_base = fetch_base,local_files_only = local_files_only,_load_token = _load_token,+ seed_plan = quant_plan if quant_plan is not None else plan,+ )+ transformer_quant_artifact = self._gguf_route_artifact(+ pipe,+ fam,+ transformer_quant_engaged,+ base = base,+ prequant_path = transformer_prequant_path,)except Exception as exc: # noqa: BLE001 - fall back to the GGUF buildself._raise_if_load_cancelled(_load_token)@@ -7189,6 +7257,15 @@ class DiffusionBackend:if transformer_quant_artifact is not None:# Beside ``source``, never in it: the frontend branches on "auto"/"explicit".resolved["transformer_quant"]["artifact"] = transformer_quant_artifact+ if gguf_offload_swap and transformer_quant_engaged is not None:+ from .diffusion_gguf_route import gguf_offload_swap_reason+ resolved["transformer_quant"]["reason"] = gguf_offload_swap_reason(+ transformer_quant_engaged,+ transformer_quant_artifact,+ plan,+ gguf_filename,+ )+ resolved["transformer_quant"]["replaced"] = f"gguf:{gguf_filename}"# Opt-in (UNSLOTH_DIFFUSION_BG_COMPILE=1): off by default so the same seed twice repeats.bg_module = (@@ -7301,6 +7378,66 @@ class DiffusionBackend:)return self.status()+ @staticmethod+ def _gguf_route_artifact(+ pipe: Any,+ fam: Any,+ scheme: Optional[str],+ *,+ base: Optional[str],+ prequant_path: Optional[str],+ ) -> Optional[str]:+ """``prequant:<repo>/<file>`` when the build loaded a pre-quantized checkpoint, else None."""+ try:+ transformer = getattr(pipe, "transformer", None)+ if scheme is None or getattr(transformer, "_unsloth_prequant_path", None) is None:+ return None+ from .diffusion_denoiser_prequant import prequant_artifact_label++ return prequant_artifact_label(+ resolve_prequant_source(fam, scheme, path_override = prequant_path, base_repo = base),+ transformer,+ )+ except Exception: # noqa: BLE001 - status only+ return None++ @staticmethod+ def _gguf_offload_prequant_placement(+ replanned: Any,+ candidate: Any,+ *,+ fam: Any,+ target: Any,+ auto: bool,+ gguf_filename: Optional[str],+ base: Optional[str],+ prequant_path: Optional[str],+ ) -> tuple[Optional[Any], Optional[str]]:+ """``diffusion_gguf_route.gguf_offload_prequant_placement`` with the cache state resolved."""+ from .diffusion_gguf_route import gguf_offload_prequant_placement++ scheme = getattr(candidate, "scheme", None)+ if candidate is None or not scheme:+ return None, None+ try:+ prequant_cached = (+ _uncached_prequant_repo(+ fam, target, scheme, base_repo = base, prequant_path = prequant_path+ )+ is None+ )+ except Exception: # noqa: BLE001 - unknown cache state: treat as uncached+ prequant_cached = False+ return gguf_offload_prequant_placement(+ replanned,+ candidate,+ scheme = scheme,+ auto = auto,+ gguf_filename = gguf_filename,+ prequant_cached = prequant_cached,+ torchao_offload_plan = torchao_offload_plan,+ )+def _load_dense_quant_pipeline(self,transformer_cls: Any,@@ -7322,9 +7459,13 @@ class DiffusionBackend:fetch_base: Optional[str] = None,local_files_only: bool = False,_load_token: Optional[int] = None,+ seed_plan: Any = None,) -> tuple[Any, str]:"""Build the opt-in fast pipeline and return ``(pipe, engaged_scheme)``.+ ``seed_plan``: the placement the build runs under; an offloading one seeds a pre-quantized checkpoint on the+ host (``prequant_seed_device``).+Two ways to get the quantized transformer, in order: (1) Pre-quantized, when a checkpoint isconfigured for the chosen scheme (an explicit ``prequant_path`` or the family's hostedrepo), loaded onto the meta device and assigned in, so the dense bf16 never lands on the GPU@@ -7370,6 +7511,12 @@ class DiffusionBackend:)check_cancelled()if source is not None:+ # A GPU seed under offload OOMed 8 GB Qwen-Image-2.1 in the streaming hooks.+ seed_device = (+ prequant_seed_device(seed_plan, device, scheme)+ if seed_plan is not None+ else device+ )transformer = load_prequantized_transformer(transformer_cls,fetch_base,@@ -7391,6 +7538,7 @@ class DiffusionBackend:# re-downloaded under the import-time constant.cache_dir = hub_cache_dir(),logger = logger,+ placement_device = None if seed_device == device else seed_device,)check_cancelled()if transformer is not None:@@ -7403,7 +7551,7 @@ class DiffusionBackend:transformer,dtype,hf_token,- device,+ seed_device,base_local_dir,fam = fam,te_quant_mode = text_encoder_quant,diff --git a/studio/backend/core/inference/diffusion_gguf_route.py b/studio/backend/core/inference/diffusion_gguf_route.pynew file mode 100644index 00000000000..90b9d9e52bf--- /dev/null+++ b/studio/backend/core/inference/diffusion_gguf_route.py@@ -0,0 +1,105 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Let a GGUF pick whose memory plan offloads load the cached hosted int8 / fp8 checkpoint under that offload.++Only a pre-quantised checkpoint, never a dense bf16 build; ``auto`` takes it only when cached and only for GGUFs less+accurate than it. Device eligibility (CUDA bf16) is the caller's.+"""++from __future__ import annotations++import os+import re+from typing import Any, Optional++# "0" restores the resident-only swap.+GGUF_OFFLOAD_PREQUANT_ENV = "UNSLOTH_DIFFUSION_GGUF_OFFLOAD_PREQUANT"++# Widest GGUF auto replaces (Qwen-Image-2.1 LPIPS vs bf16: Q5_K_M 0.077, int8 0.059, Q6_K 0.042, fp8 0.10-0.11).+_MAX_REPLACED_GGUF_BITS = {"int8": 5}+_DEFAULT_MAX_REPLACED_GGUF_BITS = 4+# Tokens led by their bit width; any other (BF16, F16, F32, unknown) is kept.+_GGUF_BITS_RE = re.compile(r"^(?:MXFP|IQ|P?TQ|P?Q)([0-9]+)")+++def gguf_offload_prequant_enabled() -> bool:+ return str(os.environ.get(GGUF_OFFLOAD_PREQUANT_ENV, "")).strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def gguf_quant_token(gguf_filename: Optional[str]) -> Optional[str]:+ if not gguf_filename:+ return None+ try:+ from hub.utils.gguf import extract_quant_token+ token = extract_quant_token(str(gguf_filename))+ except Exception: # noqa: BLE001 - an unparseable name is treated as unknown precision+ token = None+ return str(token).upper() if token else None+++def gguf_outranks_scheme(gguf_filename: Optional[str], scheme: Optional[str]) -> bool:+ """Whether the GGUF is at least as accurate as ``scheme``'s checkpoint; an unknown quant counts as outranking."""+ token = gguf_quant_token(gguf_filename)+ if token is None:+ return True+ match = _GGUF_BITS_RE.match(token[3:] if token.startswith("UD-") else token)+ if match is None:+ return True+ return int(match.group(1)) > _MAX_REPLACED_GGUF_BITS.get(+ str(scheme), _DEFAULT_MAX_REPLACED_GGUF_BITS+ )+++def gguf_offload_prequant_placement(+ replanned: Any,+ candidate: Any,+ *,+ scheme: Optional[str],+ auto: bool,+ gguf_filename: Optional[str],+ prequant_cached: bool,+ torchao_offload_plan: Any,+) -> tuple[Optional[Any], Optional[str]]:+ """``(placement, None)`` to swap, ``(None, reason)`` to keep the GGUF by this rule, ``(None, None)`` when it+ does not apply."""+ if not gguf_offload_prequant_enabled():+ return None, None+ if candidate is None or not bool(getattr(candidate, "prequant", False)) or not scheme:+ return None, None+ if auto and not prequant_cached:+ return None, (+ f"the plan offloads the denoiser and the hosted {scheme} checkpoint is not cached; an auto quant never "+ "downloads a second transformer for a GGUF pick"+ )+ if auto and gguf_outranks_scheme(gguf_filename, scheme):+ return None, (+ f"the plan offloads the denoiser, and the picked GGUF ({gguf_quant_token(gguf_filename) or 'unknown quant'}) "+ f"is at least as accurate as the hosted {scheme} checkpoint, so auto keeps the GGUF"+ )+ try:+ placement = torchao_offload_plan(replanned, scheme)+ except Exception: # noqa: BLE001 - an unanswerable placement keeps the GGUF+ placement = None+ return placement, None+++def gguf_offload_swap_reason(+ scheme: str, artifact: Optional[str], placement: Any, gguf_filename: Optional[str]+) -> str:+ what = (+ artifact.split(":", 1)[1]+ if artifact and ":" in artifact+ else (artifact or f"the hosted {scheme} checkpoint")+ )+ token = gguf_quant_token(gguf_filename) or "GGUF"+ return (+ f"the {token} GGUF pick was replaced by {what}: the memory plan offloads "+ f"('{getattr(placement, 'offload_policy', 'offload')}'), where the {scheme} checkpoint runs faster than "+ "dequantising the GGUF every step. Set Precision to Off to run the GGUF itself"+ )diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 49f56a40a6a..79783323cb9 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -4662,6 +4662,11 @@ class DiffusionResolvedControl(BaseModel):"weights being quantised in memory. Declared here or pydantic drops it and no API client ""ever sees the provenance. Null on every other control and on a runtime quantise.",)+ replaced: Optional[str] = Field(+ None,+ description = 'The picked checkpoint that did NOT run, as "gguf:<file>", when a GGUF pick whose memory plan '+ "offloads loaded the hosted pre-quantized checkpoint (``artifact``) instead. Null otherwise.",+ )class DiffusionDownloadPlanEntry(BaseModel):diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex f60f1eb8556..2d41b06e38e 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -6796,6 +6796,7 @@ def test_diffusion_status_response_carries_resolved():"status": "applied","reason": "blackwell","artifact": None,+ "replaced": None,}}# Absent by default (nothing resolved / native engine).@@ -6818,7 +6819,7 @@ def test_diffusion_status_response_carries_requested_precision():}resp = DiffusionStatusResponse(loaded = True, resolved = rec)assert resp.model_dump()["resolved"] == {- "transformer_quant": {**rec["transformer_quant"], "artifact": None}+ "transformer_quant": {**rec["transformer_quant"], "artifact": None, "replaced": None}}@@ -14194,3 +14195,228 @@ def test_candidate_overrides_price_the_encoder_the_load_opens(monkeypatch):]== 0)+++_Q4 = "m-Q4_K_M.gguf"+++def _gguf_offload_swap_setup(+ monkeypatch,+ tmp_path,+ *,+ scheme = "int8",+ policy = "model",+):+ """A CUDA GGUF pick whose plan and every quantised re-plan offload, with a cached hosted checkpoint."""+ import dataclasses++ from core.inference import diffusion as dmod++ (tmp_path / _Q4).write_bytes(b"x")+ (tmp_path / "m-Q8_0.gguf").write_bytes(b"x")+ backend = DiffusionBackend()+ _force_cuda_target(backend, monkeypatch)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GGUF_OFFLOAD_PREQUANT", raising = False)+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda target: True)+ monkeypatch.setattr(+ dmod, "select_transformer_quant_scheme", lambda target, mode, family = None, **_kw: scheme+ )+ monkeypatch.setattr(dmod, "_uncached_prequant_repo", lambda *a, **k: None)+ monkeypatch.setattr(+ DiffusionBackend, "_auto_prequant_retry_scheme", staticmethod(lambda *a, **k: None)+ )+ monkeypatch.setattr(+ dmod,+ "resolve_dense_quant_candidate",+ lambda **kw: types.SimpleNamespace(+ scheme = scheme, transient_transformer_mib = 7_000, companions_mib = 8_000, prequant = True+ ),+ )+ source = types.SimpleNamespace(+ kind = "repo", location = "unsloth/Z-FP8", filename = "Z-INT8.safetensors"+ )+ monkeypatch.setattr(dmod, "resolve_prequant_source", lambda *a, **k: source)+ monkeypatch.setattr(dmod, "torchao_offload_plan", lambda plan, s, **k: plan)+ orig_plan = DiffusionBackend._plan_memory++ def spy_plan(self, *a, **k):+ return dataclasses.replace(orig_plan(self, *a, **k), offload_policy = policy)++ monkeypatch.setattr(DiffusionBackend, "_plan_memory", spy_plan)+ calls: list = []++ def fake_dense_load(self, *a, **k):+ calls.append(k)+ pipe = _FakePipe()+ pipe.transformer = types.SimpleNamespace(_unsloth_prequant_path = "/hub/Z-INT8.safetensors")+ return pipe, scheme++ monkeypatch.setattr(DiffusionBackend, "_load_dense_quant_pipeline", fake_dense_load)+ return backend, calls+++def _load_gguf(+ backend,+ tmp_path,+ name = _Q4,+ **kwargs,+):+ return backend.load_pipeline(+ str(tmp_path), gguf_filename = name, family_override = "z-image", **kwargs+ )+++def test_offloading_gguf_pick_loads_the_cached_int8_checkpoint(fake_runtime, tmp_path, monkeypatch):+ backend, calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)+ status = _load_gguf(backend, tmp_path)+ assert len(calls) == 1+ assert calls[0]["allow_dense_fallback"] is False+ assert calls[0]["seed_plan"].offload_policy == "model"+ assert status["transformer_quant"] == "int8"+ assert status["offload_policy"] == "model"+ tq = status["resolved"]["transformer_quant"]+ assert tq["value"] == "int8" and tq["source"] == "auto"+ assert tq["artifact"] == "prequant:unsloth/Z-FP8/Z-INT8.safetensors"+ assert tq["replaced"] == f"gguf:{_Q4}"+ assert "Q4_K_M GGUF pick was replaced by unsloth/Z-FP8/Z-INT8.safetensors" in tq["reason"]+++def test_gguf_offload_swap_kill_switch_keeps_the_gguf(fake_runtime, tmp_path, monkeypatch):+ backend, calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GGUF_OFFLOAD_PREQUANT", "0")+ status = _load_gguf(backend, tmp_path)+ assert calls == []+ assert status["transformer_quant"] is None+ assert _FakeTransformer.last["path"].endswith(_Q4)+ assert "replaced" not in status["resolved"]["transformer_quant"]+++def test_gguf_offload_swap_keeps_a_q8_pick_on_auto(fake_runtime, tmp_path, monkeypatch):+ backend, calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)+ status = _load_gguf(backend, tmp_path, name = "m-Q8_0.gguf")+ assert calls == []+ assert status["transformer_quant"] is None+ reason = status["resolved"]["transformer_quant"]["reason"]+ assert "Q8_0" in reason and "at least as accurate" in reason++[email protected]("request_", ["off", "none"])+def test_gguf_offload_swap_never_overrides_precision_off(+ fake_runtime, tmp_path, monkeypatch, request_+):+ backend, calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)+ status = _load_gguf(backend, tmp_path, transformer_quant = request_)+ assert calls == []+ assert status["transformer_quant"] is None+ assert _FakeTransformer.last["path"].endswith(_Q4)+++def test_gguf_offload_swap_falls_back_to_the_gguf_when_the_build_fails(+ fake_runtime, tmp_path, monkeypatch+):+ backend, _calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)++ def boom(self, *a, **k):+ raise RuntimeError("checkpoint unreadable")++ monkeypatch.setattr(DiffusionBackend, "_load_dense_quant_pipeline", boom)+ status = _load_gguf(backend, tmp_path)+ assert status["transformer_quant"] is None+ assert _FakeTransformer.last["path"].endswith(_Q4)+ assert "replaced" not in status["resolved"]["transformer_quant"]+++def test_gguf_offload_swap_on_mps_keeps_the_gguf(fake_runtime, tmp_path, monkeypatch):+ from core.inference import diffusion as dmod++ backend, calls = _gguf_offload_swap_setup(monkeypatch, tmp_path)+ torch = sys.modules["torch"]+ monkeypatch.setattr(+ backend, "_target_for_ordinal", lambda fam, ordinal = None: _mps_target(torch)+ )+ # The real device gate, not the test's CUDA override.+ from core.inference.diffusion_transformer_quant import dense_transformer_supported++ monkeypatch.setattr(dmod, "dense_transformer_supported", dense_transformer_supported)+ status = _load_gguf(backend, tmp_path)+ assert calls == []+ assert status["transformer_quant"] is None+++ "policy, stream, env, placed_on_host",+ [+ ("streaming", True, None, True),+ ("group", True, None, True),+ ("model", True, None, True),+ ("none", True, None, False),+ ("group", False, None, False), # denoiser stays resident: seed where it runs+ ("streaming", True, "0", False), # UNSLOTH_DIFFUSION_PREQUANT_SEED_ON_HOST=0+ ],+)+def test_gguf_route_prequant_seed_lands_on_the_host_when_the_plan_offloads(+ fake_runtime, monkeypatch, policy, stream, env, placed_on_host+):+ from core.inference import diffusion as dmod++ if env is None:+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PREQUANT_SEED_ON_HOST", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PREQUANT_SEED_ON_HOST", env)+ monkeypatch.setattr(dmod, "_planned_quant_scheme", lambda *a, **k: "int8")+ monkeypatch.setattr(+ dmod,+ "resolve_prequant_source",+ lambda *a, **k: types.SimpleNamespace(kind = "repo", location = "o/r", filename = "f"),+ )+ loaded: dict = {}++ def fake_load(*a, **k):+ loaded.update(k)+ return object()++ monkeypatch.setattr(dmod, "load_prequantized_transformer", fake_load)+ assembled: list = []+ monkeypatch.setattr(+ DiffusionBackend,+ "_assemble_pipe",+ staticmethod(+ lambda pcls, base, tr, dtype, tok, device, *a, **k: assembled.append(device)+ or _FakePipe()+ ),+ )+ _pipe, scheme = DiffusionBackend()._load_dense_quant_pipeline(+ object,+ object,+ "base/repo",+ "cuda:0",+ "bf16",+ None,+ types.SimpleNamespace(device = "cuda", dtype = "bf16"),+ "int8",+ fam = types.SimpleNamespace(name = "qwen-image-2.1"),+ seed_plan = types.SimpleNamespace(offload_policy = policy, stream_transformer = stream),+ )+ assert scheme == "int8"+ assert loaded["placement_device"] == ("cpu" if placed_on_host else None)+ assert assembled == ["cpu" if placed_on_host else "cuda:0"]+++def test_diffusion_status_response_keeps_the_gguf_a_swap_replaced():+ # Pydantic drops undeclared keys.+ from models.inference import DiffusionStatusResponse++ rec = {+ "transformer_quant": {+ "value": "int8",+ "source": "auto",+ "reason": "the Q4_K_M GGUF pick was replaced",+ "artifact": "prequant:o/r/f.safetensors",+ "replaced": "gguf:m-Q4_K_M.gguf",+ }+ }+ dumped = DiffusionStatusResponse(loaded = True, resolved = rec).model_dump()["resolved"][+ "transformer_quant"+ ]+ assert dumped["replaced"] == "gguf:m-Q4_K_M.gguf"+ assert dumped["artifact"] == "prequant:o/r/f.safetensors"diff --git a/studio/backend/tests/test_diffusion_gguf_offload_route.py b/studio/backend/tests/test_diffusion_gguf_offload_route.pynew file mode 100644index 00000000000..d6be197ca68--- /dev/null+++ b/studio/backend/tests/test_diffusion_gguf_offload_route.py@@ -0,0 +1,179 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A GGUF pick whose memory plan offloads loads the cached hosted int8 / fp8 checkpoint under the same placement."""++from __future__ import annotations++import types+from dataclasses import replace++import pytest++from core.inference import diffusion_gguf_route as route+from core.inference import diffusion_memory as mem+from core.inference.diffusion_memory import (+ OFFLOAD_GROUP,+ OFFLOAD_NONE,+ OFFLOAD_SEQUENTIAL,+ OFFLOAD_STREAMING,+ DeviceMemory,+ plan_diffusion_memory,+)++[email protected](autouse = True)+def _roomy_host(monkeypatch):+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 10**7)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: (0, 40))+ monkeypatch.delenv(route.GGUF_OFFLOAD_PREQUANT_ENV, raising = False)+++def _plan(policy, *, stream_transformer = True):+ device = DeviceMemory("cuda", "cuda:0", "discrete_vram", 11_000, 12_288)+ plan = plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = device,+ model_dense_mib = 15_000,+ runtime_headroom_mib = 2_000,+ companion_dense_mib = 8_000,+ text_encoder_dense_mib = 7_800,+ )+ return replace(plan, offload_policy = policy, stream_transformer = stream_transformer)+++def _ao(version = (0, 18)):+ return lambda plan, scheme: mem.torchao_offload_plan(plan, scheme, torchao_version = version)+++INT8 = types.SimpleNamespace(scheme = "int8", prequant = True, transient_transformer_mib = 7_000)+DENSE = types.SimpleNamespace(scheme = "int8", prequant = False, transient_transformer_mib = 20_000)+++def _decide(+ plan,+ candidate = INT8,+ *,+ auto = True,+ gguf = "qwen-image-2.1-Q4_K_M.gguf",+ cached = True,+ version = (0, 18),+):+ return route.gguf_offload_prequant_placement(+ plan,+ candidate,+ scheme = getattr(candidate, "scheme", None),+ auto = auto,+ gguf_filename = gguf,+ prequant_cached = cached,+ torchao_offload_plan = _ao(version),+ )++[email protected]("policy", [OFFLOAD_STREAMING, OFFLOAD_GROUP])+def test_offloading_q4_pick_takes_the_cached_int8_checkpoint(policy):+ placement, reason = _decide(_plan(policy))+ assert reason is None+ assert placement is not None and placement.offload_policy == policy+++def test_sequential_offload_keeps_the_gguf():+ # torchao never measured under sequential offload: the caller's decline stands.+ assert _decide(_plan(OFFLOAD_SEQUENTIAL)) == (None, None)+++def test_torchao_too_old_to_stream_int8_keeps_the_gguf():+ assert _decide(_plan(OFFLOAD_STREAMING), version = (0, 16)) == (None, None)++[email protected]("value", ["0", "off", "false", "no"])+def test_kill_switch_restores_the_resident_only_rule(monkeypatch, value):+ monkeypatch.setenv(route.GGUF_OFFLOAD_PREQUANT_ENV, value)+ assert _decide(_plan(OFFLOAD_STREAMING)) == (None, None)+++def test_never_builds_dense_under_offload():+ assert _decide(_plan(OFFLOAD_STREAMING), DENSE) == (None, None)+++def test_auto_never_downloads_a_second_denoiser():+ placement, reason = _decide(_plan(OFFLOAD_STREAMING), cached = False)+ assert placement is None and "not cached" in reason+++def test_explicit_scheme_may_fetch_its_checkpoint():+ placement, reason = _decide(_plan(OFFLOAD_STREAMING), cached = False, auto = False)+ assert reason is None and placement is not None+++ "gguf, outranks",+ [+ ("qwen-image-2.1-Q2_K.gguf", False),+ ("qwen-image-2.1-Q3_K_M.gguf", False),+ ("qwen-image-2.1-Q4_K_M.gguf", False),+ ("qwen-image-2.1-Q5_K_S.gguf", False),+ ("qwen-image-2.1-Q5_K_M.gguf", False),+ ("model-UD-Q4_K_XL.gguf", False),+ ("model-IQ4_XS.gguf", False),+ ("model-MXFP4.gguf", False),+ ("qwen-image-2.1-Q6_K.gguf", True),+ ("qwen-image-2.1-Q6_K_XL.gguf", True),+ ("model-UD-Q6_K_XL.gguf", True),+ ("qwen-image-2.1-Q8_0.gguf", True),+ ("qwen-image-2.1-F16.gguf", True),+ ("qwen-image-2.1-BF16.gguf", True),+ ("model.gguf", True),+ ("model-MXFP8.gguf", True),+ ("model-IQ6_XS.gguf", True),+ ("model-PQ8_0.gguf", True),+ ("model-TQ1_0.gguf", False),+ ("model-Q5_0.gguf", False),+ (None, True),+ ],+)+def test_accuracy_gate_by_quant(gguf, outranks):+ assert route.gguf_outranks_scheme(gguf, "int8") is outranks+++ "gguf, outranks",+ [+ ("x-Q4_K_M.gguf", False),+ ("x-Q5_K_M.gguf", True),+ ("x-Q6_K.gguf", True),+ ("x-MXFP8.gguf", True),+ ],+)+def test_fp8_only_replaces_4_bit_and_narrower(gguf, outranks):+ assert route.gguf_outranks_scheme(gguf, "fp8") is outranks+++def test_auto_keeps_a_q8_pick_explicit_int8_still_swaps():+ placement, reason = _decide(_plan(OFFLOAD_STREAMING), gguf = "qwen-image-2.1-Q8_0.gguf")+ assert placement is None and "Q8_0" in reason and "at least as accurate" in reason+ placement, reason = _decide(+ _plan(OFFLOAD_STREAMING), gguf = "qwen-image-2.1-Q8_0.gguf", auto = False+ )+ assert placement is not None and reason is None+++def test_swap_reason_names_what_ran_and_how_to_keep_the_gguf():+ text = route.gguf_offload_swap_reason(+ "int8",+ "prequant:unsloth/Qwen-Image-2.1-FP8/Qwen-Image-2.1-INT8.safetensors",+ types.SimpleNamespace(offload_policy = OFFLOAD_STREAMING),+ "qwen-image-2.1-Q4_K_M.gguf",+ )+ assert (+ "Q4_K_M GGUF pick was replaced by unsloth/Qwen-Image-2.1-FP8/Qwen-Image-2.1-INT8.safetensors"+ in text+ )+ assert "'streaming'" in text and "Precision to Off" in text+++def test_resident_plans_are_not_this_rules_business():+ placement, _ = _decide(_plan(OFFLOAD_NONE))+ assert placement.offload_policy == OFFLOAD_NONEdiff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.pyindex 59688f7f167..48f527f3e26 100644--- a/studio/backend/tests/test_diffusion_routes.py+++ b/studio/backend/tests/test_diffusion_routes.py@@ -1471,6 +1471,7 @@ def test_status_passes_through_resolved(client, monkeypatch):assert body["resolved"]["transformer_quant"] == {**resolved["transformer_quant"],"artifact": None,+ "replaced": None,}# Entries from an older backend (no requested/status) still parse, defaulted to "applied".assert body["resolved"]["speed_mode"]["requested"] is Nonediff --git a/studio/backend/tests/test_video_routes.py b/studio/backend/tests/test_video_routes.pyindex fe9fc83eb4e..e056979c180 100644--- a/studio/backend/tests/test_video_routes.py+++ b/studio/backend/tests/test_video_routes.py@@ -888,14 +888,16 @@ def test_status_passthrough(client, monkeypatch):)body = client.get("/api/inference/video/status").json()assert body["loaded"] is True and body["family"] == "ltx-2"- # ``artifact`` is additive on the response model: a record naming no hosted checkpoint is null.+ # ``artifact`` / ``replaced`` are additive on the response model: a record naming neither is null.assert body["resolved"]["transformer_quant"] == {**resolved["transformer_quant"],"artifact": None,+ "replaced": None,}assert body["resolved"]["text_encoder_quant"] == {**resolved["text_encoder_quant"],"artifact": None,+ "replaced": None,}# Entries from an older backend (no requested/status) still parse, defaulted to "applied".assert body["resolved"]["speed_mode"]["requested"] is Nonediff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.tsindex 3cbf95c4b8a..cd98f9b499c 100644--- a/studio/frontend/src/features/images/api.ts+++ b/studio/frontend/src/features/images/api.ts@@ -23,6 +23,8 @@ export interface DiffusionResolvedControl {reason: string;// "prequant:<repo>/<file>" when a hosted checkpoint was seeded; absent on a runtime quantise.artifact?: string | null;+ // "gguf:<file>" the `artifact` replaced.+ replaced?: string | null;}export interface DiffusionStatus {