Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: official image picks default to the hosted FP8 or INT8 checkpoint (#10883)

Daniel Han584b91e8a1fa
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index ef1a3f6bcdc..1db0ea71aca 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -88,6 +88,7 @@ from .diffusion_krea2 import KREA2_FAMILY_NAME, load_krea2_pipeline
from .diffusion_memory import (
MEMORY_MODE_BALANCED,
MEMORY_MODE_LOW_VRAM,
+ DeviceMemory,
OFFLOAD_NONE,
OFFLOAD_STREAMING,
apply_memory_plan,
@@ -110,6 +111,7 @@ from .diffusion_memory import (
from .diffusion_torchao_patches import install_torchao_int_mm_patch
from .diffusion_speed import (
SPEED_DEFAULT,
+ SPEED_EAGER,
SPEED_MAX,
SPEED_OFF,
apply_speed_optims,
@@ -157,7 +159,16 @@ from .diffusion_precision import (
torchao_quantize_importable,
)
from .diffusion_te_prequant import te_prequant_pipe_kwargs
+from .diffusion_denoiser_prequant import (
+ DENOISER_COMPONENT,
+ PIPELINE_SEED_DECLINED,
+ denoiser_prequant_cached,
+ denoiser_prequant_source,
+ pipeline_seed_supported,
+ prequant_artifact_label,
+)
from .diffusion_prequant import (
+ hosted_fast_accum_conflict,
load_prequantized_transformer,
prequant_checkpoint_cached,
resolve_prequant_source,
@@ -169,6 +180,7 @@ from .diffusion_auto_policy import (
RESOLVED_UNSUPPORTED,
base_repo_bf16_components_gb,
build_resolved_record,
+ estimate_dense_quant,
family_bf16_components_gb,
precision_fallback_allowed,
precision_refusal_message,
@@ -178,12 +190,20 @@ from .diffusion_auto_policy import (
from .diffusion_transformer_quant import (
TQ_AUTO,
DEFAULT_MIN_LINEAR_FEATURES,
+ DenoiserView as _DenoiserView,
+ dense_quant_blocker,
+ dense_quant_supported_kind,
+ dense_quant_unsupported_kind_reason,
dense_transformer_supported,
dense_transformer_unsupported_reason,
+ denoiser_modules,
explain_unusable_scheme,
+ mark_source_precision,
normalize_transformer_quant,
quantize_transformer,
select_transformer_quant_scheme,
+ stored_denoiser_precision,
+ transformer_is_quantised,
)
from utils.paths.path_utils import (
any_not_appledouble_metadata,
@@ -920,6 +940,11 @@ class _LoadingState:
# Where the companion BYTES land: ``base_repo``, or its mirror when one was swapped in. Never surfaced
# (``base_repo`` stays the id status() reports), but the cache scan and the delete guard must look here.
fetch_repo: Optional[str] = None
+ # Third repos this load downloads from; the cache scan and the delete guard both need them.
+ asset_repos: tuple[str, ...] = ()
+ # ``(repo, filename, declared size, repo bytes on disk when this load claimed it)`` when we fetch
+ # ONE named file: the scan counts what this load adds, not every byte already in the repo.
+ asset_files: tuple[tuple[str, str, int, int], ...] = ()
account_id: Optional[str] = None
@@ -1223,6 +1248,35 @@ def _memory_request_forces_offload(memory_mode: Optional[str], cpu_offload: bool
return mode is None and bool(cpu_offload)
+def _pipeline_quant_uncompilable_reason(
+ target: Any, fam: Any, speed_mode: Optional[str], *, model_kind: str
+) -> Optional[str]:
+ """Why a PIPELINE load must keep its dense weights rather than quantise them, or None.
+
+ An uncompiled torchao transformer is ~30x slower than the bf16 it replaced, so converting
+ without a compile is a pessimisation. Both callers ask here: the route preflight, so the refusal
+ lands before the arbiter evicts anything, and the loader, so an automatic request declines to
+ bf16. GGUF is out of scope, substituting dense base weights and falling back to the packed file.
+
+ speed=off is absent on purpose: an engaged quant upgrades it to `default`, and an AUTO request
+ under it is rewritten to off long before either caller."""
+ if model_kind != "pipeline":
+ return None
+ # A string compare, not resolve_speed_mode: this runs on the route, where an unvalidated value must not raise.
+ if str(speed_mode or "").strip().lower() == SPEED_EAGER:
+ return (
+ "Speed is set to 'eager', and a quantised transformer that is not compiled runs far "
+ "slower than the bf16 weights it replaces. Pick a compiling speed mode to combine the two"
+ )
+ if not compile_eligible(target, is_gguf = False, family = fam):
+ return (
+ "this process cannot run a torch.compile (no Triton, TORCHDYNAMO_DISABLE, or a "
+ "family/device that does not compile), and a quantised transformer that is not "
+ "compiled runs far slower than the bf16 weights it replaces"
+ )
+ return None
+
+
def _clear_exception_frames(exc: BaseException) -> None:
"""Release failed-call locals while preserving traceback locations."""
errors, seen = [exc], set()
@@ -1438,6 +1492,7 @@ class DiffusionBackend:
text_encoder_quant: Optional[str] = None,
memory_mode: Optional[str] = None,
cpu_offload: bool = False,
+ speed_mode: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
) -> None:
"""Raise ``RuntimeError`` (the route's 409) when an EXPLICIT precision cannot run here.
@@ -1471,6 +1526,7 @@ class DiffusionBackend:
te_mode = te_mode,
memory_mode = memory_mode,
cpu_offload = cpu_offload,
+ speed_mode = speed_mode,
)
def _assert_precision_for_target(
@@ -1483,14 +1539,20 @@ class DiffusionBackend:
te_mode: Optional[str],
memory_mode: Optional[str],
cpu_offload: bool,
+ speed_mode: Optional[str] = None,
) -> None:
"""The body of ``assert_precision_available``, run with the selected card current."""
if pinned is not None and pinned != TQ_AUTO:
reason = None
- if model_kind != "gguf":
+ if not dense_quant_supported_kind(model_kind):
+ reason = dense_quant_unsupported_kind_reason(model_kind)
+ elif getattr(fam, "denoiser_attr", "transformer") != "transformer":
+ # The family table already answers this, so refuse without network or assembly. Without it
+ # dense_quant_blocker says the same only after the eviction and the full download.
reason = (
- f"the dense transformer-quant path applies to GGUF picks only, and this is a "
- f"'{model_kind}' load, which runs the precision its checkpoint carries"
+ f"'{getattr(fam, 'name', None)}' denoises with a "
+ f"{getattr(fam, 'denoiser_attr', 'unet')}, not a transformer, and the dense torchao schemes "
+ "do not cover it"
)
elif _memory_request_forces_offload(memory_mode, cpu_offload):
# Not a measurement: balanced and low_vram name their policy outright, and the legacy flag forces
@@ -1505,6 +1567,13 @@ class DiffusionBackend:
)
elif not dense_transformer_supported(target):
reason = dense_transformer_unsupported_reason(target)
+ elif (
+ uncompilable := _pipeline_quant_uncompilable_reason(
+ target, fam, speed_mode, model_kind = model_kind
+ )
+ ) is not None:
+ # Deterministic from the request: else the loader refuses only after the eviction and download.
+ reason = uncompilable
elif (
select_transformer_quant_scheme(
target,
@@ -1840,6 +1909,71 @@ class DiffusionBackend:
return candidate
return None
+ def _restore_skipped_transformer_shards(
+ self,
+ repo_id: str,
+ base: str,
+ skipped: Sequence[str],
+ hf_token: Optional[str],
+ *,
+ fetch_base: Optional[str] = None,
+ base_local_dir: Optional[str] = None,
+ local_files_only: bool = False,
+ ) -> Optional[str]:
+ """Stage the shards a seeded plan left out of the pull; returns the snapshot dir to assemble
+ from, or None for the hub id. ``from_pretrained`` treats a local dir as terminal, so a
+ failed seed has nothing."""
+ files = [name for name in (skipped or ()) if name]
+ if not files:
+ return base_local_dir
+ logger.warning(
+ "diffusion.denoiser_prequant: restoring the %d released denoiser shard(s) the "
+ "pre-download skipped",
+ len(files),
+ )
+ restored = self._prefetch_files(
+ repo_id,
+ None,
+ base,
+ [*files, "model_index.json"],
+ hf_token,
+ fetch_base = fetch_base,
+ local_files_only = local_files_only,
+ )
+ # Accept only the one snapshot BOTH the staging and this top-up vouched for. If they resolved through
+ # different cache roots no local directory holds everything, and the hub id, which resolves each file
+ # through its own root, is then the only complete source.
+ if restored is None or base_local_dir is None or Path(restored) != Path(base_local_dir):
+ return None
+ return base_local_dir
+
+ def _fetch_denoiser_prequant(
+ self,
+ entry: Optional[tuple[str, str, int]],
+ hf_token: Optional[str],
+ *,
+ cancel_event: Optional[threading.Event] = None,
+ ) -> None:
+ """Stage the seeded denoiser, best-effort; done here so an unload can preempt the fetch."""
+ if entry is None:
+ return
+ repo, filename, _size = entry
+ try:
+ from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
+ hf_hub_download_with_xet_fallback(
+ repo,
+ filename,
+ hf_token,
+ cancel_event = cancel_event if cancel_event is not None else self._cancel_event,
+ reuse_other_cache_root = True,
+ )
+ except Exception as exc: # noqa: BLE001 -- the loader re-resolves and falls back to dense
+ if isinstance(exc, RuntimeError) and str(exc) == "Cancelled":
+ raise
+ logger.warning(
+ "diffusion.denoiser_prequant_fetch_failed: %s/%s: %s", repo, filename, exc
+ )
+
def _prefetch_files(
self,
repo_id: str,
@@ -2116,6 +2250,9 @@ class DiffusionBackend:
model_kind = resolve_model_kind(gguf_filename, model_kind),
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
+ # An uncompiled torchao transformer loses to the bf16 it replaces, so 'eager' stays dense. Refusing
+ # an explicit scheme here says so before the download.
+ speed_mode = speed_mode,
gpu_ordinal = gpu_ordinal,
)
@@ -2197,7 +2334,44 @@ class DiffusionBackend:
local_files_only = local_files_only,
base_repo = base,
)
+ pipeline_planned = self._pipeline_planned_denoiser_scheme(
+ fam,
+ base = base,
+ kind = kind,
+ transformer_quant = kwargs.get("transformer_quant"),
+ speed_mode = kwargs.get("speed_mode"),
+ memory_mode = kwargs.get("memory_mode"),
+ cpu_offload = bool(kwargs.get("cpu_offload")),
+ loras = kwargs.get("loras"),
+ transformer_prequant_path = kwargs.get("transformer_prequant_path"),
+ gpu_ordinal = kwargs.get("gpu_ordinal"),
+ repo_id = kwargs["repo_id"],
+ fast_accum = kwargs.get("transformer_quant_fast_accum"),
+ )
+ if local_files_only and pipeline_planned not in (None, PIPELINE_SEED_DECLINED):
+ # Offline twin of the Hub probe below. The first load left the released shards out of the cache,
+ # so suppressing the seed would assemble bf16 from a snapshot that has none. Cache only, never a request.
+ if not denoiser_prequant_cached(
+ fam,
+ pipeline_planned,
+ base_repo = base,
+ path_override = kwargs.get("transformer_prequant_path"),
+ cache_dir = hub_cache_dir(),
+ ):
+ pipeline_planned = None
+ dit_prequant = (
+ None
+ if local_files_only
+ else self._dit_prequant_plan_source(
+ fam,
+ kind,
+ kwargs.get("hf_token"),
+ {**kwargs, "_pipeline_prequant_planned": pipeline_planned},
+ )
+ )
+ skip_transformer_weights = kind == "pipeline" and dit_prequant is not None
resident_sizes: list[tuple[str, int]] = []
+ skipped_transformer_files: list[str] = []
fetch_repos: dict[str, str] = {}
expected, base_files = self._estimate_download_bytes(
kwargs["repo_id"],
@@ -2220,10 +2394,39 @@ class DiffusionBackend:
else False
),
skip_te_components = tuple(te_prequant_files),
+ skip_transformer_weights = skip_transformer_weights,
+ skipped_files_out = skipped_transformer_files,
local_files_only = local_files_only,
resident_file_sizes_out = resident_sizes,
fetch_repos_out = fetch_repos,
)
+ # An OPERATOR'S OWN checkpoint has no Hub entry, so the plan drops no shards and the seed costs no bytes.
+ # Suppressing it would ignore the file the load was asked to use and quantise the released bf16 shards in
+ # memory instead. Cache only, never a request.
+ local_seed = (
+ bool(kwargs.get("transformer_prequant_path"))
+ and pipeline_planned not in (None, PIPELINE_SEED_DECLINED)
+ and denoiser_prequant_cached(
+ fam,
+ pipeline_planned,
+ base_repo = base,
+ path_override = kwargs.get("transformer_prequant_path"),
+ cache_dir = hub_cache_dir(),
+ )
+ )
+ # Pinned into the load, decline included: the pull was scoped on this answer.
+ kwargs["_pipeline_prequant_planned"] = (
+ PIPELINE_SEED_DECLINED
+ if pipeline_planned == PIPELINE_SEED_DECLINED
+ else (
+ pipeline_planned
+ if skip_transformer_weights or local_files_only or local_seed
+ else None
+ )
+ )
+ kwargs["_pipeline_prequant_skipped"] = tuple(skipped_transformer_files)
+ if dit_prequant is not None:
+ expected += int(dit_prequant[2])
# Only shards this prefetch staged may be materialised by the dense fallback, so read it off the staged
# list: a failed size estimate drops every base file too. A LOCAL base directory has no listing to fail at
# (model_info raises on a path) and its shards are already there, so it counts as staged on the filesystem
@@ -2263,7 +2466,8 @@ class DiffusionBackend:
base,
kind = kind,
declared_files = resident_sizes,
- prequant_bytes = _prequant_plan_bytes(te_prequant_files),
+ prequant_bytes = _prequant_plan_bytes(te_prequant_files)
+ + (int(dit_prequant[2]) if skip_transformer_weights else 0),
extra_bf16_bytes = _predownload_encoder_bf16_bytes(
fam, te_prequant_files, pipeline_declared = bool(resident_sizes)
),
@@ -2278,11 +2482,32 @@ class DiffusionBackend:
shortfall,
)
raise RuntimeError(shortfall)
+ # Outside the lock: a disk scan, and nothing of ours has written to the artifact repo yet.
+ asset_baseline = self._cache_bytes(dit_prequant[0]) if skip_transformer_weights else 0
with self._load_cancel_lock:
if self._load_token == token and self._loading is not None:
self._loading.base_repo = base
self._loading.fetch_repo = fetch_base
self._loading.expected_bytes = expected
+ if skip_transformer_weights:
+ # Claimed before a byte moves: a mid-fetch delete would leave this load with nothing.
+ self._loading.asset_repos = tuple(
+ dict.fromkeys(self._loading.asset_repos + (dit_prequant[0],))
+ )
+ self._loading.asset_files += (
+ (
+ dit_prequant[0],
+ dit_prequant[1],
+ int(dit_prequant[2]),
+ asset_baseline,
+ ),
+ )
+ if skip_transformer_weights:
+ self._fetch_denoiser_prequant(
+ dit_prequant,
+ kwargs.get("hf_token"),
+ cancel_event = cancel_event,
+ )
# Download outside the lock so unload/an eviction can preempt the pull. The carried snapshot is the
# fallback, never the override: it fires only when the estimate came back empty, since the metadata that
# fills it is the same call whose failure earned the escape. Without it the load 401s with every byte
@@ -2364,6 +2589,19 @@ class DiffusionBackend:
downloaded = self._cache_bytes(loading.repo_id)
if companion and companion != loading.repo_id:
downloaded += self._cache_bytes(companion)
+ scoped = {entry[0] for entry in loading.asset_files}
+ for asset in loading.asset_repos:
+ if asset and asset not in (loading.repo_id, companion) and asset not in scoped:
+ downloaded += self._cache_bytes(asset)
+ for repo, filename, size, baseline in loading.asset_files:
+ if not repo or repo in (loading.repo_id, companion):
+ continue
+ # The finished file plus what this load added since it claimed the repo (in-flight
+ # ``blobs/*.incomplete``), capped at the one file. A sibling scheme sits in the baseline.
+ downloaded += min(
+ size,
+ self._cache_file_bytes(repo, filename) + max(0, self._cache_bytes(repo) - baseline),
+ )
expected = loading.expected_bytes
# Downloads done, still finalizing. The cache scan can exceed the estimate, so clamp to 100%.
if expected > 0 and downloaded >= expected * 0.999:
@@ -2376,9 +2614,8 @@ class DiffusionBackend:
def loading_repo_ids(self) -> tuple[str, ...]:
"""Repo ids an in-flight background load is downloading (empty when idle). The delete-cached
guard needs this: during a load ``status()["loaded"]`` is still False, but deleting the
- target repo (or its companion base) would yank blobs and snapshot files from under the
- download/assembly. Includes the mirror when one was swapped in: that is where the
- companion bytes land."""
+ target repo (or its companion base) would yank blobs from under the download. Includes the
+ mirror when one was swapped in, and a seeded denoiser's repo."""
# _load_cancel_lock alone: _loading lives under it, and this predicate runs inside the GPU
# arbiter's lock (release_if), so taking _lock here let a multi-minute build stall every
# other modality's acquire.
@@ -2386,7 +2623,12 @@ class DiffusionBackend:
loading = self._loading
if loading is None or loading.error is not None:
return ()
- ids = (loading.repo_id, loading.base_repo, loading.fetch_repo)
+ ids = (
+ loading.repo_id,
+ loading.base_repo,
+ loading.fetch_repo,
+ *loading.asset_repos,
+ )
return tuple(dict.fromkeys(r for r in ids if r))
def draining_repo_ids(self) -> tuple[str, ...]:
@@ -2461,6 +2703,130 @@ class DiffusionBackend:
logger.warning("diffusion.te_prequant_plan_failed: %s", exc)
return {}
+ def _pipeline_planned_denoiser_scheme(
+ self,
+ fam: Any,
+ *,
+ base: Optional[str],
+ kind: str,
+ transformer_quant: Optional[str],
+ speed_mode: Optional[str],
+ memory_mode: Optional[str] = None,
+ cpu_offload: bool = False,
+ loras: Any = None,
+ transformer_prequant_path: Optional[str] = None,
+ gpu_ordinal: Optional[int] = None,
+ repo_id: Optional[str] = None,
+ fetch_base: Optional[str] = None,
+ fast_accum: Optional[bool] = None,
+ ) -> Optional[str]:
+ """The scheme an official ``kind == "pipeline"`` pick seeds its denoiser from, settled BEFORE
+ anything is downloaded; None keeps the released bf16 weights and ``PIPELINE_SEED_DECLINED``
+ marks an artifact-sized plan that would still offload. Never raises."""
+ try:
+ if kind != "pipeline" or fam is None:
+ return None
+ if not pipeline_seed_supported(fam):
+ return None
+ raw = transformer_quant
+ auto = raw is None or str(raw).strip().lower() in ("", "auto")
+ # load_pipeline rewrites an AUTO precision under Speed="off" to "off", so no seed is wanted. An EXPLICIT
+ # scheme is still honored and upgrades the speed to `default`, so seeding it keeps that load off the
+ # released bf16 shards. Same split `_dit_prequant_plan_source` applies to a GGUF pick.
+ if auto and speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF:
+ return None
+ mode = TQ_AUTO if auto else normalize_transformer_quant(raw)
+ if mode is None:
+ return None
+ if _has_active_lora(loras):
+ return None
+ if _memory_request_forces_offload(memory_mode, cpu_offload):
+ return None
+ # SCOPED, not pinned: the pooled asyncio.to_thread thread must not be handed back set to this card.
+ with diffusion_device_scope(gpu_ordinal):
+ target = self._target_for_ordinal(fam, gpu_ordinal)
+ if not dense_transformer_supported(target):
+ return None
+ if (
+ _pipeline_quant_uncompilable_reason(target, fam, speed_mode, model_kind = kind)
+ is not None
+ ):
+ return None
+ scheme = select_transformer_quant_scheme(
+ target, mode, family = getattr(fam, "name", None)
+ )
+ if scheme is None or scheme == TQ_AUTO:
+ return None
+ # Under auto, walk the ladder below the winner: a rung whose artifact-sized plan still offloads
+ # (Qwen-Image int8 on a 32 GB card) yields to the next resident rung instead of pinning a decline.
+ # An explicit scheme is honored or refused, never swapped.
+ rungs: list[str] = [scheme]
+ if auto:
+ try:
+ from .diffusion_transformer_quant import auto_scheme_candidates
+ below = list(auto_scheme_candidates(target, getattr(fam, "name", None)))
+ if scheme in below:
+ rungs.extend(below[below.index(scheme) + 1 :])
+ except Exception: # noqa: BLE001 -- no lower rungs is just "no retry"
+ pass
+ declined = False
+ memory = snapshot_device_memory(target)
+ for rung in rungs:
+ source = denoiser_prequant_source(
+ fam,
+ rung,
+ base_repo = base,
+ path_override = transformer_prequant_path,
+ )
+ if source is None:
+ continue
+ # The loader's checkpoint validation refuses a FORCED fp8 accumulate the artifact does not bake,
+ # but only after the plan dropped the released shards. An operator's own file is left to it.
+ if getattr(source, "kind", None) == "repo" and hosted_fast_accum_conflict(
+ rung, fast_accum
+ ):
+ continue
+ candidate = resolve_dense_quant_candidate(
+ fam = fam,
+ target = target,
+ requested = rung,
+ base_repo = base,
+ prequant_path = transformer_prequant_path,
+ force_dense = False,
+ logger = None,
+ )
+ if candidate is None or not candidate.prequant:
+ continue
+ planned = self._plan_memory(
+ target,
+ None,
+ base or repo_id or "",
+ fam,
+ memory_mode,
+ cpu_offload,
+ kind = kind,
+ repo_id = repo_id,
+ fetch_base = fetch_base,
+ transformer_resident_override_mib = candidate.steady_transformer_mib,
+ companion_override_mib = candidate.companions_mib,
+ text_encoder_override_mib = candidate.text_encoders_mib,
+ device_memory_override = replace(memory, free_mib = memory.total_mib),
+ )
+ if planned.offload_policy != OFFLOAD_NONE:
+ logger.info(
+ "diffusion.denoiser_prequant: an artifact-sized plan for %s still offloads "
+ "on this card, and offload moves the denoiser via Module.to(), so the "
+ "released shards are kept",
+ rung,
+ )
+ declined = True
+ continue
+ return rung
+ return PIPELINE_SEED_DECLINED if declined else None
+ except Exception as exc: # noqa: BLE001 -- an unanswerable probe keeps the released shards
+ logger.warning("diffusion.denoiser_prequant_plan_failed: %s", exc)
+ return None
+
def _dit_prequant_plan_source(
self,
fam: Any,
@@ -2470,15 +2836,30 @@ class DiffusionBackend:
failures_out: Optional[list] = None,
) -> Optional[tuple[str, str, int]]:
"""The hosted PRE-QUANTIZED transformer this pick loads INSTEAD of the base repo's dense
- shards, as ``(repo, filename, declared_size)``, or None when no such artifact is used.
- Those shards are already excluded for a GGUF pick, so without this the plan neither
- counts nor stages the multi-GB denoiser the load really keeps on disk: the footprint
- reads short and the file is pulled INLINE during the load, outside the manager's
- progress, cancel and disk preflight. Mirrors the prequant gates in
- ``_load_dense_quant_pipeline``."""
- if kind != "gguf" or fam is None:
+ shards, as ``(repo, filename, declared_size)``, or None. Those shards are already excluded
+ for a GGUF pick, so without this the plan neither counts nor stages the multi-GB denoiser:
+ the footprint reads short and the file is pulled INLINE during the load, outside the
+ manager's progress, cancel and disk preflight. Mirrors the gates in
+ ``_load_dense_quant_pipeline``; a ``pipeline`` pick arrives only once
+ ``_pipeline_planned_denoiser_scheme`` applied them all."""
+ if fam is None:
+ return None
+ planned = kwargs.get("_pipeline_prequant_planned") if kind == "pipeline" else None
+ if kind == "pipeline":
+ if not planned or planned == PIPELINE_SEED_DECLINED:
+ return None
+ elif kind != "gguf":
return None
try:
+ if planned:
+ with diffusion_device_scope(kwargs.get("gpu_ordinal")):
+ source = denoiser_prequant_source(
+ fam,
+ planned,
+ base_repo = kwargs.get("base_repo"),
+ path_override = kwargs.get("transformer_prequant_path"),
+ )
+ return self._prequant_source_hub_entry(source, hf_token)
raw = kwargs.get("transformer_quant")
auto = raw is None or str(raw).strip().lower() in ("", "auto")
# An AUTO quant under an explicit Speed="off" is forced to "off" by load_pipeline, which normalizes to
@@ -2547,31 +2928,7 @@ class DiffusionBackend:
path_override = kwargs.get("transformer_prequant_path"),
base_repo = kwargs.get("base_repo"),
)
- if source is None or source.kind != "repo":
- return None
- from huggingface_hub import HfApi
-
- info = HfApi(token = hf_token or None).model_info(
- source.location, files_metadata = True
- )
- sizes = {
- s.rfilename: int(getattr(s, "size", 0) or 0) for s in (info.siblings or [])
- }
- # Primary name first, then the legacy one, in the order the loader tries them.
- for name in (source.filename, source.fallback_filename):
- if name and name in sizes:
- return (source.location, name, int(sizes[name]))
- # The repo answered and holds NEITHER name. Not "no prequant is used" -- this pick
- # is configured to use one and the dense shards are already excluded for it, so the
- # plan names no transformer source at all and must say it is partial.
- if failures_out is not None:
- failures_out.append(
- RuntimeError(
- f"prequant artifact missing from {source.location}: "
- f"{source.filename!r} / {source.fallback_filename!r}"
- )
- )
- return None
+ return self._prequant_source_hub_entry(source, hf_token, failures_out)
except Exception as exc: # noqa: BLE001 -- an unsizable prequant must not fail the plan
logger.warning("diffusion.dit_prequant_plan_failed: %s", exc)
# Best-effort for the UI, but NOT for a caller that must not download afterwards: the
@@ -2582,6 +2939,34 @@ class DiffusionBackend:
failures_out.append(exc)
return None
+ @staticmethod
+ def _prequant_source_hub_entry(
+ source: Any,
+ hf_token: Optional[str],
+ failures_out: Optional[list] = None,
+ ) -> Optional[tuple[str, str, int]]:
+ """``(repo, filename, declared_size)`` for a hosted checkpoint that exists, else None."""
+ if source is None or getattr(source, "kind", None) != "repo":
+ return None
+ from huggingface_hub import HfApi
+
+ info = HfApi(token = hf_token or None).model_info(source.location, files_metadata = True)
+ sizes = {s.rfilename: int(getattr(s, "size", 0) or 0) for s in (info.siblings or [])}
+ # Primary name first, then the legacy one, in the order the loader tries them.
+ for name in (source.filename, source.fallback_filename):
+ if name and name in sizes:
+ return (source.location, name, int(sizes[name]))
+ # The repo answered and holds NEITHER name. Not "no prequant is used": this pick is configured to
+ # use one and its dense shards are already excluded, so the plan has no transformer and is partial.
+ if failures_out is not None:
+ failures_out.append(
+ RuntimeError(
+ f"prequant artifact missing from {source.location}: "
+ f"{source.filename!r} / {source.fallback_filename!r}"
+ )
+ )
+ return None
+
@staticmethod
def _estimate_download_bytes(
repo_id: str,
@@ -2599,39 +2984,37 @@ class DiffusionBackend:
revisions_out: Optional[dict[str, str]] = None,
fetch_repos_out: Optional[dict[str, str]] = None,
skip_te_components: tuple[str, ...] = (),
+ skip_transformer_weights: bool = False,
+ skipped_files_out: Optional[list[str]] = None,
failures_out: Optional[list] = None,
local_files_only: bool = False,
) -> tuple[int, list[str]]:
"""Total download size for the progress bar, plus the base-repo files to fetch (the prefetch
reuses this list, so the base is listed only once).
- ``sizes_out``, when given, is filled with per-repo byte totals so the download plan can size
- one job per repo off this same single pair of Hub lookups. ``resident_file_sizes_out``
- receives only the selected pipeline components and default weight variant, which is the set
- that ``from_pretrained`` materialises in memory. ``revisions_out`` records the commit each
- lookup described, so a cache probe can ask about the SAME revision the sizes came from.
- ``fetch_repos_out`` records the repo that supplied pipeline metadata, so staging reads that
- same pinned file set instead of making a second mirror choice.
-
- For a ``pipeline`` load the whole repo IS the pipeline, so the transformer/ subfolder is
- INCLUDED, unlike the GGUF / single-file paths where the transformer is the single file. For
- a ``single_file_is_pipeline`` family (SDXL) the base repo supplies only config/tokenizer and
- its weight files are skipped.
-
- ``include_transformer`` may be a CALLABLE ``(companions, transformer_files) -> bool``,
- called once with this repo's actual listing split either side of ``transformer/``: the
- widening decision turns on what those two sets say about the cache and about which repo the
- fetch will resolve to, and that listing exists only here.
-
- ``skip_te_components`` names the text encoders this pick loads PRE-CAST from a hosted
- checkpoint, so their dense weight shards are not counted or fetched (staging the dense
- encoder for a pre-cast load wastes tens of GB and nothing ever opens them). Everything else
- in the component folder is kept: the pre-cast loader meta-inits the encoder from the base
- repo's config.
-
- ``local_files_only`` returns the metadata-unavailable answer, ``(0, [])``, instead of
- asking. Every byte this counts belongs to a download that is not permitted, and the empty
- file list is what makes the prefetch stage nothing.
+ ``sizes_out`` takes per-repo byte totals, so the download plan can size one job per repo off
+ this same pair of Hub lookups. ``resident_file_sizes_out`` takes only the selected pipeline
+ components and default weight variant, the set ``from_pretrained`` materialises in memory.
+ ``revisions_out`` records the commit each lookup described, so a cache probe can ask about
+ the SAME revision. ``fetch_repos_out`` records the repo that supplied pipeline metadata, so
+ staging reads that pinned file set rather than making a second mirror choice.
+
+ For a ``pipeline`` load the whole repo IS the pipeline, so transformer/ is INCLUDED, unlike
+ the GGUF / single-file paths. For a ``single_file_is_pipeline`` family (SDXL) the base repo
+ supplies only config/tokenizer and its weight files are skipped.
+
+ ``include_transformer`` may be a CALLABLE ``(companions, transformer_files) -> bool``, called
+ once with this repo's listing split either side of ``transformer/``: the widening decision
+ turns on what those sets say about the cache and the fetch repo, and that listing lives here.
+
+ ``skip_te_components`` names the text encoders this pick loads PRE-CAST, so their dense
+ shards are not counted or fetched (tens of wasted GB nothing ever opens). The rest of the
+ component folder is kept: the pre-cast loader meta-inits from the base repo's config.
+ ``skip_transformer_weights`` does the same for a seeded pick's DENOISER, keeping
+ ``transformer/config.json`` for the same reason.
+
+ ``local_files_only`` returns the metadata-unavailable answer ``(0, [])`` instead of asking:
+ the bytes belong to a download that is not permitted, and the empty list stages nothing.
"""
if local_files_only:
return 0, []
@@ -2650,6 +3033,11 @@ class DiffusionBackend:
rfilename, skip_te_components
)
+ def _seeded_denoiser_shard(rfilename: str) -> bool:
+ return skip_transformer_weights and is_prequant_covered_weight(
+ rfilename, (DENOISER_COMPONENT,)
+ )
+
try:
if kind == "pipeline":
@@ -2683,6 +3071,12 @@ class DiffusionBackend:
and s.rfilename.rsplit("/", 1)[0] in st_dirs
)
]
+ # Split off AFTER the variant preference, else a .bin twin fetches weights nothing opens.
+ seeded = [s for s in picked if _seeded_denoiser_shard(s.rfilename)]
+ if seeded:
+ picked = [s for s in picked if not _seeded_denoiser_shard(s.rfilename)]
+ if skipped_files_out is not None:
+ skipped_files_out[:] = [s.rfilename for s in seeded]
return info, components, picked
metadata_repo = prefer_ungated_mirror(
@@ -2838,12 +3232,49 @@ class DiffusionBackend:
if allow_device_probe
else {}
)
+ # Settled before the listing: only a checkpoint that resolves on the Hub may drop the released shards.
+ pipeline_planned = (
+ self._pipeline_planned_denoiser_scheme(
+ fam,
+ base = base,
+ kind = kind,
+ transformer_quant = load_kwargs.get("transformer_quant"),
+ speed_mode = load_kwargs.get("speed_mode"),
+ memory_mode = load_kwargs.get("memory_mode"),
+ cpu_offload = bool(load_kwargs.get("cpu_offload")),
+ loras = load_kwargs.get("loras"),
+ transformer_prequant_path = load_kwargs.get("transformer_prequant_path"),
+ gpu_ordinal = load_kwargs.get("gpu_ordinal"),
+ repo_id = repo_id,
+ fast_accum = load_kwargs.get("transformer_quant_fast_accum"),
+ )
+ if allow_device_probe
+ else None
+ )
+ # Declared before the prequant lookup so a repo holding neither artifact name records the miss
+ # HERE: the dense shards are already dropped, so the plan must call itself partial.
+ plan_failures: list = []
+ dit_prequant = (
+ self._dit_prequant_plan_source(
+ fam,
+ kind,
+ hf_token,
+ {
+ **load_kwargs,
+ "base_repo": base,
+ "_pipeline_prequant_planned": pipeline_planned,
+ },
+ plan_failures,
+ )
+ if allow_device_probe
+ else None
+ )
+ skip_transformer_weights = kind == "pipeline" and dit_prequant is not None
sizes: dict[str, int] = {}
file_sizes: dict[str, dict[str, int]] = {}
resident_file_sizes: list[tuple[str, int]] = []
revisions: dict[str, str] = {}
fetch_repos: dict[str, str] = {}
- plan_failures: list = []
required_total, base_files = self._estimate_download_bytes(
repo_id,
gguf_filename,
@@ -2874,6 +3305,7 @@ class DiffusionBackend:
revisions_out = revisions,
fetch_repos_out = fetch_repos,
skip_te_components = tuple(te_files),
+ skip_transformer_weights = skip_transformer_weights,
failures_out = plan_failures,
)
# Decided once, from the staged file list, and both probed and reported: a gated base answers model_info
@@ -2892,18 +3324,9 @@ class DiffusionBackend:
)
te_prequant_bytes = _prequant_plan_bytes(te_files)
required_total += te_prequant_bytes
- # The dense transformer/ shards are excluded for a GGUF pick, so a hosted prequant that replaces them is real
- # footprint the plan would otherwise never report. Sized against the RESOLVED base, as the load passes it: a
- # variant base picks its own prequant repo.
- dit_prequant = (
- self._dit_prequant_plan_source(
- fam, kind, hf_token, {**load_kwargs, "base_repo": base}, plan_failures
- )
- if allow_device_probe
- else None
- )
- if dit_prequant is not None:
- required_total += dit_prequant[2]
+ # The prequant replacing the dropped transformer/ shards is footprint nothing else reports.
+ dit_prequant_bytes = int(dit_prequant[2]) if dit_prequant is not None else 0
+ required_total += dit_prequant_bytes
scoped_files: dict[str, list[str]] = {}
scoped_gguf: dict[str, Optional[str]] = {}
@@ -3036,10 +3459,7 @@ class DiffusionBackend:
checkpoint = kind == "pipeline",
)
if dit_prequant is not None:
- # Staged, not just counted: _load_dense_quant_pipeline fetches this checkpoint during the load, under the
- # load lock and after the previous pipeline was evicted, so leaving it out means a multi-GB inline pull
- # with no progress, no cancel and no disk preflight. A companion of the checkpoint, never the selected
- # model itself.
+ # Staged, not just counted: leaving it out means a multi-GB inline pull under the load lock.
prequant_repo, prequant_file, prequant_size = dit_prequant
add_missing_entry(prequant_repo, [prequant_file], {prequant_file: prequant_size})
if incompatible is None and allow_device_probe and memory_verdict:
@@ -3049,7 +3469,10 @@ class DiffusionBackend:
base,
kind = kind,
declared_files = resident_file_sizes,
- prequant_bytes = te_prequant_bytes,
+ # The artifact carries the whole denoiser share, already stored at the precision it loads at.
+ prequant_bytes = te_prequant_bytes + dit_prequant_bytes
+ if skip_transformer_weights
+ else te_prequant_bytes,
extra_bf16_bytes = _predownload_encoder_bf16_bytes(
fam, te_files, pipeline_declared = bool(resident_file_sizes)
),
@@ -3279,6 +3702,23 @@ class DiffusionBackend:
snapshot = repo_dir / "snapshots" / rev
return snapshot if snapshot.is_dir() else None
+ @staticmethod
+ def _cache_file_bytes(repo_id: str, filename: str) -> int:
+ """On-disk bytes of ONE file of ``repo_id``, across cache roots, 0 when it is not there.
+
+ The largest copy, not the sum: the same file present in two roots is one logical file, the
+ rule ``_cache_bytes`` dedupes by snapshot-relative path for."""
+ best = 0
+ for repo_dir in DiffusionBackend._hub_cache_repo_dirs(repo_id):
+ snapshot = DiffusionBackend._live_snapshot_dir(repo_dir)
+ if snapshot is None:
+ continue
+ try:
+ best = max(best, (snapshot / filename).stat().st_size)
+ except OSError:
+ continue # not in this root / broken symlink / unreadable
+ return best
+
@staticmethod
def _cache_bytes(repo_id: str) -> int:
"""Bytes of ``repo_id`` on disk across every cache root, for progress and the pipeline plan.
@@ -3515,6 +3955,9 @@ class DiffusionBackend:
# The repo the background load staged the companions from; re-derived below for a direct call, which has no
# prefetch phase.
_fetch_base: Optional[str] = None,
+ # The scheme the plan settled, or PIPELINE_SEED_DECLINED; None for a direct call, which the pull never scoped.
+ _pipeline_prequant_planned: Optional[str] = None,
+ _pipeline_prequant_skipped: tuple[str, ...] = (),
) -> dict[str, Any]:
with self._load_cancel_lock:
if _load_token is None:
@@ -3631,6 +4074,48 @@ class DiffusionBackend:
base_local_dir = _base_local_dir,
fetch_base = fetch_base,
)
+ pipeline_seed_scheme: Optional[str] = None
+ bf16_pipeline_plan = plan
+ if kind == "pipeline" and _pipeline_prequant_planned not in (
+ None,
+ PIPELINE_SEED_DECLINED,
+ ):
+ pipeline_seed_scheme = _pipeline_prequant_planned
+ seed_estimate = estimate_dense_quant(
+ fam, pipeline_seed_scheme, base_repo = base, prequant_available = True
+ )
+ seeded_plan = (
+ self._plan_memory(
+ target,
+ single_file_path,
+ base,
+ fam,
+ memory_mode,
+ cpu_offload,
+ kind = kind,
+ repo_id = repo_id,
+ base_local_dir = _base_local_dir,
+ fetch_base = fetch_base,
+ transformer_resident_override_mib = (
+ seed_estimate.steady_transformer_mib
+ ),
+ companion_override_mib = seed_estimate.companions_mib,
+ text_encoder_override_mib = seed_estimate.text_encoders_mib,
+ )
+ if seed_estimate is not None
+ else None
+ )
+ if seeded_plan is None or seeded_plan.offload_policy != OFFLOAD_NONE:
+ # Offload hooks use Module.to(), which torchao tensors reject, and live free
+ # memory can undercut the CAPACITY the plan settled this against.
+ logger.info(
+ "diffusion.denoiser_prequant: an artifact-sized plan for %s offloads on "
+ "this card, so the released denoiser is loaded instead",
+ pipeline_seed_scheme,
+ )
+ pipeline_seed_scheme = None
+ else:
+ plan = seeded_plan
# On unified memory the plan above is final (the quant re-plans below are CUDA-only) and its 'none'
# policy is a placement, not a fit. Refuse here, after the eviction above freed the previous pipeline
# and before any weight is materialised. A pipeline's weight term is cached SHARD bytes, which is a
@@ -3670,16 +4155,14 @@ class DiffusionBackend:
pipe = transformer = None
pipe_kwargs: dict[str, Any] = {}
transformer_quant_engaged = None
+ transformer_quant_artifact: Optional[str] = None
quant_plan = None
# Why the dense quant did not engage, in the caller's terms. Threaded into `resolved` so a fallback is
# visible, and into the refusal so it is actionable.
transformer_quant_decline: Optional[str] = None
transformer_quant_decline_status = RESOLVED_FELL_BACK
- if transformer_quant_pinned is not None and kind != "gguf":
- transformer_quant_decline = (
- f"the dense transformer-quant path applies to GGUF picks only, and this is "
- f"a '{kind}' load, which runs the precision its checkpoint carries"
- )
+ if transformer_quant_pinned is not None and not dense_quant_supported_kind(kind):
+ transformer_quant_decline = dense_quant_unsupported_kind_reason(kind)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
elif transformer_quant_pinned is not None and not dense_transformer_supported(
target
@@ -4253,15 +4736,8 @@ class DiffusionBackend:
"adapters, free VRAM, or pick a smaller model."
)
- # Fail closed on a declined EXPLICIT precision. Loading the GGUF here produced a perfectly good image
- # at a precision the caller never asked for, and nothing in the response said so, which is why a
- # successful render could not be taken as proof the requested precision ran. `auto` is untouched:
- # falling down the ladder is what it asks for.
- if (
- pipe is None
- and transformer_quant_pinned is not None
- and not precision_fallback_allowed()
- ):
+ # GGUF reaches its verdict here; pipeline quantisation is attempted after assembly.
+ def _refuse_pinned_precision() -> None:
raise RuntimeError(
precision_refusal_message(
"transformer_quant",
@@ -4272,6 +4748,17 @@ class DiffusionBackend:
)
)
+ if (
+ transformer_quant_pinned is not None
+ and not precision_fallback_allowed()
+ # Refuse known failures before downloading a pipeline.
+ and (
+ transformer_quant_decline is not None
+ or (pipe is None and kind == "gguf")
+ )
+ ):
+ _refuse_pinned_precision()
+
if pipe is None:
if kind == "pipeline":
if fam.name == KREA2_FAMILY_NAME:
@@ -4346,6 +4833,84 @@ class DiffusionBackend:
local_files_only = local_files_only,
)
)
+ if pipeline_seed_scheme is not None:
+ from .diffusion_denoiser_prequant import (
+ denoiser_prequant_pipe_kwargs,
+ )
+
+ seeded = denoiser_prequant_pipe_kwargs(
+ fam,
+ fetch_base,
+ scheme = pipeline_seed_scheme,
+ dtype = dtype,
+ device = device,
+ hf_token = hf_token,
+ target = target,
+ path_override = transformer_prequant_path,
+ fast_accum = transformer_quant_fast_accum,
+ local_files_only = local_files_only,
+ cache_dir = hub_cache_dir(),
+ logger = logger,
+ )
+ pipe_kwargs.update(seeded)
+ if seeded:
+ transformer_quant_engaged = pipeline_seed_scheme
+ transformer_quant_artifact = prequant_artifact_label(
+ denoiser_prequant_source(
+ fam,
+ pipeline_seed_scheme,
+ base_repo = base,
+ path_override = transformer_prequant_path,
+ ),
+ next(iter(seeded.values()), None),
+ )
+ else:
+ # The plan was priced on the seed landing, so re-plan at bf16 without it.
+ logger.warning(
+ "diffusion.denoiser_prequant: no pre-quantized denoiser was "
+ "seeded; re-planning memory at the released bf16 size"
+ )
+ pipeline_seed_scheme = None
+ plan = bf16_pipeline_plan
+ raise_on_unified_memory_shortfall(
+ self._resident_sized_plan(
+ plan,
+ fam,
+ base,
+ target,
+ kind,
+ text_encoder_quant = text_encoder_quant,
+ ),
+ family = getattr(fam, "name", None),
+ logger = logger,
+ )
+ if pipeline_seed_scheme is None and _pipeline_prequant_skipped:
+ # from_pretrained cannot re-fetch these from a local snapshot dir, so top them up here.
+ _base_local_dir = self._restore_skipped_transformer_shards(
+ repo_id,
+ base,
+ _pipeline_prequant_skipped,
+ hf_token,
+ fetch_base = fetch_base,
+ base_local_dir = _base_local_dir,
+ local_files_only = local_files_only,
+ )
+ # A pipeline plan prices CACHED bytes, and the plan above saw companions only
+ # (transformer/ was skipped). Re-plan now the dense shards are back, else an
+ # under-counted 'none' keeps the bf16 denoiser resident.
+ plan = self._plan_memory(
+ target,
+ single_file_path,
+ base,
+ fam,
+ memory_mode,
+ cpu_offload,
+ kind = kind,
+ repo_id = repo_id,
+ base_local_dir = _base_local_dir,
+ fetch_base = fetch_base,
+ )
+ bf16_pipeline_plan = plan
self._raise_if_load_cancelled(_load_token)
# The prefetched snapshot dir keeps from_pretrained off the hub (24 GB per FLUX.1
# otherwise)
@@ -4457,6 +5022,176 @@ class DiffusionBackend:
_base_local_dir or fetch_base, **pipe_kwargs
)
+ # The same helper the route preflight asked, so the two cannot disagree.
+ pipeline_quant_uncompilable = _pipeline_quant_uncompilable_reason(
+ target, fam, speed_mode, model_kind = kind
+ )
+
+ # Quantise dense bf16 pipeline denoisers in place. The blocker excludes UNet and pre-quantised
+ # pipelines; offloaded plans stay dense because torchao tensors cannot move. The pipeline is still
+ # on the CPU here, unlike the GGUF path, which quantises after _assemble_pipe places it. Both
+ # orders give bit-identical output: apply_memory_plan's one-shot `pipe.to(placement)` is survived
+ # by the subclasses (measured on sm_89, fp8 and int8, max|diff| 0.0); only the per-forward offload
+ # hooks are not.
+ if (
+ pipe is not None
+ and kind == "pipeline"
+ and transformer_quant_engaged is None
+ and normalize_transformer_quant(transformer_quant) is not None
+ and dense_transformer_supported(target)
+ ):
+ # from_pretrained widens a raw fp8/int8 checkpoint to bf16, erasing the one thing the blocker
+ # below reads, so recover it from the shard header (Ideogram's loader stamps its own).
+ # `fetch_base` too: a LOCAL diffusers directory stages nothing, so `_base_local_dir` is None
+ # exactly where a hand-converted fp8 checkpoint is likeliest. Non-directories are ignored.
+ source_precision = stored_denoiser_precision(_base_local_dir or fetch_base)
+ if source_precision is not None:
+ for _attr, denoiser in denoiser_modules(pipe):
+ mark_source_precision(denoiser, source_precision)
+ pipeline_quant_blocker = pipeline_quant_uncompilable or dense_quant_blocker(
+ pipe
+ )
+ if pipeline_quant_blocker is not None:
+ logger.info(
+ "diffusion.transformer_quant: skipped (%s)", pipeline_quant_blocker
+ )
+ transformer_quant_decline = pipeline_quant_blocker
+ transformer_quant_decline_status = RESOLVED_UNSUPPORTED
+ else:
+ # Re-plan against the quantised steady size. The build peak remains bf16.
+ bf16_plan = plan
+ if plan.offload_policy != OFFLOAD_NONE:
+ preview_scheme = select_transformer_quant_scheme(
+ target, transformer_quant, family = getattr(fam, "name", None)
+ )
+ # This in-memory rewrite needs no cache-space or hosted-checkpoint checks.
+ estimate = (
+ estimate_dense_quant(fam, preview_scheme, base_repo = base)
+ if preview_scheme is not None
+ else None
+ )
+ if estimate is not None:
+ replanned = self._plan_memory(
+ target,
+ single_file_path,
+ base,
+ fam,
+ memory_mode,
+ cpu_offload,
+ kind = kind,
+ repo_id = repo_id,
+ fetch_base = fetch_base,
+ base_local_dir = _base_local_dir,
+ transformer_resident_override_mib = (
+ estimate.steady_transformer_mib
+ ),
+ companion_override_mib = estimate.companions_mib,
+ text_encoder_override_mib = estimate.text_encoders_mib,
+ )
+ if replanned.offload_policy == OFFLOAD_NONE:
+ logger.info(
+ "diffusion.transformer_quant: %s fits resident (%d MiB "
+ "steady); dropping the bf16 plan's '%s' offload",
+ preview_scheme,
+ estimate.steady_transformer_mib,
+ plan.offload_policy,
+ )
+ plan = replanned
+ if plan.offload_policy != OFFLOAD_NONE:
+ logger.info(
+ "diffusion.transformer_quant: skipped (the memory plan picked '%s' "
+ "offload, which moves the transformer via Module.to())",
+ plan.offload_policy,
+ )
+ transformer_quant_decline = (
+ f"the memory plan picked '{plan.offload_policy}' offload, which moves "
+ "the transformer via Module.to(); torchao quantised tensors reject "
+ "that. Pin a resident memory mode to combine the two"
+ )
+ else:
+ if _has_active_lora(loras):
+ # PEFT must wrap dense Linears before torchao converts their base layers.
+ baked = self._resolve_lora_set(
+ [(i, w) for (i, w) in loras if w != 0],
+ family = getattr(fam, "name", None),
+ hf_token = hf_token,
+ )
+ for name, path, _weight in baked:
+ pipe.load_lora_weights(path, adapter_name = name)
+ pipe.set_adapters(
+ [n for (n, _p, _w) in baked],
+ adapter_weights = [w for (_n, _p, w) in baked],
+ )
+ pipe._unsloth_loras = baked
+ pipe._unsloth_loras_baked = True
+ logger.info(
+ "diffusion.lora_bake: %d adapter(s) attached before the pipeline "
+ "quantize",
+ len(baked),
+ )
+ # Convert every denoiser so multi-branch pipelines use one precision.
+ denoisers = denoiser_modules(pipe)
+ engaged: list[str] = []
+ for attr, _module in denoisers:
+ scheme = quantize_transformer(
+ pipe
+ if attr == "transformer"
+ else _DenoiserView(pipe, attr),
+ target,
+ mode = transformer_quant,
+ family = getattr(fam, "name", None),
+ fast_accum = transformer_quant_fast_accum,
+ logger = logger,
+ )
+ if scheme is None:
+ break
+ engaged.append(scheme)
+ if engaged and len(engaged) == len(denoisers):
+ transformer_quant_engaged = engaged[0]
+ else:
+ # A clean decline can remain bf16; a partial in-place conversion is unusable.
+ dirty = [
+ attr
+ for attr, module in denoisers
+ if transformer_is_quantised(module)
+ ]
+ if dirty:
+ # Rebound, not deleted: the enclosing finally reads `pipe`.
+ pipe = None
+ try:
+ clear_gpu_cache()
+ except Exception: # noqa: BLE001
+ pass
+ raise RuntimeError(
+ f"transformer_quant='{transformer_quant}' converted "
+ f"{', '.join(dirty)} and then failed, leaving the "
+ "transformer neither dense nor usable. Reload with "
+ "Precision set to Off to run the checkpoint as-is."
+ )
+ transformer_quant_decline = (
+ f"'{transformer_quant}' did not engage on family "
+ f"'{getattr(fam, 'name', None)}' with this GPU (the scheme is "
+ "unsupported here, or the family's measured deny list rules it "
+ "out); see the server log"
+ )
+ transformer_quant_decline_status = RESOLVED_UNSUPPORTED
+ if plan is not bf16_plan and transformer_quant_engaged is None:
+ # The quant-sized placement was only ever valid for the quantised build.
+ plan = bf16_plan
+ # Pinned schemes fail closed, including blocker declines that skipped conversion.
+ if (
+ transformer_quant_engaged is None
+ and transformer_quant_pinned is not None
+ and not precision_fallback_allowed()
+ ):
+ # Rebound, not deleted: the enclosing finally reads `pipe`.
+ pipe = None
+ try:
+ clear_gpu_cache()
+ except Exception: # noqa: BLE001 -- the refusal matters more than the sweep
+ pass
+ _refuse_pinned_precision()
+
self._raise_if_load_cancelled(_load_token)
# Lazy import (these modules import torch) keeps diffusion.py torch-free to import.
from .diffusion_eager_patches import (
@@ -4708,6 +5443,9 @@ class DiffusionBackend:
)
)
if transformer_quant_engaged is None
+ else f"seeded from the hosted checkpoint "
+ f"{transformer_quant_artifact.split(':', 1)[1]}"
+ if transformer_quant_artifact is not None
else "re-planned resident for the quantised artifact"
if quant_plan is not None
else "engaged on the dense fast path",
@@ -4770,6 +5508,9 @@ class DiffusionBackend:
),
}
)
+ if transformer_quant_artifact is not None:
+ # Beside ``source``, never in it: the frontend branches on "auto"/"explicit".
+ resolved["transformer_quant"]["artifact"] = transformer_quant_artifact
state = _LoadState(
pipe = pipe,
@@ -5318,37 +6059,51 @@ class DiffusionBackend:
text_encoder_override_mib: Optional[int] = None,
base_local_dir: Optional[str] = None,
fetch_base: Optional[str] = None,
+ device_memory_override: Optional[DeviceMemory] = None,
):
"""Build the memory plan for this load: snapshot free device memory and estimate the model's
resident footprint, then let the planner pick an offload policy + VAE memory savers. Kept on
the backend so the cached base repo (companion text-encoder / VAE) feeds the size estimate.
The size estimate is per-kind: diffusers keeps GGUF weights packed (per-matmul transient
- dequant), so a GGUF loads near its on-disk size; a safetensors single-file loads near its
- on-disk size, except an fp8 transformer file that gets upcast to bf16 on load (~2x
- resident); and a full pipeline is one cached download, already compressed.
-
- ``transformer_resident_override_mib`` replaces the file-size transformer estimate when the
- loader is planning for a DIFFERENT artifact than the file on disk (the dense
- transformer-quant candidate); ``companion_override_mib`` likewise replaces the cached
- companion total on that re-plan, so the base repo's PREFETCHED transformer/ shards are not
- counted as companions on top of it; ``text_encoder_override_mib`` carries that override's
- TEXT-ENCODER share, which the planner needs to price the streamed-text-encoder group tier.
- All come from the same family component table.
-
- ``base_local_dir`` is the snapshot the load will actually read, carried into the size
- lookups as an extra source alongside the cache roots: it is additive and never a
- replacement, since under-counting here leaves an auto plan resident and OOMing on weights it
- never budgeted.
-
- ``fetch_base`` is the repo the bytes were staged from, so every cache scan below reads it:
- sizing an upstream id whose cache is empty folds the VAE/text-encoder to zero and wrongly
- picks resident placement. ``base`` and ``repo_id`` keep the upstream identity for the
+ dequant), so a GGUF loads near its on-disk size; so does a safetensors single-file, except an
+ fp8 transformer upcast to bf16 on load (~2x resident); a full pipeline is one cached
+ download, already compressed.
+
+ The three ``*_override_mib`` arguments re-price a plan against a DIFFERENT artifact than the
+ file on disk (the dense transformer-quant candidate): ``transformer_resident_override_mib``
+ replaces the file-size transformer estimate, ``companion_override_mib`` the cached companion
+ total, so the base repo's prefetched transformer/ shards are not counted twice, and
+ ``text_encoder_override_mib`` carries that override's TEXT-ENCODER share, which prices the
+ streamed-text-encoder group tier. All come from the family component table and apply to
+ every kind.
+
+ ``base_local_dir`` is the snapshot the load will read, added to the size lookups alongside
+ the cache roots. Additive, never a replacement: under-counting leaves an auto plan resident
+ and OOMing on weights it never budgeted.
+
+ ``fetch_base`` is the repo the bytes were staged from, so every cache scan reads it: sizing
+ an upstream id whose cache is empty folds the VAE/text-encoder to zero and wrongly picks
+ resident placement. ``base`` and ``repo_id`` keep the upstream identity for the
family/variant checks.
+
+ ``device_memory_override`` replaces the live reading for a plan taken BEFORE the download,
+ where free memory still describes the OLD model; capacity bounds any later free reading.
"""
# Settled (max-over-reads) on cuda: a transient foreign allocation would make an empty card look full
- device_memory = settled_snapshot_device_memory(target)
- if kind == "pipeline":
+ device_memory = (
+ device_memory_override
+ if device_memory_override is not None
+ else settled_snapshot_device_memory(target)
+ )
+ if kind == "pipeline" and transformer_resident_override_mib is not None:
+ # Re-planning an assembled pipeline against its dense-quant candidate. The family estimate already
+ # splits transformer from companions; the cache scan below would price the bf16 transformer this
+ # re-plan replaces, sizing the candidate at the footprint it is meant to shrink.
+ companion_mib = companion_override_mib
+ text_encoder_mib = text_encoder_override_mib
+ model_dense_mib = transformer_resident_override_mib + (companion_mib or 0)
+ elif kind == "pipeline":
# The whole repo is one cached download, so cached bytes are the resident estimate; a LOCAL path is not
# cached, so sum its on-disk weights.
local_repo = Path(repo_id).expanduser() if repo_id else None
diff --git a/studio/backend/core/inference/diffusion_denoiser_prequant.py b/studio/backend/core/inference/diffusion_denoiser_prequant.py
new file mode 100644
index 00000000000..2855869a85e
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_denoiser_prequant.py
@@ -0,0 +1,187 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Seed an official image PIPELINE pick with its hosted pre-quantized denoiser.
+
+A ``kind == "pipeline"`` pick assembles every component from one repo, so the only way to run a
+quantised denoiser without materialising the bf16 shards is handing ``from_pretrained`` an
+already-quantised ``transformer=``. Torch-free at import."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from .diffusion_families import IDEOGRAM4_FAMILY_NAME
+from .diffusion_krea2 import KREA2_FAMILY_NAME
+
+# The planner DECIDED against the seed, unlike never having asked: the pull kept the dense shards, so
+# the loader must not re-take the decision and fetch the artifact inline.
+PIPELINE_SEED_DECLINED = "__declined__"
+
+DENOISER_COMPONENT = "transformer"
+
+# Families assembled PER COMPONENT, plus ideogram's second denoiser: their assemblers never see
+# ``pipe_kwargs``, so a seed would be dropped after the plan had dropped their dense shards.
+_UNSEEDABLE_PIPELINE_FAMILIES = (KREA2_FAMILY_NAME, IDEOGRAM4_FAMILY_NAME)
+
+
+def pipeline_seed_supported(fam: Any) -> bool:
+ """Whether a seeded denoiser reaches this family's pipeline assembly at all."""
+ name = str(getattr(fam, "name", "") or "")
+ return bool(name) and name not in _UNSEEDABLE_PIPELINE_FAMILIES
+
+
+def denoiser_prequant_source(
+ fam: Any,
+ scheme: Optional[str],
+ *,
+ base_repo: Optional[str],
+ path_override: Optional[str] = None,
+) -> Optional[Any]:
+ """The ``PrequantSource`` a pipeline pick would seed its denoiser from, or None.
+ ``usable_prequant_source``, not ``resolve_prequant_source``: an override the loader would refuse
+ must read as no source here. Never raises."""
+ wanted = (scheme or "").strip().lower()
+ if wanted in ("", "auto", "off", "none") or wanted == PIPELINE_SEED_DECLINED:
+ return None
+ try:
+ from .diffusion_prequant import usable_prequant_source
+ return usable_prequant_source(fam, wanted, path_override = path_override, base_repo = base_repo)
+ except Exception: # noqa: BLE001 -- an unanswerable registry keeps the released bf16 denoiser
+ return None
+
+
+def denoiser_prequant_cached(
+ fam: Any,
+ scheme: Optional[str],
+ *,
+ base_repo: Optional[str],
+ path_override: Optional[str] = None,
+ cache_dir: Optional[str] = None,
+) -> bool:
+ """Whether the artifact this seed would open is ALREADY on disk, answered without a Hub call.
+ The offline twin of the ``model_info`` probe the online plan makes, so a load that may not
+ download can still seed from the cache an earlier load built. Never raises."""
+ source = denoiser_prequant_source(fam, scheme, base_repo = base_repo, path_override = path_override)
+ if source is None:
+ return False
+ if getattr(source, "kind", None) != "repo":
+ # ``usable_prequant_source`` already proved a local override present and baked for this scheme.
+ return True
+ try:
+ from .diffusion_prequant import prequant_checkpoint_cached
+ return prequant_checkpoint_cached(source, cache_dir = cache_dir)
+ except Exception: # noqa: BLE001 -- an unreadable cache is not proof the artifact is there
+ return False
+
+
+def denoiser_prequant_pipe_kwargs(
+ fam: Any,
+ base_repo: str,
+ *,
+ scheme: Optional[str],
+ dtype: Any,
+ device: str,
+ hf_token: Optional[str] = None,
+ target: Any = None,
+ path_override: Optional[str] = None,
+ fast_accum: Optional[bool] = None,
+ local_files_only: bool = False,
+ cache_dir: Optional[str] = None,
+ logger: Any = None,
+) -> dict[str, Any]:
+ """``{"transformer": module}`` for pipeline assembly, or ``{}`` when it cannot be seeded, which
+ obliges the caller to re-plan at bf16 and restore the dropped shards."""
+ try:
+ if not pipeline_seed_supported(fam):
+ return {}
+ source = denoiser_prequant_source(
+ fam, scheme, base_repo = base_repo, path_override = path_override
+ )
+ if source is None:
+ return {}
+ if target is not None:
+ from .diffusion_transformer_quant import dense_transformer_supported
+ if not dense_transformer_supported(target):
+ return {}
+ import diffusers
+
+ from .diffusion_prequant import load_prequantized_transformer
+ from .diffusion_transformer_quant import DEFAULT_MIN_LINEAR_FEATURES
+
+ transformer_cls = getattr(diffusers, str(getattr(fam, "transformer_class", "")), None)
+ if transformer_cls is None:
+ _warn(
+ logger,
+ str(scheme or ""),
+ ValueError(
+ f"transformer class {getattr(fam, 'transformer_class', None)!r} "
+ "not found in diffusers"
+ ),
+ )
+ return {}
+ module = load_prequantized_transformer(
+ transformer_cls,
+ base_repo,
+ source,
+ device = device,
+ dtype = dtype,
+ hf_token = hf_token,
+ scheme = scheme,
+ min_features = DEFAULT_MIN_LINEAR_FEATURES,
+ fast_accum = fast_accum,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ logger = logger,
+ )
+ if module is None:
+ if logger is not None:
+ logger.info(
+ "diffusion.denoiser_prequant: no usable %s checkpoint for %s, so the released "
+ "denoiser is loaded and quantised at runtime instead",
+ scheme,
+ getattr(fam, "name", None),
+ )
+ return {}
+ if logger is not None:
+ logger.info(
+ "diffusion.denoiser_prequant: seeded the %s pre-quantized denoiser for %s (%s)",
+ scheme,
+ getattr(fam, "name", None),
+ source.location,
+ )
+ return {DENOISER_COMPONENT: module}
+ except Exception as exc: # noqa: BLE001 -- seeding is an optimisation, never a blocker
+ _warn(logger, "pipe_kwargs", exc)
+ return {}
+
+
+def prequant_artifact_label(source: Any, module: Any = None) -> Optional[str]:
+ """``prequant:<repo>/<file>`` for a hosted artifact, ``prequant:<path>`` for a local override;
+ the scheme alone cannot tell either from a runtime quantise. ``module`` is the seeded denoiser,
+ which carries the file that really loaded: a repo holding only ``fallback_filename`` serves
+ that one, and ``source.filename`` would then name a file nobody fetched."""
+ if source is None:
+ return None
+ kind = getattr(source, "kind", None)
+ location = getattr(source, "location", None)
+ if not location:
+ return None
+ if kind != "repo":
+ return f"prequant:{location}"
+ filename = _loaded_filename(module) or getattr(source, "filename", None)
+ return f"prequant:{location}/{filename}" if filename else f"prequant:{location}"
+
+
+def _loaded_filename(module: Any) -> Optional[str]:
+ path = getattr(module, "_unsloth_prequant_path", None)
+ if not path:
+ return None
+ import os
+
+ return os.path.basename(str(path)) or None
+
+
+def _warn(logger: Any, what: str, exc: Exception) -> None:
+ if logger is not None:
+ logger.warning("diffusion.denoiser_prequant: %s failed: %s", what, exc)
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 24716d8c7db..647245b7e98 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -304,9 +304,18 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
pipeline_class = "QwenImagePipeline",
transformer_class = "QwenImageTransformer2DModel",
base_repo = "Qwen/Qwen-Image",
- # int8 only: no fp8 DiT checkpoint is published for this family yet. fp8 is no longer denied for inference, so
- # adding one here would now be live rather than dead.
- prequant_repos = (("int8", "unsloth/Qwen-Image-FP8"),),
+ # Qwen-Image-FP8.pt was rebuilt with the activation scale floor (LPIPS 0.044 against the bf16
+ # render at 1024, seed 20260914), so an fp8 host seeds it instead of quantising bf16 in memory.
+ prequant_repos = (
+ ("int8", "unsloth/Qwen-Image-FP8"),
+ ("fp8", "unsloth/Qwen-Image-FP8"),
+ ),
+ # 2512 is a different checkpoint with its own baked artifacts; without these rows a 2512 pick
+ # plans the Qwen-Image artifact, refused by base_model_id once the shards are already dropped.
+ prequant_variant_repos = (
+ ("qwen/qwen-image-2512", "int8", "unsloth/Qwen-Image-2512-FP8"),
+ ("qwen/qwen-image-2512", "fp8", "unsloth/Qwen-Image-2512-FP8"),
+ ),
# Pre-cast Qwen2.5-VL-7B (16.6 -> 8.8 GB). Always was independent of the DiT scheme rules.
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Qwen-Image-FP8"),),
cfg_kwarg = "true_cfg_scale",
@@ -436,6 +445,10 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
("int8", "unsloth/HiDream-I1-Full-FP8"),
("fp8", "unsloth/HiDream-I1-Full-FP8"),
),
+ # Dev and Fast are distillations of Full, so the hosted Full checkpoint is baked from other
+ # weights and neither has its own artifact. Excluded rather than inherited: base_model_id
+ # refuses the Full artifact only after the plan has dropped their released shards.
+ prequant_excluded_bases = ("hidream-ai/hidream-i1-dev", "hidream-ai/hidream-i1-fast"),
# Pre-cast Llama-3.1-8B TE4 (16.1 -> 8.1 GB). The generic TE pass only covers text_encoder.._3, so TE4 engages
# via hidream_te4_kwargs.
te_prequant_repos = (("fp8", "text_encoder_4", "unsloth/HiDream-I1-Full-FP8"),),
diff --git a/studio/backend/core/inference/diffusion_ideogram4.py b/studio/backend/core/inference/diffusion_ideogram4.py
index f97e74edb06..64e90102958 100644
--- a/studio/backend/core/inference/diffusion_ideogram4.py
+++ b/studio/backend/core/inference/diffusion_ideogram4.py
@@ -37,6 +37,7 @@ from typing import Any, Callable, Optional
from loggers import get_logger
from .diffusion_krea2 import load_krea2_text_encoder, load_krea2_tokenizer
+from .diffusion_transformer_quant import mark_source_precision
logger = get_logger(__name__)
@@ -419,7 +420,8 @@ def load_ideogram4_transformer(
)
check_cancelled()
model.to(dtype)
- return model
+ # Preserve the published source precision after widening the tensors to bf16.
+ return mark_source_precision(model, "fp8")
def load_ideogram4_pipeline(
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index 2c242ba5b35..3701c1ca6d7 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -710,6 +710,12 @@ def load_prequantized_transformer(
transformer._unsloth_runtime_quant = scheme
except Exception: # noqa: BLE001 - marker is best-effort
pass
+ try:
+ # The file that really loaded (the fallback name when the primary is absent): the
+ # provenance label has no other way to know which one it was.
+ transformer._unsloth_prequant_path = path
+ except Exception: # noqa: BLE001 - marker is best-effort
+ pass
if logger is not None:
logger.info(
"diffusion.prequant: loaded %s checkpoint (%s) onto %s",
@@ -957,6 +963,20 @@ def _validate_activation_rotation(ckpt_format: Any, meta: Any, scheme: str, logg
return True
+def hosted_fast_accum_conflict(scheme: str, fast_accum: Optional[bool]) -> bool:
+ """Whether a FORCED fp8 accumulate rules out every HOSTED checkpoint for ``scheme``.
+
+ ``scripts/build_prequant_checkpoint.py`` bakes the auto choice (``_resolve_fast_accum(None)``)
+ into the fp8 artifacts, and ``_validate_checkpoint`` refuses a baked value differing from a
+ forced one. A planner that seeds without asking drops the released shards for a checkpoint the
+ load must reject. Only fp8 bakes the field, so every other scheme is False. No IO, no torch."""
+ from .diffusion_transformer_quant import TQ_FP8, _resolve_fast_accum
+
+ if fast_accum is None or scheme != TQ_FP8:
+ return False
+ return bool(fast_accum) != _resolve_fast_accum(None)
+
+
def _validate_checkpoint(
ckpt: Any,
scheme: str,
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 611c0c64031..934bb8677d5 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -4,20 +4,19 @@
"""Opt-in low-precision quantisation of the diffusion DiT transformer.
The default GGUF path stores weights 4-bit but DEQUANTISES to bf16 on every matmul, so it runs at
-bf16 tensor-core rate: a memory win that costs speed. This module is the opt-in alternative: load
-the DENSE bf16 transformer and torchao-quantise it with a DYNAMIC-ACTIVATION scheme so the matmul
-runs on low-precision tensor cores. Measured on B200 (Z-Image-Turbo, 1024px/8 steps) vs GGUF+compile
-(0.802s, LPIPS 0.083): fp8 0.585s (1.37x), int8 0.603s (1.33x), both at LOWER LPIPS than GGUF --
-faster and slightly more accurate, at a higher-memory dense load. Strictly opt-in; GGUF stays the
-low-memory default and fallback.
+bf16 tensor-core rate: a memory win that costs speed. The opt-in alternative here loads the DENSE
+bf16 transformer and torchao-quantises it with a DYNAMIC-ACTIVATION scheme, so the matmul runs on
+low-precision tensor cores. Measured on B200 (Z-Image-Turbo, 1024px/8 steps) against GGUF+compile
+(0.802s, LPIPS 0.083): fp8 0.585s (1.37x), int8 0.603s (1.33x), both at LOWER LPIPS, at a
+higher-memory dense load. GGUF stays the low-memory default and fallback.
-Scheme by architecture (``auto`` picks the best supported, best first): nvfp4 / mxfp8 for Blackwell
-sm_100+ FP4 / MX tensor cores (biggest win; prototype), fp8 for Ada / Hopper / Blackwell (sm_89+),
-int8 for Ampere+ (sm_80+), the broadest-hardware lever.
+Scheme by architecture (``auto`` takes the best supported): int8 leads every tier, then fp8 on
+Ada / Hopper / Blackwell (sm_89+) and mxfp8 on Blackwell (sm_100+); Ampere (sm_80+) has int8 alone.
+nvfp4 is explicit opt-in, outside the auto ladder. ``_AUTO_LADDER`` is what ships.
Every scheme needs ``torch.compile`` for the speedup (dynamic quant is ~30x slower eager); the
loader compiles the repeated block after this. torch / torchao imported lazily; every probe is
-best-effort: an unsupported scheme yields None and the caller loads GGUF.
+best-effort, yielding None so the caller loads GGUF.
"""
from __future__ import annotations
@@ -205,17 +204,13 @@ def exclude_tokens_for_scheme(scheme: str, family: Optional[str] = None) -> tupl
return ()
-# Per-arch preference for ``auto``, best first. On Blackwell fp8 leads: on B200 plain fp8 dynamic is faster AND more
-# accurate at DiT shapes, while mxfp8 block scaling only adds overhead. nvfp4's FP4 GEMM is real with torch>=2.11 but
-# wins only on very large GEMMs (0.81x on Z-Image 1024px, LPIPS 0.166 vs fp8 0.044), so it is kept OUT of the ladder
-# below and stays an explicit opt-in (transformer_quant="nvfp4"): auto must never silently drop to a scheme that is
-# both slower and less accurate. Restore the commented Blackwell tier to re-enable it once the FP4 tensor-core GEMM
-# wins at the DiT's real shapes (hidden ~3072, MLP ~12288, M ~4096) and its accuracy is validated by the prequant
-# gate. Consumer / workstation GPUs move int8 first: they halve fp8/fp16 FP32-accumulate.
+# Per-arch preference for ``auto``, best first. int8 leads every tier: it runs full-rate on consumer and
+# workstation cards (which halve fp8 FP32 accumulate) and is within a few percent of fp8 elsewhere.
+# nvfp4 is explicit opt-in and kept OUT of the ladder, being slower and less accurate at DiT shapes.
_AUTO_LADDER: tuple[tuple[tuple[int, int], tuple[str, ...]], ...] = (
- ((10, 0), (TQ_FP8, TQ_MXFP8, TQ_INT8)), # Blackwell sm_100+ (nvfp4 is explicit opt-in only)
- # ((10, 0), (TQ_FP8, TQ_NVFP4, TQ_MXFP8, TQ_INT8)), # restore to re-enable nvfp4 under auto
- ((8, 9), (TQ_FP8, TQ_INT8)), # Ada sm_89 / Hopper sm_90
+ ((10, 0), (TQ_INT8, TQ_FP8, TQ_MXFP8)), # Blackwell sm_100+ (nvfp4 is explicit opt-in only)
+ # ((10, 0), (TQ_INT8, TQ_FP8, TQ_NVFP4, TQ_MXFP8)), # restore to re-enable nvfp4 under auto
+ ((8, 9), (TQ_INT8, TQ_FP8)), # Ada sm_89 / Hopper sm_90
((8, 0), (TQ_INT8,)), # Ampere sm_80 / sm_86
)
@@ -395,10 +390,9 @@ _PROFESSIONAL_GPU_MARKERS = ("RTX PRO 6000", "RTX 6000 ADA")
def _is_consumer_gpu(device: Any = None) -> bool:
"""Whether the active GPU is consumer-class (GDDR), where fp8 FP32 accumulate is halved so fast
- (FP16) accumulate is a ~2x win. Data-center HBM and professional parts are not nerfed (return
- False -> precise accumulate, fp8 first). Heuristic on the device name: GeForce / TITAN ->
- consumer; a data-center token or professional marker -> not; anything else defaults to
- consumer (fast accumulate is free on data-center, a win on consumer). True on any failure."""
+ (FP16) accumulate is a ~2x win. Data-center HBM and professional parts are not nerfed and return
+ False. Heuristic on the device name, defaulting to consumer, which is free on data-center parts
+ and a win on consumer ones. True on any failure."""
try:
import re
@@ -429,6 +423,183 @@ def normalize_transformer_quant(value: Optional[str]) -> Optional[str]:
return normalized
+# GGUF substitutes dense base weights; pipeline rewrites its loaded weights in place; single-file
+# artifacts keep their stored precision. ``dense_quant_blocker`` checks pipeline eligibility later.
+DENSE_QUANT_KINDS: tuple[str, ...] = ("gguf", "pipeline")
+
+
+def dense_quant_supported_kind(model_kind: Optional[str]) -> bool:
+ """Whether ``model_kind`` can reach the dense transformer-quant path at all."""
+ return (model_kind or "").strip().lower() in DENSE_QUANT_KINDS
+
+
+def dense_quant_unsupported_kind_reason(model_kind: Optional[str]) -> str:
+ return (
+ f"the dense transformer-quant path applies to GGUF and pipeline picks, and this is a "
+ f"'{model_kind}' load, which runs the precision its checkpoint carries"
+ )
+
+
+# Both Ideogram branches must use the same precision.
+DENOISER_ATTRS: tuple[str, ...] = ("transformer", "unconditional_transformer")
+
+# Some dense DiTs retain norms or embedders in fp32. Other parameter dtypes are not dense sources.
+_DENSE_PARAM_DTYPE_NAMES = ("bfloat16", "float32")
+
+
+class DenoiserView:
+ """Expose ``pipe.<attr>`` as ``pipe.transformer`` for the quantisation helpers."""
+
+ def __init__(self, pipe: Any, attr: str) -> None:
+ object.__setattr__(self, "_pipe", pipe)
+ object.__setattr__(self, "_attr", attr)
+
+ @property
+ def transformer(self) -> Any:
+ return getattr(
+ object.__getattribute__(self, "_pipe"), object.__getattribute__(self, "_attr")
+ )
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(object.__getattribute__(self, "_pipe"), name)
+
+
+def denoiser_modules(pipe: Any) -> tuple[tuple[str, Any], ...]:
+ """The ``(attribute, module)`` denoisers present on ``pipe``, in conversion order."""
+ found = []
+ for attr in DENOISER_ATTRS:
+ module = getattr(pipe, attr, None)
+ if module is not None:
+ found.append((attr, module))
+ return tuple(found)
+
+
+# Records a quantised source that a loader widened to a dense dtype.
+SOURCE_PRECISION_ATTR = "_unsloth_source_precision"
+
+
+def mark_source_precision(module: Any, precision: str) -> Any:
+ """Record the precision from which a loader dequantised ``module``."""
+ try:
+ setattr(module, SOURCE_PRECISION_ATTR, precision)
+ except Exception: # noqa: BLE001 -- a marker must never fail the load it describes
+ pass
+ return module
+
+
+# safetensors header dtype strings narrower than bf16, mapped to the name the refusal uses. Header
+# strings, not torch dtypes: this is read before anything is instantiated.
+_NARROW_STORED_DTYPES: dict[str, str] = {
+ "F8_E4M3": "fp8",
+ "F8_E5M2": "fp8",
+ "F8_E8M0": "fp8",
+ "I8": "int8",
+ "U8": "uint8",
+ "I4": "int4",
+ "U4": "uint4",
+ "F4": "fp4",
+}
+
+
+def stored_denoiser_precision(local_dir: Optional[str]) -> Optional[str]:
+ """The narrow precision a LOCAL snapshot's denoiser shards are stored at, or None.
+
+ ``from_pretrained(torch_dtype = bfloat16)`` widens a raw fp8 checkpoint as it loads, so by the
+ time ``dense_quant_blocker`` walks parameters every tensor reads bf16 and the evidence is gone.
+ Quantising it again compounds a loss already taken. Ideogram's loader records this itself with
+ ``mark_source_precision``; every other family reaches the generic loader, where the shard header
+ is the only place left to look.
+
+ Headers only, and only under a directory we already have: no download, no tensor read, and no
+ verdict without a local snapshot, which is the behaviour without this check. A hub id is not a
+ directory and answers None, so the caller may pass the staged snapshot or the load's own base."""
+ if not local_dir:
+ return None
+ try:
+ from pathlib import Path
+
+ import safetensors
+
+ root = Path(local_dir).expanduser()
+ for attr in DENOISER_ATTRS:
+ sub = root / attr
+ if not sub.is_dir():
+ continue
+ # EVERY shard: a narrow checkpoint keeps its norms and embeddings wide (why
+ # _DENSE_PARAM_DTYPE_NAMES admits float32), and shards are cut by size, so the first
+ # file can be entirely wide while a later one holds the fp8 linears.
+ for shard in sorted(sub.glob("*.safetensors")):
+ with safetensors.safe_open(str(shard), "pt") as handle:
+ for key in handle.keys():
+ dtype = str(handle.get_slice(key).get_dtype()).upper()
+ narrow = _NARROW_STORED_DTYPES.get(dtype)
+ if narrow is not None:
+ return narrow
+ except Exception: # noqa: BLE001 -- an unreadable header is not evidence of anything
+ return None
+ return None
+
+
+def _module_quantised_marker(module: Any) -> Optional[str]:
+ """Explain why ``module`` is not a dense bf16 source, or return None."""
+ if getattr(module, "_unsloth_runtime_quant", None):
+ return "it is already quantised by this loader"
+ source = getattr(module, SOURCE_PRECISION_ATTR, None)
+ if source:
+ return (
+ f"its published weights are {source} and were widened to bf16 on load, so quantising "
+ "them again would compound that loss"
+ )
+ for flag, what in (("is_loaded_in_4bit", "4-bit"), ("is_loaded_in_8bit", "8-bit")):
+ if getattr(module, flag, False):
+ return f"it was loaded {what} (bitsandbytes)"
+ config = getattr(module, "config", None)
+ if getattr(config, "quantization_config", None) is not None:
+ return "its checkpoint declares a quantization_config"
+ try:
+ for param in module.parameters(recurse = True):
+ name = str(getattr(param, "dtype", "")).rsplit(".", 1)[-1]
+ if name not in _DENSE_PARAM_DTYPE_NAMES:
+ return f"its weights are stored as {name}, not dense bfloat16"
+ except Exception: # noqa: BLE001 -- an unwalkable module is not evidence of anything
+ return None
+ return None
+
+
+def dense_quant_blocker(pipe: Any) -> Optional[str]:
+ """Explain why a pipeline cannot be quantised in place, or return None."""
+ denoisers = denoiser_modules(pipe)
+ if not denoisers:
+ return (
+ "this pipeline has no transformer to quantise (its denoiser is a UNet, which the "
+ "dense torchao schemes do not cover)"
+ )
+ for attr, module in denoisers:
+ marker = _module_quantised_marker(module)
+ if marker is not None:
+ where = "its transformer" if attr == "transformer" else f"its {attr}"
+ return f"{where} is not a dense bf16 source: {marker}"
+ return None
+
+
+def transformer_is_quantised(module: Any) -> bool:
+ """Whether any Linear weight has been replaced by a torchao tensor subclass."""
+ try:
+ import torch
+ for sub in module.modules():
+ if not isinstance(sub, torch.nn.Linear):
+ continue
+ weight = getattr(sub, "weight", None)
+ data = getattr(weight, "data", None)
+ if data is not None and type(data) is not torch.Tensor:
+ return True
+ if weight is not None and type(weight) not in (torch.nn.Parameter, torch.Tensor):
+ return True
+ except Exception: # noqa: BLE001 -- a probe must never replace the failure it is describing
+ return False
+ return False
+
+
def dense_transformer_supported(target: Any) -> bool:
"""Whether the dense-source quant path is usable for ``target``: a CUDA device with bf16 dtype
(the only config any torchao dynamic scheme accelerates). Cheap loader pre-check."""
@@ -496,7 +667,7 @@ def select_transformer_quant_scheme(
return None
for floor, schemes in _AUTO_LADDER:
if cap >= floor:
- for scheme in _prefer_consumer_scheme(schemes, device):
+ for scheme in schemes:
if _family_denied(family, scheme):
continue
if _scheme_supported(scheme, device):
@@ -505,6 +676,43 @@ def select_transformer_quant_scheme(
return None
+def dense_quant_host_capable(target: Any) -> bool:
+ """Whether an ``auto`` request could engage a dense scheme on this host.
+
+ One bit, because that is all the picker can honestly act on. A load's actual scheme depends on
+ the request (precision, speed, memory), the family deny list and the smoke probe, none of which
+ belong on an unclicked row; ``resolved`` reports what ran once there is a load.
+
+ Cheap and non-allocating, so a polled status route can ask it: the arch floors come from
+ ``_AUTO_LADDER`` and the probe is only READ from ``_SMOKE_CACHE``, where the load path already
+ paid for it. An unprobed scheme counts as usable, like the ``unproven_ok`` path; a probed
+ failure does not.
+
+ ``auto``, not "any scheme": the ladder leaves nvfp4 out, so a host that can only run nvfp4
+ honours an explicit request and has nothing automatic to offer."""
+ if not dense_transformer_supported(target):
+ return False
+ # The loader keeps a pipeline dense when nothing can compile the result, so a host without
+ # inductor has no fast path to advertise. Host-level only: the per-family and per-request
+ # halves of that decision belong to the load, not to a row.
+ from .diffusion_speed import compile_eligible
+
+ if not compile_eligible(target, is_gguf = False, family = None):
+ return False
+ # Not redundant with the arch floor: `_scheme_supported` rejects every scheme when the torchao
+ # import fails, so an ABI skew would advertise a fast path every load then falls back from.
+ if torchao_unavailable_reason() is not None:
+ return False
+ cap = _capability()
+ if cap is None:
+ return False
+ card = _smoke_cache_device_key(str(getattr(target, "device", "cuda")))
+ for floor, schemes in _AUTO_LADDER:
+ if cap >= floor:
+ return any(_SMOKE_CACHE.get((scheme, card), True) for scheme in schemes)
+ return False
+
+
def auto_scheme_candidates(target: Any, family: Optional[str] = None) -> tuple[str, ...]:
"""Every scheme ``auto`` would accept on this device, best first.
``select_transformer_quant_scheme`` returns only the winner, which is all the load needs
@@ -522,19 +730,33 @@ def auto_scheme_candidates(target: Any, family: Optional[str] = None) -> tuple[s
if cap >= floor:
return tuple(
scheme
- for scheme in _prefer_consumer_scheme(schemes, device)
+ for scheme in schemes
if not _family_denied(family, scheme) and _scheme_supported(scheme, device)
)
return ()
-def _prefer_consumer_scheme(schemes: tuple[str, ...], device: Any) -> tuple[str, ...]:
- """Reorder an arch tier's schemes for the GPU class. On consumer / workstation cards move int8
- first: they halve fp8/fp16 FP32-accumulate while int8 runs full-rate, so int8 is as fast or
- faster (and the only path on pre-Ada consumer). Data-center parts keep fp8 first."""
- if TQ_INT8 in schemes and schemes[0] != TQ_INT8 and _is_consumer_gpu(device):
- return (TQ_INT8,) + tuple(s for s in schemes if s != TQ_INT8)
- return schemes
+def auto_scheme_candidates_cached(target: Any, family: Optional[str] = None) -> tuple[str, ...]:
+ """``auto_scheme_candidates`` answered from ``_SMOKE_CACHE`` alone, for a polled status route.
+
+ Same ladder and deny list, but no probe: ``_scheme_supported`` can spawn the child smoke probe
+ or allocate in this process, and neither belongs on a route polled every few seconds. Unprobed
+ counts as usable and a probed failure does not, as in ``dense_quant_host_capable``, so the
+ published ladder sharpens as loads record verdicts instead of paying for them here."""
+ if not dense_transformer_supported(target):
+ return ()
+ cap = _capability()
+ if cap is None:
+ return ()
+ card = _smoke_cache_device_key(str(getattr(target, "device", "cuda")))
+ for floor, schemes in _AUTO_LADDER:
+ if cap >= floor:
+ return tuple(
+ scheme
+ for scheme in schemes
+ if not _family_denied(family, scheme) and _SMOKE_CACHE.get((scheme, card), True)
+ )
+ return ()
def _capability() -> Optional[tuple[int, int]]:
diff --git a/studio/backend/main.py b/studio/backend/main.py
index 86e842c4573..16f3c10fbfd 100644
--- a/studio/backend/main.py
+++ b/studio/backend/main.py
@@ -610,6 +610,18 @@ def _post_warm_background_work(generation: Optional[int] = None) -> None:
# greyed out for the session.
_structlog.get_logger(__name__).warning("mlx autorepair skipped: %s", _mlx_exc)
+ if _post_warm_retired(generation):
+ return
+ # Off the polled path on purpose: /api/system must not import torchao itself (see
+ # _dense_quant_supported). Gated on torch being up rather than assumed, since
+ # UNSLOTH_STUDIO_DISABLE_TORCH_WARM=1 exists precisely to keep the ML stack cold.
+ if "torch" in sys.modules:
+ try:
+ _refresh_dense_quant_capability()
+ except Exception as _dq_exc: # noqa: BLE001 -- a picker label must never break the warm
+ import structlog as _structlog
+ _structlog.get_logger(__name__).debug("dense quant capability skipped: %s", _dq_exc)
+
if _post_warm_retired(generation):
return
_start_linked_folder_auto_sync(generation)
@@ -2164,6 +2176,103 @@ def _get_cached_system_gpu_info(
return combined_info
+def _probe_dense_quant_supported() -> bool:
+ """Whether an ``auto`` request could engage a dense quant on EVERY visible card.
+
+ The picker cannot see which card a load lands on, so a mixed host answers for the least capable.
+
+ IMPORTS the ML stack, so only ``_refresh_dense_quant_capability`` calls it, and only from the
+ post-warm worker or a request that already has both modules. Never memoised: the answer
+ sharpens, since an unprobed scheme counts as usable and a later load can record a kernel
+ failure in ``_SMOKE_CACHE``."""
+ try:
+ from core.inference.diffusion_device import (
+ diffusion_device_scope,
+ resolve_diffusion_device_target,
+ )
+ from core.inference.diffusion_transformer_quant import dense_quant_host_capable
+
+ import torch
+
+ count = torch.cuda.device_count() if torch.cuda.is_available() else 0
+ if count <= 1:
+ return bool(dense_quant_host_capable(resolve_diffusion_device_target()))
+ for ordinal in range(count):
+ with diffusion_device_scope(ordinal):
+ if not dense_quant_host_capable(resolve_diffusion_device_target(ordinal = ordinal)):
+ return False
+ return True
+ except Exception: # noqa: BLE001 -- a capability probe must never fail a status request
+ return False
+
+
+def _probe_dense_quant_schemes() -> list[str]:
+ """The auto ladder's schemes for this host, best first, on the same ladder and deny list the
+ loader's ``auto_scheme_candidates`` reads; a mixed host answers with the INTERSECTION. IMPORTS
+ the ML stack.
+
+ The CACHED variant, since a request holding torch reaches this from the polled ``/api/system``:
+ the load-time helper runs ``_scheme_supported``, which spawns the smoke probe or allocates in
+ this process. Like the capability bit, it sharpens as loads record verdicts in ``_SMOKE_CACHE``."""
+ try:
+ from core.inference.diffusion_device import (
+ diffusion_device_scope,
+ resolve_diffusion_device_target,
+ )
+ from core.inference.diffusion_transformer_quant import auto_scheme_candidates_cached
+
+ import torch
+
+ count = torch.cuda.device_count() if torch.cuda.is_available() else 0
+ if count <= 1:
+ return list(auto_scheme_candidates_cached(resolve_diffusion_device_target()))
+ common: Optional[list[str]] = None
+ for ordinal in range(count):
+ with diffusion_device_scope(ordinal):
+ schemes = list(
+ auto_scheme_candidates_cached(resolve_diffusion_device_target(ordinal = ordinal))
+ )
+ common = schemes if common is None else [s for s in common if s in schemes]
+ return common or []
+ except Exception: # noqa: BLE001 -- a capability probe must never fail a status request
+ return []
+
+
+# Resolved off the polled path: None until the post-warm worker or a request already holding the
+# ML stack has answered.
+_dense_quant_capability: Optional[bool] = None
+_dense_quant_scheme_ladder: list[str] = []
+
+
+def _refresh_dense_quant_capability() -> bool:
+ """Resolve the dense-quant bit and cache it. Imports torch and torchao; never call from a route
+ that has not already got them."""
+ global _dense_quant_capability, _dense_quant_scheme_ladder
+ _dense_quant_capability = _probe_dense_quant_supported()
+ _dense_quant_scheme_ladder = _probe_dense_quant_schemes() if _dense_quant_capability else []
+ return _dense_quant_capability
+
+
+def _dense_quant_supported() -> bool:
+ """The dense-quant bit for ``/api/system``, from already-loaded state only.
+
+ This route is polled throughout startup, and ``import torch`` and ``import torchao.quantization``
+ cost ~0.8s each and hold the GIL, the stall ``_await_hardware_detection`` already keeps off this
+ path. So never import here. The post-warm worker resolves it once the warm has the stack up, and
+ a request finding both modules loaded refreshes it, so a load's kernel verdict reaches the picker
+ on the next poll. False before that is honest: the picker renders no fast label, as it does for
+ an unknown VRAM budget."""
+ if "torch" in sys.modules and "torchao" in sys.modules:
+ return _refresh_dense_quant_capability()
+ return bool(_dense_quant_capability)
+
+
+def _dense_quant_schemes() -> list[str]:
+ """The scheme ladder for ``/api/system``, a pure read of already-resolved state: the polled route
+ must never import torch, and the entry beside it refreshed both in one pass."""
+ return list(_dense_quant_scheme_ladder)
+
+
@app.get("/api/system")
def get_system_info(
current_subject: str = Depends(get_current_subject), refresh_memory: bool = False
@@ -2259,6 +2368,11 @@ def get_system_info(
**export_capability(),
# Video capability + reason, same shape. Additive: older clients ignore the extra keys.
**video_capability(),
+ # One bit cannot tell an Ampere host (int8 only) from an Ada one, nor spot an unsupported
+ # CUDA card, so the picker gets the scheme list too. The bit resolves first; the list is a
+ # pure read of that same pass.
+ "dense_quant_supported": _dense_quant_supported(),
+ "dense_quant_schemes": _dense_quant_schemes(),
}
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index f12302975b9..fd780cf33ce 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -3703,7 +3703,7 @@ class DiffusionLoadRequest(BaseModel):
description = "Transformer compute dtype. UNSET or auto (the default) picks the "
"fastest precision the hardware supports: the DENSE bf16 transformer "
"is loaded instead of the GGUF and torchao-quantised onto the "
- "low-precision tensor cores (data-center fp8, consumer/Ampere int8), "
+ "low-precision tensor cores (int8 first on every tier, then fp8 on sm_89+), "
"falling back to the GGUF when the device, VRAM or disk cannot take "
"it. none/off pins running the GGUF as-is; an explicit scheme forces "
"that scheme. Dense path needs CUDA + bf16. An EXPLICIT scheme fails "
@@ -4197,6 +4197,13 @@ class DiffusionResolvedControl(BaseModel):
"so a client reading an older backend's payload still parses.",
)
reason: str = Field("", description = "Short human-readable reason for the resolved value.")
+ artifact: Optional[str] = Field(
+ None,
+ description = "The hosted or local file the engaged value came from, as "
+ '"prequant:<repo>/<file>", when a pre-quantized checkpoint was seeded rather than the '
+ "weights being quantised in memory. Declared here or pydantic drops it and no API client "
+ "ever sees the provenance. Null on every other control and on a runtime quantise.",
+ )
class DiffusionDownloadPlanEntry(BaseModel):
@@ -4637,7 +4644,7 @@ class VideoLoadRequest(BaseModel):
None,
description = "Quantise the dense DiT(s) on a full-pipeline load. On a diffusers "
"pipeline load the dense bf16 transformer(s) are torchao-quantised in place onto "
- "the low-precision tensor cores (data-center fp8, consumer/Ampere int8), which is "
+ "the low-precision tensor cores (int8 first on every tier, then fp8 on sm_89+), which is "
"faster than running dense bf16. For a dual-expert MoE family (Wan2.2-A14B) BOTH "
"experts are quantised with the same scheme. null/none/off keeps the DiT(s) at "
"their loaded precision; an explicit scheme forces it. Needs CUDA + bf16; ignored "
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index a8d586ee925..aef29eec7e7 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -37140,6 +37140,9 @@ async def diffusion_download_plan(
# anything is measured, and an offloaded transformer skips the dense quant.
memory_mode = getattr(request, "memory_mode", None),
cpu_offload = bool(getattr(request, "cpu_offload", False)),
+ # An uncompiled torchao transformer is slower than the bf16 it replaces, so the
+ # loader keeps a pipeline dense under eager.
+ speed_mode = getattr(request, "speed_mode", None),
# Judged on the card this pick would load on, as the loader does.
gpu_ordinal = gpu_ordinal,
)
@@ -37165,6 +37168,9 @@ async def diffusion_download_plan(
memory_mode = request.memory_mode,
cpu_offload = request.cpu_offload,
transformer_prequant_path = request.transformer_prequant_path,
+ # A forced accumulate the hosted checkpoint cannot bake declines the seed, so without
+ # this the plan stages a file the load refuses and replaces with dense shards.
+ transformer_quant_fast_accum = request.transformer_quant_fast_accum,
loras = request.loras,
# Only the verdict, not the probe: the panel stages exactly what this reports.
# Clearing the probe drops the hosted DiT prequant, so a GGUF pick naming an explicit
@@ -37345,6 +37351,7 @@ async def load_diffusion_model_gated(
text_encoder_quant = request.text_encoder_quant,
memory_mode = getattr(request, "memory_mode", None),
cpu_offload = bool(getattr(request, "cpu_offload", False)),
+ speed_mode = getattr(request, "speed_mode", None),
gpu_ordinal = gpu_ordinal,
)
elif fam is not None and pending_name == ENGINE_SD_CPP:
@@ -37410,6 +37417,7 @@ async def load_diffusion_model_gated(
text_encoder_quant = request.text_encoder_quant,
memory_mode = getattr(request, "memory_mode", None),
cpu_offload = bool(getattr(request, "cpu_offload", False)),
+ speed_mode = getattr(request, "speed_mode", None),
gpu_ordinal = gpu_ordinal,
)
diff --git a/studio/backend/tests/test_dense_quant_capability.py b/studio/backend/tests/test_dense_quant_capability.py
new file mode 100644
index 00000000000..4e1dc0c51a7
--- /dev/null
+++ b/studio/backend/tests/test_dense_quant_capability.py
@@ -0,0 +1,285 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for the dense quant capability reported by ``/api/system``."""
+
+import ast
+import sys
+import types
+from pathlib import Path
+from typing import Optional
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parent.parent
+
+
+def _src(name: str) -> str:
+ src = (_BACKEND / "main.py").read_text(encoding = "utf-8")
+ node = next(
+ n for n in ast.walk(ast.parse(src)) if isinstance(n, ast.FunctionDef) and n.name == name
+ )
+ return ast.get_source_segment(src, node)
+
+
+def _run(monkeypatch, *, device_count, capable_by_ordinal):
+ """Run an uncached copy of ``_dense_quant_supported`` with mocked dependencies."""
+ scoped: list = []
+
+ fake_torch = types.SimpleNamespace(
+ cuda = types.SimpleNamespace(
+ is_available = lambda: device_count > 0,
+ device_count = lambda: device_count,
+ )
+ )
+
+ class _Scope:
+ def __init__(self, ordinal):
+ self.ordinal = ordinal
+
+ def __enter__(self):
+ scoped.append(self.ordinal)
+ return None
+
+ def __exit__(self, *exc):
+ return False
+
+ fake_device = types.ModuleType("core.inference.diffusion_device")
+ fake_device.diffusion_device_scope = _Scope
+ fake_device.resolve_diffusion_device_target = lambda ordinal = None: ordinal
+ fake_quant = types.ModuleType("core.inference.diffusion_transformer_quant")
+ fake_quant.dense_quant_host_capable = lambda target: capable_by_ordinal[target]
+
+ monkeypatch.setitem(sys.modules, "torch", fake_torch)
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_device", fake_device)
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_transformer_quant", fake_quant)
+
+ namespace: dict = {}
+ exec(_src("_probe_dense_quant_supported"), namespace) # noqa: S102 -- the real body
+ return namespace["_probe_dense_quant_supported"](), scoped
+
+
+def test_a_single_capable_gpu_reports_capable(monkeypatch):
+ result, scoped = _run(monkeypatch, device_count = 1, capable_by_ordinal = {None: True})
+ assert result is True
+ assert scoped == []
+
+
+def test_a_single_incapable_gpu_reports_incapable(monkeypatch):
+ result, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = {None: False})
+ assert result is False
+
+
+def test_every_visible_card_must_be_capable(monkeypatch):
+ result, scoped = _run(monkeypatch, device_count = 2, capable_by_ordinal = {0: True, 1: False})
+ assert result is False
+ # Probe each ordinal under its own device scope.
+ assert scoped == [0, 1]
+
+
+def test_a_homogeneous_multi_gpu_host_still_reports_capable(monkeypatch):
+ result, scoped = _run(monkeypatch, device_count = 2, capable_by_ordinal = {0: True, 1: True})
+ assert result is True
+ assert scoped == [0, 1]
+
+
+def test_no_gpu_reports_incapable(monkeypatch):
+ result, _ = _run(monkeypatch, device_count = 0, capable_by_ordinal = {None: False})
+ assert result is False
+
+
+def test_a_probe_failure_reports_incapable(monkeypatch):
+ """A probe that raises must report the conservative result, not fail the status request."""
+
+ class _Boom(dict):
+ def __getitem__(self, key):
+ raise RuntimeError("driver went away")
+
+ result, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = _Boom())
+ assert result is False
+
+
[email protected]("needle", ["diffusion_device_scope", "device_count"])
+def test_the_wiring_stays_in_place(needle):
+ assert needle in _src("_probe_dense_quant_supported")
+
+
+def test_the_capability_is_published_and_never_memoised():
+ """`/api/system` carries the bit, and the probe must not be pinned.
+
+ `dense_quant_host_capable` counts an UNPROBED scheme as usable, so an early yes can be undone by
+ a later load recording a kernel failure. Memoising would pin the optimistic answer forever.
+ """
+ src = (_BACKEND / "main.py").read_text(encoding = "utf-8")
+ assert '"dense_quant_supported": _dense_quant_supported()' in src
+ node = next(
+ n
+ for n in ast.walk(ast.parse(src))
+ if isinstance(n, ast.FunctionDef) and n.name == "_probe_dense_quant_supported"
+ )
+ assert node.decorator_list == []
+
+
+def test_the_polled_route_never_imports_the_ml_stack():
+ """torch and torchao cost ~0.8s each and hold the GIL; /api/system is polled through startup.
+
+ The reader answers from `sys.modules` and a value the post-warm worker resolved, so a poll on a
+ cold backend imports nothing. `_await_hardware_detection` avoids the same stall.
+ """
+ reader = _src("_dense_quant_supported")
+ assert '"torch" in sys.modules' in reader and '"torchao" in sys.modules' in reader
+ # The reader itself must not reach the importing probe except behind that guard.
+ assert "_probe_dense_quant_supported" not in reader
+ src = (_BACKEND / "main.py").read_text(encoding = "utf-8")
+ # ...and something off the polled path has to resolve it, or the label never appears.
+ assert "_refresh_dense_quant_capability()" in _src("_post_warm_background_work")
+
+
+def test_the_probe_follows_the_smoke_cache_as_it_warms(monkeypatch):
+ """A verdict the loader has since paid for must reach the next resolution."""
+ answers = {None: True}
+ result, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = answers)
+ assert result is True
+ answers[None] = False
+ result, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = answers)
+ assert result is False
+
+
+def test_the_scheme_ladder_is_published_beside_the_bit():
+ """``/api/system`` publishes the scheme ladder beside the capability bit, from the same
+ ``auto_scheme_candidates`` the loader's selector reads."""
+ src = (_BACKEND / "main.py").read_text(encoding = "utf-8")
+ assert '"dense_quant_schemes": _dense_quant_schemes()' in src
+ assert "auto_scheme_candidates" in _src("_probe_dense_quant_schemes")
+
+
+def test_the_ladder_is_read_off_the_same_refresh_as_the_bit():
+ """The reader beside the bit is a pure read, so the polled route never probes or imports torch a
+ second time, and both entries come from one pass."""
+ reader = _src("_dense_quant_schemes")
+ assert "_probe_dense_quant_schemes" not in reader
+ assert "_refresh_dense_quant_capability" not in reader
+ refresh = _src("_refresh_dense_quant_capability")
+ assert "_probe_dense_quant_schemes()" in refresh
+ src = (_BACKEND / "main.py").read_text(encoding = "utf-8")
+ assert src.index('"dense_quant_supported": _dense_quant_supported()') < src.index(
+ '"dense_quant_schemes": _dense_quant_schemes()'
+ )
+
+
+def _run_schemes(monkeypatch, *, device_count, schemes_by_ordinal):
+ """Run an uncached copy of ``_probe_dense_quant_schemes`` with mocked dependencies."""
+ scoped: list = []
+
+ fake_torch = types.SimpleNamespace(
+ cuda = types.SimpleNamespace(
+ is_available = lambda: device_count > 0,
+ device_count = lambda: device_count,
+ )
+ )
+
+ class _Scope:
+ def __init__(self, ordinal):
+ self.ordinal = ordinal
+
+ def __enter__(self):
+ scoped.append(self.ordinal)
+ return None
+
+ def __exit__(self, *exc):
+ return False
+
+ fake_device = types.ModuleType("core.inference.diffusion_device")
+ fake_device.diffusion_device_scope = _Scope
+ fake_device.resolve_diffusion_device_target = lambda ordinal = None: ordinal
+ fake_quant = types.ModuleType("core.inference.diffusion_transformer_quant")
+ fake_quant.auto_scheme_candidates_cached = lambda target: schemes_by_ordinal[target]
+
+ monkeypatch.setitem(sys.modules, "torch", fake_torch)
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_device", fake_device)
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_transformer_quant", fake_quant)
+
+ namespace: dict = {"Optional": Optional}
+ exec(_src("_probe_dense_quant_schemes"), namespace) # noqa: S102 -- the real body
+ return namespace["_probe_dense_quant_schemes"](), scoped
+
+
+def test_an_ada_host_publishes_fp8_first(monkeypatch):
+ result, scoped = _run_schemes(
+ monkeypatch, device_count = 1, schemes_by_ordinal = {None: ("fp8", "int8")}
+ )
+ assert result == ["fp8", "int8"]
+ assert scoped == []
+
+
+def test_an_ampere_host_publishes_int8(monkeypatch):
+ result, _ = _run_schemes(monkeypatch, device_count = 1, schemes_by_ordinal = {None: ("int8",)})
+ assert result == ["int8"]
+
+
+def test_a_mixed_host_publishes_only_what_every_card_runs(monkeypatch):
+ result, scoped = _run_schemes(
+ monkeypatch,
+ device_count = 2,
+ schemes_by_ordinal = {0: ("fp8", "int8"), 1: ("int8",)},
+ )
+ assert result == ["int8"]
+ assert scoped == [0, 1]
+
+
+def test_an_unsupported_host_publishes_nothing(monkeypatch):
+ result, _ = _run_schemes(monkeypatch, device_count = 0, schemes_by_ordinal = {None: ()})
+ assert result == []
+
+
+def test_a_scheme_probe_failure_publishes_nothing(monkeypatch):
+ class _Boom(dict):
+ def __getitem__(self, key):
+ raise RuntimeError("driver went away")
+
+ result, _ = _run_schemes(monkeypatch, device_count = 1, schemes_by_ordinal = _Boom())
+ assert result == []
+
+
+def test_an_incapable_host_never_publishes_a_ladder():
+ refresh = _src("_refresh_dense_quant_capability")
+ assert "if _dense_quant_capability else []" in refresh
+
+
+def test_the_warm_refresh_honours_the_torch_kill_switch():
+ """UNSLOTH_STUDIO_DISABLE_TORCH_WARM=1 must keep the stack cold.
+
+ `start_background_warm` is then a no-op and `join_background_warm` returns at once, so the
+ post-warm worker reaches this point with torch unimported. Refreshing there would import torch
+ and torchao and defeat the switch, so it is gated on torch already being up.
+ """
+ body = _src("_post_warm_background_work")
+ refresh = body.index("_refresh_dense_quant_capability()")
+ guard = body.rindex('"torch" in sys.modules', 0, refresh)
+ assert guard != -1
+ # The guard must not swallow the rest of the worker.
+ assert "_start_linked_folder_auto_sync" in body[refresh:]
+
+
+def test_the_polled_ladder_never_runs_the_allocating_smoke_probe(monkeypatch):
+ """The ladder on the polled route reads ``_SMOKE_CACHE`` and nothing else.
+
+ ``_scheme_supported`` spawns the up-to-180s child smoke probe, or allocates in this process when
+ it cannot, and an allocator failure is deliberately not cached, so reaching it from
+ ``/api/system`` would repeat the work on every poll.
+ """
+ from core.inference import diffusion_transformer_quant as tq
+
+ def _never(*_a, **_k):
+ raise AssertionError("the polled ladder ran the smoke probe")
+
+ monkeypatch.setattr(tq, "_scheme_supported", _never)
+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda _target: True)
+ monkeypatch.setattr(tq, "_capability", lambda: (8, 9))
+ monkeypatch.setattr(tq, "_smoke_cache_device_key", lambda _device: "cuda:0")
+ monkeypatch.setattr(tq, "_SMOKE_CACHE", {("fp8", "cuda:0"): False})
+ # An unprobed scheme counts as usable; a probed failure does not.
+ assert tq.auto_scheme_candidates_cached(object()) == ("int8",)
+ monkeypatch.setattr(tq, "_SMOKE_CACHE", {})
+ assert tq.auto_scheme_candidates_cached(object()) == ("int8", "fp8")
+ assert "auto_scheme_candidates_cached" in _src("_probe_dense_quant_schemes")
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 50ec3f35d8f..f31906818f8 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -11,6 +11,7 @@ GPU, weights, or network access is needed (sub-second, CI-friendly).
from __future__ import annotations
import contextlib
+import dataclasses
import re
import sys
import threading
@@ -2274,6 +2275,67 @@ def test_load_progress_downloading_then_finalizing(monkeypatch):
assert backend.load_progress()["phase"] == "finalizing" # 1000/1000
+def test_load_progress_counts_only_the_selected_prequant_file_in_its_artifact_repo(monkeypatch):
+ # unsloth/Qwen-Image-FP8 hosts one checkpoint per scheme, so a card that already ran fp8 and now
+ # loads int8 has 19 GB of a sibling file in that repo while expected_bytes counts only int8.
+ # Counting the whole repo reported finalizing with most of the selected checkpoint still to come.
+ backend = DiffusionBackend()
+ backend._loading = _LoadingState(
+ repo_id = "Qwen/Qwen-Image",
+ base_repo = "Qwen/Qwen-Image",
+ expected_bytes = 1000,
+ )
+ backend._loading.asset_repos = ("unsloth/Qwen-Image-FP8",)
+ # 600 bytes of the fp8 sibling were already there when the load claimed the repo.
+ backend._loading.asset_files = (("unsloth/Qwen-Image-FP8", "Qwen-Image-INT8.pt", 400, 600),)
+ monkeypatch.setattr(DiffusionBackend, "_cache_bytes", staticmethod(lambda repo: 0))
+ monkeypatch.setattr(DiffusionBackend, "_cache_file_bytes", staticmethod(lambda repo, f: 0))
+
+ # 100 of the selected file's 400 bytes have landed as an .incomplete blob.
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_cache_bytes",
+ staticmethod(lambda repo: 700 if repo == "unsloth/Qwen-Image-FP8" else 500),
+ )
+ p = backend.load_progress()
+ assert p["phase"] == "downloading"
+ assert p["bytes_downloaded"] == 600 # 500 base + 100 of the selected file, not 500 + 700
+
+ # Finished: the file is in the snapshot, so it counts in full and exactly once.
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_cache_file_bytes",
+ staticmethod(lambda repo, f: 400 if f == "Qwen-Image-INT8.pt" else 0),
+ )
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_cache_bytes",
+ staticmethod(lambda repo: 1000 if repo == "unsloth/Qwen-Image-FP8" else 600),
+ )
+ assert backend.load_progress()["phase"] == "finalizing" # 600 + 400
+ assert backend.load_progress()["bytes_downloaded"] == 1000
+
+
+def test_load_progress_reports_a_prequant_file_that_was_already_cached(monkeypatch):
+ # Nothing to download from the artifact repo: the selected file must still count, or the bar
+ # stalls short of 100% for the rest of the load.
+ backend = DiffusionBackend()
+ backend._loading = _LoadingState(
+ repo_id = "Qwen/Qwen-Image",
+ base_repo = "Qwen/Qwen-Image",
+ expected_bytes = 1000,
+ )
+ backend._loading.asset_repos = ("unsloth/Qwen-Image-FP8",)
+ backend._loading.asset_files = (("unsloth/Qwen-Image-FP8", "Qwen-Image-INT8.pt", 400, 1000),)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_cache_bytes",
+ staticmethod(lambda repo: 1000 if repo == "unsloth/Qwen-Image-FP8" else 600),
+ )
+ monkeypatch.setattr(DiffusionBackend, "_cache_file_bytes", staticmethod(lambda repo, f: 400))
+ assert backend.load_progress()["phase"] == "finalizing"
+
+
def test_load_progress_counts_a_mirrored_pipeline_repo_once(monkeypatch):
# base_repo == repo_id, so count once. Summing adds the upstream's stale partial blobs -- the
# very thing that selects the mirror -- to the mirror's live bytes, pegging the bar at 100%.
@@ -6273,6 +6335,7 @@ def test_diffusion_status_response_carries_resolved():
"source": "auto",
"status": "applied",
"reason": "blackwell",
+ "artifact": None,
}
}
# Absent by default (nothing resolved / native engine).
@@ -6294,7 +6357,27 @@ def test_diffusion_status_response_carries_requested_precision():
}
}
resp = DiffusionStatusResponse(loaded = True, resolved = rec)
- assert resp.model_dump()["resolved"] == rec
+ assert resp.model_dump()["resolved"] == {
+ "transformer_quant": {**rec["transformer_quant"], "artifact": None}
+ }
+
+
+def test_diffusion_status_response_carries_the_prequant_artifact():
+ # The loader records which hosted file the engaged precision came from; undeclared here, pydantic
+ # drops it and no API client sees the provenance.
+ from models.inference import DiffusionStatusResponse
+
+ rec = {
+ "transformer_quant": {
+ "value": "fp8",
+ "source": "auto",
+ "reason": "seeded",
+ "artifact": "prequant:unsloth/Z-Image-Turbo-FP8/Z-Image-Turbo-FP8.pt",
+ }
+ }
+ resp = DiffusionStatusResponse(loaded = True, resolved = rec)
+ dumped = resp.model_dump()["resolved"]["transformer_quant"]
+ assert dumped["artifact"] == "prequant:unsloth/Z-Image-Turbo-FP8/Z-Image-Turbo-FP8.pt"
def test_diffusion_status_response_carries_gguf_variant():
@@ -6351,6 +6434,45 @@ def test_plan_memory_dense_replan_does_not_double_count_prefetched_transformer(m
assert plan.offload_policy == OFFLOAD_NONE
+def test_plan_memory_pipeline_replan_prices_the_quantised_transformer(monkeypatch):
+ """A pipeline re-plan reads the quant-size overrides instead of the whole-repo cache.
+
+ The pipeline branch sizes the repo as one download, the bf16 footprint the re-plan exists to
+ replace; ignoring the overrides returned the bf16 plan unchanged, so an offloaded pipeline
+ could never reach the fast path.
+ """
+ from core.inference import diffusion as dmod
+ from core.inference.diffusion_memory import OFFLOAD_NONE, DeviceMemory
+
+ backend = DiffusionBackend()
+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", supports_model_cpu_offload = True)
+ # 40 GiB card: room for the int8 transformer + companions + headroom, not for the bf16 one.
+ monkeypatch.setattr(
+ dmod,
+ "settled_snapshot_device_memory",
+ lambda t: DeviceMemory("cuda", "cuda", "discrete_vram", 40000, 40960),
+ )
+ monkeypatch.setattr(dmod, "estimate_image_runtime_mib", lambda **kw: 4000)
+ # The cached repo holds the bf16 transformer; consulting it would offload.
+ monkeypatch.setattr(
+ DiffusionBackend, "_cache_bytes", staticmethod(lambda repo: (24000 + 8000) * 1024 * 1024)
+ )
+ plan = backend._plan_memory(
+ target,
+ None,
+ "org/base",
+ types.SimpleNamespace(name = "z-image", base_repo = "org/base"),
+ None,
+ False,
+ kind = "pipeline",
+ repo_id = "org/base",
+ transformer_resident_override_mib = 12000,
+ companion_override_mib = 8000,
+ text_encoder_override_mib = 6000,
+ )
+ assert plan.offload_policy == OFFLOAD_NONE
+
+
def _split_cache_roots(
tmp_path,
monkeypatch,
@@ -10975,6 +11097,488 @@ def test_generation_in_flight_never_builds_a_backend(fake_runtime, monkeypatch):
assert diffusion_mod.generation_in_flight() is False
+# Pipeline dense quantisation.
+
+
+class _FakeDenoiser:
+ """A denoiser whose parameters expose a dtype to the quantisation gate."""
+
+ def __init__(self, dtype = "torch.bfloat16") -> None:
+ self._params = [types.SimpleNamespace(dtype = dtype)]
+
+ def parameters(self, recurse = True):
+ return iter(self._params)
+
+
+def _init_with_denoiser(dtype):
+ """A ``_FakePipe.__init__`` whose transformer reports ``dtype``, for the gate's dtype walk."""
+ real_init = _FakePipe.__init__
+
+ def _init(self):
+ real_init(self)
+ self.transformer = _FakeDenoiser(dtype)
+
+ return _init
+
+
+def _stub_pipeline_dense_quant(
+ backend,
+ monkeypatch,
+ *,
+ engages = "fp8",
+ denoisers = ("transformer",),
+):
+ """Stub a capable CUDA host and record transformer quantisation calls."""
+ from core.inference import diffusion as dmod
+
+ calls: list = []
+ _force_cuda_target(backend, monkeypatch)
+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(
+ dmod, "select_transformer_quant_scheme", lambda target, mode, family = None: "fp8"
+ )
+ real_init = _FakePipe.__init__
+
+ def _init(self):
+ real_init(self)
+ for attr in denoisers:
+ setattr(self, attr, _FakeDenoiser())
+
+ monkeypatch.setattr(_FakePipe, "__init__", _init)
+
+ def _quantize(pipe, target, **kwargs):
+ calls.append({"pipe": pipe, "transformer": pipe.transformer, **kwargs})
+ return engages
+
+ monkeypatch.setattr(dmod, "quantize_transformer", _quantize)
+ return calls
+
+
+def test_a_pipeline_pick_quantises_its_transformer_in_place(fake_runtime, tmp_path, monkeypatch):
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert status["transformer_quant"] == "fp8"
+ assert len(calls) == 1
+ assert calls[0]["transformer"] is backend._state.pipe.transformer
+ resolved = status["resolved"]["transformer_quant"]
+ assert resolved["value"] == "fp8"
+ assert resolved["source"] == "auto" and resolved["status"] == "applied"
+ backend.unload()
+
+
+def test_a_pipeline_pick_keeps_bf16_when_the_scheme_declines(fake_runtime, tmp_path, monkeypatch):
+ backend = DiffusionBackend()
+ _stub_pipeline_dense_quant(backend, monkeypatch, engages = None)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert status["transformer_quant"] is None
+ resolved = status["resolved"]["transformer_quant"]
+ assert resolved["value"] == "off" and resolved["source"] == "auto"
+ assert resolved["status"] == "applied"
+ backend.unload()
+
+
+def test_a_pipeline_pick_refuses_an_explicit_scheme_that_did_not_engage(
+ fake_runtime, tmp_path, monkeypatch
+):
+ backend = DiffusionBackend()
+ _stub_pipeline_dense_quant(backend, monkeypatch, engages = None)
+ with pytest.raises(RuntimeError) as excinfo:
+ backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ _base_local_dir = str(tmp_path),
+ )
+ assert "transformer_quant='fp8' could not be used" in str(excinfo.value)
+
+
+def test_a_pipeline_pick_does_not_quantise_under_offload(fake_runtime, tmp_path, monkeypatch):
+ """Offloaded pipelines stay dense because torchao tensors cannot move."""
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ real_plan = DiffusionBackend._plan_memory
+
+ def _offloading_plan(self, *args, **kwargs):
+ plan = real_plan(self, *args, **kwargs)
+ return dataclasses.replace(plan, offload_policy = "model")
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _offloading_plan)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert "offload" in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+
+
+def test_an_offloaded_pipeline_replans_against_the_quantised_size(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference.diffusion_memory import OFFLOAD_NONE
+
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ overrides: list = []
+ real_plan = DiffusionBackend._plan_memory
+
+ def _plan(self, *args, **kwargs):
+ plan = real_plan(self, *args, **kwargs)
+ override = kwargs.get("transformer_resident_override_mib")
+ overrides.append(override)
+ policy = OFFLOAD_NONE if override is not None else "model"
+ return dataclasses.replace(plan, offload_policy = policy)
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ # Re-planned against the quantised transformer, so the offload that would have blocked the
+ # conversion is gone and the pipeline is quantised.
+ assert any(override is not None for override in overrides)
+ assert len(calls) == 1
+ assert status["transformer_quant"] == "fp8"
+ assert status["offload_policy"] == OFFLOAD_NONE
+ backend.unload()
+
+
+def test_a_declined_quant_gives_back_the_bf16_placement(fake_runtime, tmp_path, monkeypatch):
+ from core.inference.diffusion_memory import OFFLOAD_NONE
+
+ backend = DiffusionBackend()
+ _stub_pipeline_dense_quant(backend, monkeypatch, engages = None)
+ real_plan = DiffusionBackend._plan_memory
+
+ def _plan(self, *args, **kwargs):
+ plan = real_plan(self, *args, **kwargs)
+ resident = kwargs.get("transformer_resident_override_mib") is not None
+ return dataclasses.replace(plan, offload_policy = OFFLOAD_NONE if resident else "model")
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert status["transformer_quant"] is None
+ # The bf16 build is what actually loaded, so it keeps the bf16 plan's offload.
+ assert status["offload_policy"] == "model"
+ backend.unload()
+
+
+def test_a_pipeline_pick_stays_dense_when_the_speed_mode_will_not_compile(
+ fake_runtime, tmp_path, monkeypatch
+):
+ """Eager torchao is far slower than the bf16 it replaces, so an uncompilable load keeps bf16."""
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ speed_mode = "eager",
+ _base_local_dir = str(tmp_path),
+ )
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert "eager" in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+
+
+def test_speed_off_keeps_the_bit_exact_bf16_pipeline(fake_runtime, tmp_path, monkeypatch):
+ """Speed=Off asks for bit-exact output, so an auto quant is rewritten to off before it runs."""
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ speed_mode = "off",
+ _base_local_dir = str(tmp_path),
+ )
+ assert calls == []
+ assert status["transformer_quant"] is None
+ # Not the compile guard: the rewrite happens well before it, so no decline reason is recorded.
+ assert status["resolved"]["transformer_quant"]["value"] == "off"
+ backend.unload()
+
+
+def test_a_pipeline_pick_stays_dense_when_this_process_cannot_compile(
+ fake_runtime, tmp_path, monkeypatch
+):
+ """A Windows install without Triton, or TORCHDYNAMO_DISABLE, must not quantise either."""
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ monkeypatch.setattr(dmod, "compile_eligible", lambda target, **kw: False)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert "compile" in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+
+
+def test_an_uncompilable_pipeline_refuses_an_explicit_scheme(fake_runtime, tmp_path, monkeypatch):
+ """A pinned scheme fails closed rather than landing a slower-than-bf16 build."""
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ _stub_pipeline_dense_quant(backend, monkeypatch)
+ monkeypatch.setattr(dmod, "compile_eligible", lambda target, **kw: False)
+ with pytest.raises(RuntimeError) as excinfo:
+ backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ _base_local_dir = str(tmp_path),
+ )
+ assert "transformer_quant='fp8' could not be used" in str(excinfo.value)
+
+
[email protected]("speed_mode", [None, "default", "max"])
+def test_a_compiling_speed_mode_still_quantises(fake_runtime, tmp_path, monkeypatch, speed_mode):
+ """The guard must not cost the default path: speed unset is upgraded to a compile.
+
+ "off" is absent on purpose: not uncompilable, but the bit-exact request that rewrites an auto
+ quant to off well before this guard, which the test below pins.
+ """
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ speed_mode = speed_mode,
+ _base_local_dir = str(tmp_path),
+ )
+ assert status["transformer_quant"] == "fp8"
+ assert len(calls) == 1
+ backend.unload()
+
+
+def test_a_pipeline_pick_bakes_its_adapters_before_quantising(fake_runtime, tmp_path, monkeypatch):
+ """Adapters are baked before torchao replaces their dense base layers."""
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ order: list = []
+ _force_cuda_target(backend, monkeypatch)
+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(
+ dmod, "select_transformer_quant_scheme", lambda target, mode, family = None: "int8"
+ )
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_resolve_lora_set",
+ lambda self, specs, **kw: [("a", "/loras/a.safetensors", 0.8)],
+ )
+ real_init = _FakePipe.__init__
+
+ def _init(self):
+ real_init(self)
+ self.transformer = _FakeDenoiser()
+
+ monkeypatch.setattr(_FakePipe, "__init__", _init)
+
+ def _quantize(pipe, target, **kwargs):
+ order.append("quantize")
+ return "int8"
+
+ monkeypatch.setattr(dmod, "quantize_transformer", _quantize)
+
+ def _load_lora(
+ self,
+ path,
+ adapter_name = None,
+ ):
+ order.append(f"bake:{adapter_name}")
+
+ monkeypatch.setattr(_FakePipe, "load_lora_weights", _load_lora, raising = False)
+ monkeypatch.setattr(
+ _FakePipe,
+ "set_adapters",
+ lambda self, names, adapter_weights = None: None,
+ raising = False,
+ )
+ backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ loras = [("a", 0.8)],
+ _base_local_dir = str(tmp_path),
+ )
+ assert order == ["bake:a", "quantize"]
+ assert backend._state.pipe._unsloth_loras_baked is True
+ backend.unload()
+
+
+def test_a_unet_pipeline_is_never_quantised(fake_runtime, tmp_path, monkeypatch):
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch, denoisers = ())
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert "UNet" in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+
+
+ ("dtype", "expected"),
+ [
+ ("torch.uint8", "uint8"), # bitsandbytes packs NF4 into uint8 storage
+ ("torch.float8_e4m3fn", "float8_e4m3fn"), # a published fp8 repo
+ ("torch.float16", "float16"),
+ ],
+)
+def test_an_already_quantised_pipeline_is_never_requantised(
+ fake_runtime, tmp_path, monkeypatch, dtype, expected
+):
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ monkeypatch.setattr(_FakePipe, "__init__", _init_with_denoiser(dtype))
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert expected in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+
+
+def test_a_blocked_pipeline_still_refuses_an_explicit_scheme(fake_runtime, tmp_path, monkeypatch):
+ backend = DiffusionBackend()
+ _stub_pipeline_dense_quant(backend, monkeypatch, denoisers = ())
+ with pytest.raises(RuntimeError) as excinfo:
+ backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ _base_local_dir = str(tmp_path),
+ )
+ assert "transformer_quant='fp8' could not be used" in str(excinfo.value)
+ assert "UNet" in str(excinfo.value)
+
+
+def test_every_denoiser_is_quantised_or_none_is(fake_runtime, tmp_path, monkeypatch):
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(
+ backend, monkeypatch, denoisers = ("transformer", "unconditional_transformer")
+ )
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert status["transformer_quant"] == "fp8"
+ pipe = backend._state.pipe
+ assert [call["transformer"] for call in calls] == [
+ pipe.transformer,
+ pipe.unconditional_transformer,
+ ]
+ backend.unload()
+
+
+def test_a_partially_converted_transformer_fails_the_load(fake_runtime, tmp_path, monkeypatch):
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ _stub_pipeline_dense_quant(backend, monkeypatch, engages = None)
+ monkeypatch.setattr(dmod, "transformer_is_quantised", lambda module: True)
+ with pytest.raises(RuntimeError) as excinfo:
+ backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert "neither dense nor usable" in str(excinfo.value)
+
+
+def test_a_clean_decline_keeps_the_dense_transformer(fake_runtime, tmp_path, monkeypatch):
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ _stub_pipeline_dense_quant(backend, monkeypatch, engages = None)
+ monkeypatch.setattr(dmod, "transformer_is_quantised", lambda module: False)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert status["transformer_quant"] is None
+ assert status["resolved"]["transformer_quant"]["status"] == "applied"
+ backend.unload()
+
+
+def test_a_raw_fp8_pipeline_is_not_quantised_a_second_time(fake_runtime, tmp_path, monkeypatch):
+ """A local fp8 checkpoint is widened to bf16 on load; quantising it again compounds loss.
+
+ Non-GGUF loads are gated to unsloth/* or a LOCAL path, so the reachable shape is a user pointing
+ at their own fp8 conversion. test_diffusion_transformer_quant.py covers the header parse against
+ real safetensors; this is the loader wiring that stamps every denoiser for the blocker.
+ """
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ seen: list = []
+
+ def _stored(local_dir):
+ seen.append(local_dir)
+ return "fp8"
+
+ monkeypatch.setattr(dmod, "stored_denoiser_precision", _stored)
+ status = backend.load_pipeline(
+ "unsloth/Qwen-Image-2512",
+ model_kind = "pipeline",
+ _base_local_dir = str(tmp_path),
+ )
+ assert seen == [str(tmp_path)]
+ assert calls == []
+ assert status["transformer_quant"] is None
+ reason = status["resolved"]["transformer_quant"]["reason"]
+ assert "fp8" in reason and "widened to bf16" in reason
+ backend.unload()
+
+
+def test_a_bf16_pipeline_is_unaffected_by_the_header_probe(fake_runtime, tmp_path, monkeypatch):
+ """The probe must not cost the ordinary official-pipeline case."""
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ monkeypatch.setattr(dmod, "stored_denoiser_precision", lambda local_dir: None)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert len(calls) == 1
+ assert status["transformer_quant"] == "fp8"
+ backend.unload()
+
+
+def test_a_local_fp8_directory_is_scanned_through_the_load_base(
+ fake_runtime, tmp_path, monkeypatch
+):
+ """A local diffusers dir stages nothing, so the probe must read the base the load reads."""
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ seen: list = []
+
+ def _stored(local_dir):
+ seen.append(local_dir)
+ return "fp8" if local_dir else None
+
+ monkeypatch.setattr(dmod, "stored_denoiser_precision", _stored)
+ status = backend.load_pipeline(
+ "unsloth/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = None
+ )
+ # Not None: the fallback handed the probe the base the pipeline was actually loaded from.
+ assert seen and seen[0]
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert "fp8" in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+
+
def test_a_prequant_repo_missing_its_artifact_marks_the_plan_incomplete(monkeypatch):
"""The repo answers and holds NEITHER the primary nor the fallback name. That is not "no
prequant is used": this pick is configured to use one and the dense shards are already excluded
diff --git a/studio/backend/tests/test_diffusion_more_families.py b/studio/backend/tests/test_diffusion_more_families.py
index c5b59c13fe8..a7075b31910 100644
--- a/studio/backend/tests/test_diffusion_more_families.py
+++ b/studio/backend/tests/test_diffusion_more_families.py
@@ -355,6 +355,63 @@ def test_hidream_prequant_wiring():
assert family_prequant_repo(fam, scheme) == "unsloth/HiDream-I1-Full-FP8"
+def test_hidream_distilled_variants_have_no_hosted_prequant_to_inherit():
+ # Dev and Fast are distillations of Full, so the hosted Full checkpoint is baked from other
+ # weights. Inheriting it made a Dev / Fast pick plan the Full artifact, drop its own shards,
+ # download several GB and only then hit the base_model_id refusal.
+ from core.inference.diffusion_families import family_prequant_repo
+ for repo_id in ("HiDream-ai/HiDream-I1-Dev", "HiDream-ai/HiDream-I1-Fast"):
+ fam = detect_family(repo_id)
+ for scheme in ("int8", "fp8"):
+ assert family_prequant_repo(fam, scheme, base_repo = repo_id) is None
+ # However the id was typed, and through the mirror the loader actually fetches.
+ assert family_prequant_repo(fam, scheme, base_repo = f" {repo_id.upper()} ") is None
+ assert (
+ family_prequant_repo(
+ fam, scheme, base_repo = repo_id.replace("HiDream-ai", "unsloth")
+ )
+ is None
+ )
+
+
+def test_qwen_image_2512_routes_to_its_own_hosted_prequant():
+ # 2512 is a different checkpoint with its own baked artifacts. Falling back to the Qwen-Image ones
+ # made a 2512 pick plan an artifact base_model_id refuses, after its shards had been dropped.
+ from core.inference.diffusion_families import family_prequant_repo
+
+ fam = detect_family("Qwen/Qwen-Image-2512")
+ assert fam is not None and fam.name == "qwen-image"
+ for scheme in ("int8", "fp8"):
+ assert family_prequant_repo(fam, scheme) == "unsloth/Qwen-Image-FP8"
+ assert (
+ family_prequant_repo(fam, scheme, base_repo = "Qwen/Qwen-Image")
+ == "unsloth/Qwen-Image-FP8"
+ )
+ for base_repo in (
+ "Qwen/Qwen-Image-2512",
+ "unsloth/Qwen-Image-2512",
+ " QWEN/QWEN-IMAGE-2512 ",
+ ):
+ assert (
+ family_prequant_repo(fam, scheme, base_repo = base_repo)
+ == "unsloth/Qwen-Image-2512-FP8"
+ )
+
+
+def test_qwen_image_2512_prequant_filenames_match_its_repo():
+ # The filename derives from the repo name, so the variant repo must serve <Model>-<SCHEME>.pt.
+ from core.inference.diffusion_prequant import resolve_prequant_source
+ fam = detect_family("Qwen/Qwen-Image-2512")
+ for scheme, filename in (
+ ("int8", "Qwen-Image-2512-INT8.pt"),
+ ("fp8", "Qwen-Image-2512-FP8.pt"),
+ ):
+ source = resolve_prequant_source(fam, scheme, base_repo = "Qwen/Qwen-Image-2512")
+ assert source is not None
+ assert source.location == "unsloth/Qwen-Image-2512-FP8"
+ assert source.filename == filename
+
+
def test_hidream_quant_schemes_not_denied_and_no_extra_excludes():
# Measured on a B200: int8 and fp8 both engage and render cleanly, including 2-3 token prompts on int8. The routed
# MoE expert Linears only see the concatenated image+text stream (M >> 16), so torch._int_mm's minimum never binds.
diff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.py
new file mode 100644
index 00000000000..e8297b18887
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py
@@ -0,0 +1,942 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Hermetic tests for seeding an image PIPELINE pick with its hosted pre-quantized denoiser, which
+REPLACES the released ``transformer/`` shards. torch, diffusers and the Hub are stubbed."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import sys
+import types
+
+import pytest
+
+from core.inference import diffusion as dmod
+from core.inference import diffusion_prequant as pqmod
+from core.inference import diffusion_transformer_quant as tqmod
+from core.inference.diffusion import DiffusionBackend
+from core.inference.diffusion_auto_policy import DenseQuantEstimate
+from core.inference.diffusion_denoiser_prequant import PIPELINE_SEED_DECLINED
+from core.inference.diffusion_device import DiffusionDeviceTarget
+from core.inference.diffusion_families import detect_family_for_pick
+from core.inference.diffusion_memory import DeviceMemory
+
+MIB = 1024 * 1024
+
+Z_IMAGE_REPO = "Tongyi-MAI/Z-Image-Turbo"
+PREQUANT_REPO = "unsloth/Z-Image-Turbo-FP8"
+PREQUANT_FILE = "Z-Image-Turbo-FP8.pt"
+PREQUANT_BYTES = 6000 * MIB
+
+Z_IMAGE_INDEX = {
+ "_class_name": "ZImagePipeline",
+ "transformer": ["diffusers", "ZImageTransformer2DModel"],
+ "text_encoder": ["transformers", "Qwen3Model"],
+ "vae": ["diffusers", "AutoencoderKL"],
+ "tokenizer": ["transformers", "Qwen2Tokenizer"],
+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],
+}
+DENOISER_SHARDS = [
+ "transformer/diffusion_pytorch_model-00001-of-00002.safetensors",
+ "transformer/diffusion_pytorch_model-00002-of-00002.safetensors",
+]
+Z_IMAGE_FILES = [
+ ("model_index.json", 100),
+ ("transformer/config.json", 200),
+ (DENOISER_SHARDS[0], 11_000 * MIB),
+ (DENOISER_SHARDS[1], 11_000 * MIB),
+ ("text_encoder/model.safetensors", 7_672 * MIB),
+ ("vae/diffusion_pytorch_model.safetensors", 160 * MIB),
+ ("tokenizer/tokenizer_config.json", 300),
+ ("scheduler/scheduler_config.json", 300),
+]
+DENOISER_BYTES = 22_000 * MIB
+ALL_BYTES = sum(size for _name, size in Z_IMAGE_FILES)
+
+
+def _family():
+ fam = detect_family_for_pick(Z_IMAGE_REPO, None, None)
+ assert fam is not None and fam.name == "z-image"
+ return fam
+
+
+def _target(*, device = "cuda", ordinal = None) -> DiffusionDeviceTarget:
+ return DiffusionDeviceTarget(
+ device = device,
+ dtype = "bfloat16",
+ backend = device,
+ vendor = "nvidia",
+ supports_model_cpu_offload = True,
+ supports_default_torch_compile = True,
+ supports_pinned_transfer = True,
+ ordinal = ordinal,
+ )
+
+
+def _estimate(prequant = True) -> DenseQuantEstimate:
+ return DenseQuantEstimate(
+ scheme = "fp8",
+ steady_transformer_mib = 11_500,
+ transient_transformer_mib = 11_500 if prequant else 23_000,
+ companions_mib = 7_500,
+ prequant = prequant,
+ download_transformer_mib = 23_000,
+ text_encoders_mib = 7_320,
+ )
+
+
+def hub(monkeypatch, tmp_path):
+ """The pipeline repo plus the hosted checkpoint's repo, as ``model_info`` sees them."""
+ repos = {
+ Z_IMAGE_REPO: Z_IMAGE_FILES,
+ PREQUANT_REPO: [(PREQUANT_FILE, PREQUANT_BYTES), ("README.md", 10)],
+ }
+ manifest = tmp_path / "model_index.json"
+ manifest.write_text(json.dumps(Z_IMAGE_INDEX), encoding = "utf-8")
+ info_calls: list = []
+
+ class _Api:
+ def __init__(self, *_a, **_k) -> None:
+ pass
+
+ def model_info(self, repo_id, **_kwargs):
+ info_calls.append(repo_id)
+ return types.SimpleNamespace(
+ siblings = [
+ types.SimpleNamespace(rfilename = name, size = size)
+ for name, size in repos[repo_id]
+ ],
+ sha = "c" * 40,
+ )
+
+ monkeypatch.setattr("huggingface_hub.HfApi", _Api)
+ monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)
+ monkeypatch.setattr(
+ "huggingface_hub.hf_hub_download", lambda repo_id, filename, **kw: str(manifest)
+ )
+ return info_calls
+
+
+def _estimate_bytes(**overrides):
+ kwargs = {
+ "kind": "pipeline",
+ "pipeline_components": None,
+ "single_file_is_pipeline": False,
+ **overrides,
+ }
+ return DiffusionBackend._estimate_download_bytes(
+ Z_IMAGE_REPO, None, Z_IMAGE_REPO, None, **kwargs
+ )
+
+
+def test_the_plan_drops_the_released_denoiser_shards_and_keeps_its_config(hub):
+ """The pull drops the denoiser shards but keeps the config the seeding loader meta-inits from."""
+ skipped: list[str] = []
+ resident: list = []
+ total, files = _estimate_bytes(
+ skip_transformer_weights = True,
+ skipped_files_out = skipped,
+ resident_file_sizes_out = resident,
+ )
+ assert "transformer/config.json" in files
+ assert not [f for f in files if f in DENOISER_SHARDS]
+ assert skipped == DENOISER_SHARDS
+ assert total == ALL_BYTES - DENOISER_BYTES
+ assert not [name for name, _size in resident if name.startswith("transformer/")]
+
+
+def test_the_plan_keeps_the_shards_when_nothing_is_seeded(hub):
+ skipped: list[str] = []
+ total, files = _estimate_bytes(skipped_files_out = skipped)
+ assert all(shard in files for shard in DENOISER_SHARDS)
+ assert skipped == []
+ assert total == ALL_BYTES
+
+
+def _plan_backend(
+ monkeypatch,
+ *,
+ planned,
+ mismatch = None,
+):
+ backend = DiffusionBackend()
+ monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: _target())
+ monkeypatch.setattr(dmod, "detect_family_for_pick", lambda *_a, **_k: _family())
+ monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)
+ monkeypatch.setattr(dmod, "_assert_base_repo_accessible", lambda *_a, **_k: None)
+ monkeypatch.setattr(DiffusionBackend, "_te_prequant_plan_files", lambda *_a, **_k: {})
+ monkeypatch.setattr(dmod, "flux2_pick_mismatch", lambda *_a, **_k: mismatch)
+ monkeypatch.setattr(dmod, "speech_pick_refusal", lambda *_a, **_k: None)
+ monkeypatch.setattr(
+ DiffusionBackend, "_pipeline_planned_denoiser_scheme", lambda *_a, **_k: planned
+ )
+ monkeypatch.setattr(
+ DiffusionBackend, "_files_already_cached", staticmethod(lambda *_a, **_k: set())
+ )
+ monkeypatch.setattr(
+ DiffusionBackend, "_hub_file_is_cached", staticmethod(lambda *_a, **_k: False)
+ )
+ monkeypatch.setattr(DiffusionBackend, "declared_footprint_shortfall", lambda *_a, **_k: None)
+ return backend
+
+
+def test_the_plan_counts_and_stages_the_hosted_checkpoint(monkeypatch, hub):
+ backend = _plan_backend(monkeypatch, planned = "fp8")
+ plan = backend.download_plan(Z_IMAGE_REPO, model_kind = "pipeline")
+
+ entries = {entry["repo_id"]: entry for entry in plan["entries"]}
+ assert entries[PREQUANT_REPO]["files"] == [PREQUANT_FILE]
+ assert entries[PREQUANT_REPO]["bytes"] == PREQUANT_BYTES
+ assert not [f for f in entries[Z_IMAGE_REPO]["files"] if f in DENOISER_SHARDS]
+ assert "transformer/config.json" in entries[Z_IMAGE_REPO]["files"]
+ assert plan["required_bytes"] == ALL_BYTES - DENOISER_BYTES + PREQUANT_BYTES
+ assert entries[Z_IMAGE_REPO]["checkpoint"] is True
+ assert entries[PREQUANT_REPO]["checkpoint"] is False
+
+
+def test_an_unplanned_pipeline_pick_plans_exactly_as_before(monkeypatch, hub):
+ """With no hosted artifact the released shards stay in the pull and the plan is unchanged."""
+ backend = _plan_backend(monkeypatch, planned = None)
+ plan = backend.download_plan(Z_IMAGE_REPO, model_kind = "pipeline")
+
+ entries = {entry["repo_id"]: entry for entry in plan["entries"]}
+ assert PREQUANT_REPO not in entries
+ assert all(shard in entries[Z_IMAGE_REPO]["files"] for shard in DENOISER_SHARDS)
+ assert plan["required_bytes"] == ALL_BYTES
+
+
+def test_a_declined_plan_keeps_the_released_shards(monkeypatch, hub):
+ backend = _plan_backend(monkeypatch, planned = PIPELINE_SEED_DECLINED)
+ plan = backend.download_plan(Z_IMAGE_REPO, model_kind = "pipeline")
+
+ entries = {entry["repo_id"]: entry for entry in plan["entries"]}
+ assert PREQUANT_REPO not in entries
+ assert all(shard in entries[Z_IMAGE_REPO]["files"] for shard in DENOISER_SHARDS)
+
+
+def test_the_artifact_carries_the_denoiser_share_of_the_unified_memory_verdict(monkeypatch, hub):
+ """The unified-memory refusal prices the denoiser from the artifact, not the dropped shards."""
+ backend = _plan_backend(monkeypatch, planned = "fp8")
+ seen: dict = {}
+
+ def _shortfall(_self, *_a, **kwargs):
+ seen.update(kwargs)
+ return None
+
+ monkeypatch.setattr(DiffusionBackend, "declared_footprint_shortfall", _shortfall)
+ backend.download_plan(Z_IMAGE_REPO, model_kind = "pipeline")
+ assert seen["prequant_bytes"] == PREQUANT_BYTES
+ assert not [name for name, _s in seen["declared_files"] if name.startswith("transformer/")]
+
+
+def test_a_gguf_pick_still_never_downloads_a_second_denoiser(monkeypatch):
+ """A GGUF pick never sizes an uncached hosted checkpoint: there it is a SECOND denoiser."""
+ backend = DiffusionBackend()
+ monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: _target())
+ monkeypatch.setattr(dmod, "_uncached_prequant_repo", lambda *_a, **_k: PREQUANT_REPO)
+
+ def _never(*_a, **_k):
+ raise AssertionError("a GGUF pick sized an uncached hosted checkpoint")
+
+ monkeypatch.setattr(DiffusionBackend, "_prequant_source_hub_entry", staticmethod(_never))
+ assert (
+ backend._dit_prequant_plan_source(
+ _family(), "gguf", None, {"base_repo": Z_IMAGE_REPO, "transformer_quant": None}
+ )
+ is None
+ )
+
+
+def test_a_planned_pipeline_pick_is_sized_from_the_hub(monkeypatch, hub):
+ backend = DiffusionBackend()
+ monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: _target())
+ monkeypatch.setattr(pqmod, "restricted_prequant_load_supported", lambda _scheme: True)
+ assert backend._dit_prequant_plan_source(
+ _family(),
+ "pipeline",
+ None,
+ {"base_repo": Z_IMAGE_REPO, "_pipeline_prequant_planned": "fp8"},
+ ) == (PREQUANT_REPO, PREQUANT_FILE, PREQUANT_BYTES)
+
+
+def test_a_pipeline_pick_with_no_planned_scheme_sizes_nothing(monkeypatch, hub):
+ backend = DiffusionBackend()
+ for planned in (None, PIPELINE_SEED_DECLINED):
+ assert (
+ backend._dit_prequant_plan_source(
+ _family(),
+ "pipeline",
+ None,
+ {"base_repo": Z_IMAGE_REPO, "_pipeline_prequant_planned": planned},
+ )
+ is None
+ )
+
+
+def _settle_backend(
+ monkeypatch,
+ *,
+ offload = "none",
+ scheme = "fp8",
+ candidate = None,
+):
+ backend = DiffusionBackend()
+ monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: _target())
+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda _t: True)
+ monkeypatch.setattr(dmod, "_pipeline_quant_uncompilable_reason", lambda *_a, **_k: None)
+ monkeypatch.setattr(
+ dmod, "select_transformer_quant_scheme", lambda target, mode, family = None: scheme
+ )
+ monkeypatch.setattr(pqmod, "restricted_prequant_load_supported", lambda _scheme: True)
+ monkeypatch.setattr(
+ dmod,
+ "resolve_dense_quant_candidate",
+ lambda **_k: _estimate() if candidate is None else candidate,
+ )
+ monkeypatch.setattr(
+ dmod,
+ "snapshot_device_memory",
+ lambda _t: DeviceMemory("cuda", "cuda", "dedicated", 1_000, 180_000),
+ )
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **_k: types.SimpleNamespace(offload_policy = offload),
+ )
+ return backend
+
+
+def _settle(backend, **overrides):
+ kwargs = {
+ "base": Z_IMAGE_REPO,
+ "kind": "pipeline",
+ "transformer_quant": None,
+ "speed_mode": None,
+ "repo_id": Z_IMAGE_REPO,
+ **overrides,
+ }
+ return backend._pipeline_planned_denoiser_scheme(_family(), **kwargs)
+
+
+def test_an_unset_precision_defaults_to_the_hosted_checkpoint(monkeypatch):
+ assert _settle(_settle_backend(monkeypatch)) == "fp8"
+
+
+def test_an_ampere_host_defaults_to_the_hosted_int8_checkpoint(monkeypatch):
+ assert _settle(_settle_backend(monkeypatch, scheme = "int8")) == "int8"
+
+
[email protected]("quant", ["none", "off"])
+def test_precision_off_keeps_the_released_weights(monkeypatch, quant):
+ assert _settle(_settle_backend(monkeypatch), transformer_quant = quant) is None
+
+
+def test_speed_off_keeps_the_released_weights(monkeypatch):
+ """Speed=off keeps the released weights: it is a bit-exact contract."""
+ assert _settle(_settle_backend(monkeypatch), speed_mode = "off") is None
+
+
+def test_an_explicit_scheme_under_speed_off_still_seeds(monkeypatch):
+ """Speed=off only silences an AUTO precision. An explicit scheme is still quantized and upgrades
+ the speed to `default`, so it must seed the hosted checkpoint rather than pull the bf16 shards."""
+ backend = _settle_backend(monkeypatch)
+ assert _settle(backend, transformer_quant = "fp8", speed_mode = "off") == "fp8"
+
+
+def test_a_baked_lora_keeps_the_dense_path(monkeypatch):
+ """A LoRA bake keeps the dense path; an all-zero list bakes nothing and is unaffected."""
+ backend = _settle_backend(monkeypatch)
+ assert _settle(backend, loras = [("adapter", 0.8)]) is None
+ assert _settle(backend, loras = [("adapter", 0.0)]) == "fp8"
+
+
[email protected]("mode", ["balanced", "low_vram"])
+def test_a_definite_offload_request_keeps_the_released_weights(monkeypatch, mode):
+ assert _settle(_settle_backend(monkeypatch), memory_mode = mode) is None
+
+
+def test_an_artifact_sized_plan_that_still_offloads_declines(monkeypatch):
+ """An artifact-sized plan that still offloads pins a decline: torchao rejects offload hooks."""
+ backend = _settle_backend(monkeypatch, offload = "sequential")
+ assert _settle(backend) == PIPELINE_SEED_DECLINED
+
+
+def _settle_backend_walking(monkeypatch, *, artifacts: tuple, candidates: tuple):
+ """A backend whose memory verdict depends on the rung: an int8-sized plan (31 GB) offloads, an
+ fp8-sized one (19 GB) stays resident. ``artifacts`` names the schemes with a hosted file,
+ ``candidates`` the auto ladder."""
+ from core.inference import diffusion_transformer_quant as tq
+
+ backend = _settle_backend(monkeypatch, scheme = candidates[0])
+ monkeypatch.setattr(tq, "auto_scheme_candidates", lambda target, family = None: candidates)
+ monkeypatch.setattr(
+ dmod,
+ "denoiser_prequant_source",
+ lambda fam, scheme, **_k: ("unsloth/Qwen-Image-FP8", f"{scheme}.pt")
+ if scheme in artifacts
+ else None,
+ )
+ monkeypatch.setattr(
+ dmod,
+ "resolve_dense_quant_candidate",
+ lambda **k: DenseQuantEstimate(
+ scheme = k["requested"],
+ steady_transformer_mib = 31_000 if k["requested"] == "int8" else 19_000,
+ transient_transformer_mib = 0,
+ companions_mib = 7_500,
+ prequant = True,
+ download_transformer_mib = 0,
+ text_encoders_mib = 7_320,
+ ),
+ )
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **k: types.SimpleNamespace(
+ offload_policy = "sequential"
+ if k["transformer_resident_override_mib"] >= 31_000
+ else "none"
+ ),
+ )
+ return backend
+
+
+def test_an_artifact_too_large_for_the_card_yields_to_the_next_hosted_rung(monkeypatch):
+ """int8 leads the ladder, but Qwen-Image's int8 file is 12 GB larger than its fp8 one: where the
+ int8-sized plan offloads, auto seeds the fp8 artifact instead of pinning a decline."""
+ backend = _settle_backend_walking(
+ monkeypatch, artifacts = ("int8", "fp8"), candidates = ("int8", "fp8")
+ )
+ assert _settle(backend) == "fp8"
+
+
+def test_a_walk_with_no_resident_rung_declines(monkeypatch):
+ """Every hosted rung offloads: the decline is pinned so plan and load agree."""
+ backend = _settle_backend_walking(monkeypatch, artifacts = ("int8",), candidates = ("int8", "fp8"))
+ assert _settle(backend) == PIPELINE_SEED_DECLINED
+
+
+def test_an_explicit_scheme_is_never_swapped_for_a_lower_rung(monkeypatch):
+ """An explicit int8 that offloads declines; auto's walk is not offered to an explicit request."""
+ backend = _settle_backend_walking(
+ monkeypatch, artifacts = ("int8", "fp8"), candidates = ("int8", "fp8")
+ )
+ assert _settle(backend, transformer_quant = "int8") == PIPELINE_SEED_DECLINED
+
+
+def test_a_family_with_no_hosted_artifact_falls_through(monkeypatch):
+ """A scheme with no hosted artifact falls through to the in-memory quantise."""
+ assert _settle(_settle_backend(monkeypatch, scheme = "nvfp4")) is None
+
+
+def test_a_base_with_no_hosted_artifact_falls_through(monkeypatch):
+ """An excluded base (the undistilled Z-Image) quantises its own weights."""
+ assert _settle(_settle_backend(monkeypatch), base = "Tongyi-MAI/Z-Image") is None
+
+
+def test_a_forced_fp8_accumulate_the_artifact_cannot_bake_keeps_the_released_weights(monkeypatch):
+ """The hosted fp8 checkpoints bake fast accumulate, and ``_validate_checkpoint`` refuses a baked
+ value differing from a FORCED one, so seeding drops the shards for a checkpoint the load rejects."""
+ backend = _settle_backend(monkeypatch)
+ assert _settle(backend, transformer_quant = "fp8", fast_accum = False) is None
+ assert _settle(backend, transformer_quant = "fp8", fast_accum = True) == "fp8"
+ assert _settle(backend, transformer_quant = "fp8") == "fp8"
+
+
+def test_a_forced_accumulate_never_blocks_a_scheme_that_bakes_none(monkeypatch):
+ """Only fp8 records ``fast_accum``, so the int8 artifact is seeded whatever the caller forces."""
+ backend = _settle_backend(monkeypatch, scheme = "int8")
+ assert _settle(backend, transformer_quant = "int8", fast_accum = False) == "int8"
+
+
+def test_a_gguf_pick_is_never_settled_here(monkeypatch):
+ assert _settle(_settle_backend(monkeypatch), kind = "gguf") is None
+
+
[email protected]("family", ["krea-2", "ideogram-4"])
+def test_a_per_component_assembler_is_never_seeded(monkeypatch, family):
+ """A per-component assembler is never seeded: it never sees ``pipe_kwargs``."""
+ backend = _settle_backend(monkeypatch)
+ fam = types.SimpleNamespace(name = family, base_repo = "x/y")
+ assert (
+ backend._pipeline_planned_denoiser_scheme(
+ fam,
+ base = "x/y",
+ kind = "pipeline",
+ transformer_quant = None,
+ speed_mode = None,
+ )
+ is None
+ )
+
+
+def test_an_uncompilable_host_keeps_the_released_weights(monkeypatch):
+ """An uncompilable host keeps the released weights, since the loader would refuse the seed."""
+ backend = _settle_backend(monkeypatch)
+ monkeypatch.setattr(dmod, "_pipeline_quant_uncompilable_reason", lambda *_a, **_k: "no triton")
+ assert _settle(backend) is None
+
+
+def test_an_unanswerable_probe_keeps_the_released_weights(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+
+ def _boom(*_a, **_k):
+ raise RuntimeError("driver went away")
+
+ monkeypatch.setattr(dmod, "select_transformer_quant_scheme", _boom)
+ assert _settle(backend) is None
+
+
+def _run_load_backend(
+ monkeypatch,
+ *,
+ planned,
+ verified = True,
+):
+ backend = DiffusionBackend()
+ backend._load_token = 1
+ backend._loading = dmod._LoadingState(repo_id = Z_IMAGE_REPO, base_repo = Z_IMAGE_REPO)
+ monkeypatch.setattr(dmod, "detect_family_for_pick", lambda *_a, **_k: _family())
+ monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)
+ monkeypatch.setattr(dmod, "_assert_base_repo_accessible", lambda *_a, **_k: None)
+ monkeypatch.setattr(dmod, "assert_flux2_pick_compatible", lambda *_a, **_k: None)
+ monkeypatch.setattr(dmod, "assert_pick_is_not_speech", lambda *_a, **_k: None)
+ monkeypatch.setattr(dmod, "_local_base_transformer_present", lambda *_a, **_k: False)
+ monkeypatch.setattr(DiffusionBackend, "_te_prequant_plan_files", lambda *_a, **_k: {})
+ monkeypatch.setattr(DiffusionBackend, "declared_footprint_shortfall", lambda *_a, **_k: None)
+ monkeypatch.setattr(
+ DiffusionBackend, "_pipeline_planned_denoiser_scheme", lambda *_a, **_k: planned
+ )
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_dit_prequant_plan_source",
+ lambda *_a, **_k: (PREQUANT_REPO, PREQUANT_FILE, PREQUANT_BYTES)
+ if verified and planned not in (None, PIPELINE_SEED_DECLINED)
+ else None,
+ )
+ fetched: list = []
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_fetch_denoiser_prequant",
+ lambda _self, entry, *_a, **_k: fetched.append(entry),
+ )
+ monkeypatch.setattr(DiffusionBackend, "_prefetch_files", lambda *_a, **_k: None)
+ seen: dict = {}
+ monkeypatch.setattr(backend, "load_pipeline", lambda **kwargs: seen.update(kwargs))
+ return backend, seen, fetched
+
+
+def test_the_load_is_pinned_to_the_plan_that_scoped_the_pull(monkeypatch, hub):
+ backend, seen, fetched = _run_load_backend(monkeypatch, planned = "fp8")
+ backend._run_load(repo_id = Z_IMAGE_REPO, model_kind = "pipeline", _load_token = 1)
+
+ assert backend._loading is None, getattr(backend._loading, "error", None)
+ assert seen["_pipeline_prequant_planned"] == "fp8"
+ assert seen["_pipeline_prequant_skipped"] == tuple(DENOISER_SHARDS)
+ assert fetched == [(PREQUANT_REPO, PREQUANT_FILE, PREQUANT_BYTES)]
+ assert PREQUANT_REPO in backend.loading_repo_ids() or backend._loading is None
+
+
+def test_an_artifact_that_does_not_resolve_keeps_the_released_shards(monkeypatch, hub):
+ """An artifact that does not resolve on the Hub keeps the released shards in the pull."""
+ backend, seen, fetched = _run_load_backend(monkeypatch, planned = "fp8", verified = False)
+ backend._run_load(repo_id = Z_IMAGE_REPO, model_kind = "pipeline", _load_token = 1)
+
+ assert seen["_pipeline_prequant_planned"] is None
+ assert seen["_pipeline_prequant_skipped"] == ()
+ assert fetched == []
+
+
+def test_a_local_checkpoint_is_still_seeded_on_an_online_load(monkeypatch, hub):
+ """An operator's own checkpoint has no Hub entry, so the plan drops no shards for it; the seed
+ must still survive the pull, or the load quantises bf16 in memory and ignores the given file."""
+ backend, seen, fetched = _run_load_backend(monkeypatch, planned = "fp8", verified = False)
+ monkeypatch.setattr(dmod, "denoiser_prequant_cached", lambda *_a, **_k: True)
+ backend._run_load(
+ repo_id = Z_IMAGE_REPO,
+ model_kind = "pipeline",
+ transformer_prequant_path = "/models/z-image-fp8.pt",
+ _load_token = 1,
+ )
+
+ assert seen["_pipeline_prequant_planned"] == "fp8"
+ assert seen["_pipeline_prequant_skipped"] == ()
+ assert fetched == []
+
+
+def test_a_local_path_the_loader_would_refuse_keeps_the_released_shards(monkeypatch, hub):
+ """The seed rides on the artifact being there: a path that does not resolve keeps the bf16
+ shards rather than pinning a seed the load cannot take."""
+ backend, seen, _fetched = _run_load_backend(monkeypatch, planned = "fp8", verified = False)
+ monkeypatch.setattr(dmod, "denoiser_prequant_cached", lambda *_a, **_k: False)
+ backend._run_load(
+ repo_id = Z_IMAGE_REPO,
+ model_kind = "pipeline",
+ transformer_prequant_path = "/models/missing.pt",
+ _load_token = 1,
+ )
+
+ assert seen["_pipeline_prequant_planned"] is None
+
+
+def test_the_decline_is_pinned_across_plan_and_load(monkeypatch, hub):
+ """A decline is pinned from plan to load, so the loader never re-takes it and fetches inline."""
+ backend, seen, _fetched = _run_load_backend(monkeypatch, planned = PIPELINE_SEED_DECLINED)
+ backend._run_load(repo_id = Z_IMAGE_REPO, model_kind = "pipeline", _load_token = 1)
+
+ assert seen["_pipeline_prequant_planned"] == PIPELINE_SEED_DECLINED
+ assert seen["_pipeline_prequant_skipped"] == ()
+
+
+def test_an_offline_load_never_probes_the_hub_for_an_artifact(monkeypatch):
+ """An offline load answers from the cache alone, so it never probes the Hub."""
+ backend, seen, fetched = _run_load_backend(monkeypatch, planned = "fp8")
+
+ def _never(*_a, **_k):
+ raise AssertionError("the offline path asked the Hub about a checkpoint")
+
+ monkeypatch.setattr(DiffusionBackend, "_dit_prequant_plan_source", _never)
+ monkeypatch.setattr(dmod, "denoiser_prequant_cached", lambda *_a, **_k: False)
+ backend._run_load(
+ repo_id = Z_IMAGE_REPO, model_kind = "pipeline", local_files_only = True, _load_token = 1
+ )
+ assert seen["_pipeline_prequant_planned"] is None
+ assert fetched == []
+
+
+def test_an_offline_reload_seeds_from_the_cached_artifact(monkeypatch):
+ """The cache the first load built is reusable: an API reload seeds instead of assembling bf16
+ from a snapshot whose released denoiser shards that load deliberately left out."""
+ backend, seen, fetched = _run_load_backend(monkeypatch, planned = "fp8")
+
+ def _never(*_a, **_k):
+ raise AssertionError("the offline path asked the Hub about a checkpoint")
+
+ monkeypatch.setattr(DiffusionBackend, "_dit_prequant_plan_source", _never)
+ monkeypatch.setattr(dmod, "denoiser_prequant_cached", lambda *_a, **_k: True)
+ backend._run_load(
+ repo_id = Z_IMAGE_REPO, model_kind = "pipeline", local_files_only = True, _load_token = 1
+ )
+ assert seen["_pipeline_prequant_planned"] == "fp8"
+ assert seen["_pipeline_prequant_skipped"] == ()
+ assert fetched == []
+
+
+class _FakePipe:
+ def __init__(self) -> None:
+ self.transformer = object()
+
+ def to(self, *_a, **_k):
+ return self
+
+
+class _FakePipeline:
+ last: dict = {}
+
+ @classmethod
+ def from_pretrained(cls, base, **kwargs):
+ _FakePipeline.last = {"base": base, **kwargs}
+ return _FakePipe()
+
+
+class _FakeTransformer:
+ pass
+
+
+def fake_runtime(monkeypatch):
+ torch = types.ModuleType("torch")
+ torch.bfloat16 = "bfloat16"
+ torch.float16 = "float16"
+ torch.float32 = "float32"
+ torch.cuda = types.SimpleNamespace(is_available = lambda: False)
+ torch.backends = types.SimpleNamespace(mps = None)
+ torch.inference_mode = lambda: contextlib.nullcontext()
+
+ diffusers = types.ModuleType("diffusers")
+ diffusers.ZImagePipeline = _FakePipeline
+ diffusers.ZImageTransformer2DModel = _FakeTransformer
+ diffusers.GGUFQuantizationConfig = lambda compute_dtype = None: ("quant", compute_dtype)
+
+ monkeypatch.setitem(sys.modules, "torch", torch)
+ monkeypatch.setitem(sys.modules, "diffusers", diffusers)
+ monkeypatch.setattr(dmod, "clear_gpu_cache", lambda: None)
+ _FakePipeline.last = {}
+ yield
+
+
+def _load_backend(
+ monkeypatch,
+ *,
+ seeded = True,
+ offload = "none",
+):
+ """A backend whose pipeline assembly, memory plan and quantiser are all observable."""
+ backend = DiffusionBackend()
+ monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: _target())
+ monkeypatch.setattr(dmod, "apply_diffusion_device_ordinal", lambda _t: None)
+ monkeypatch.setattr(dmod, "prefer_ungated_mirror", lambda base, *_a, **_k: base)
+ monkeypatch.setattr(dmod, "apply_memory_plan", lambda *_a, **_k: ("none", False))
+ monkeypatch.setattr(dmod, "raise_on_unified_memory_shortfall", lambda *_a, **_k: None)
+ monkeypatch.setattr(DiffusionBackend, "_resident_sized_plan", lambda _s, plan, *_a, **_k: plan)
+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda _t: True)
+ monkeypatch.setattr(tqmod, "dense_transformer_supported", lambda _t: True)
+ monkeypatch.setattr(
+ dmod, "select_transformer_quant_scheme", lambda target, mode, family = None: "fp8"
+ )
+ monkeypatch.setattr(dmod, "dense_quant_blocker", lambda _pipe: None)
+ monkeypatch.setattr(dmod, "_pipeline_quant_uncompilable_reason", lambda *_a, **_k: None)
+ monkeypatch.setattr(dmod, "stored_denoiser_precision", lambda *_a, **_k: None)
+ monkeypatch.setattr(dmod, "denoiser_modules", lambda pipe: [("transformer", object())])
+ monkeypatch.setattr(pqmod, "restricted_prequant_load_supported", lambda _scheme: True)
+
+ plans: list = []
+
+ def _plan_memory(_self, *_a, **kwargs):
+ plans.append(kwargs)
+ policy = offload if kwargs.get("transformer_resident_override_mib") is not None else "none"
+ return types.SimpleNamespace(
+ offload_policy = policy, requested_mode = None, estimates = {}, reasons = ()
+ )
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan_memory)
+
+ quantised: list = []
+
+ def _quantize(pipe, target, **kwargs):
+ quantised.append(kwargs.get("mode"))
+ return "fp8"
+
+ monkeypatch.setattr(dmod, "quantize_transformer", _quantize)
+
+ seeds: list = []
+
+ def _load_prequantized(_cls, base, source, **kwargs):
+ seeds.append({"base": base, "source": source, **kwargs})
+ return _FakeTransformer() if seeded else None
+
+ monkeypatch.setattr(pqmod, "load_prequantized_transformer", _load_prequantized)
+
+ restored: list = []
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_prefetch_files",
+ lambda _self, repo_id, gguf, base, files, *_a, **_k: restored.append(tuple(files)),
+ )
+ return backend, types.SimpleNamespace(
+ plans = plans, quantised = quantised, seeds = seeds, restored = restored
+ )
+
+
+def _load(backend, **overrides):
+ kwargs = {
+ "model_kind": "pipeline",
+ "_pipeline_prequant_planned": "fp8",
+ "_pipeline_prequant_skipped": tuple(DENOISER_SHARDS),
+ **overrides,
+ }
+ return backend.load_pipeline(Z_IMAGE_REPO, **kwargs)
+
+
+def test_a_seeded_denoiser_engages_the_quant_without_a_second_conversion(fake_runtime, monkeypatch):
+ backend, spy = _load_backend(monkeypatch)
+ status = _load(backend)
+
+ assert isinstance(_FakePipeline.last["transformer"], _FakeTransformer)
+ assert status["transformer_quant"] == "fp8"
+ assert spy.quantised == []
+ assert spy.restored == []
+ seed = spy.seeds[0]
+ assert seed["source"].location == PREQUANT_REPO
+ assert seed["source"].filename == PREQUANT_FILE
+ assert seed["scheme"] == "fp8"
+ assert seed["min_features"] == dmod.DEFAULT_MIN_LINEAR_FEATURES
+ assert seed["cache_dir"] == dmod.hub_cache_dir()
+
+
+def test_the_resolved_record_names_the_hosted_file(fake_runtime, monkeypatch):
+ """``resolved`` names the hosted FILE, which renders differently from a runtime quantise."""
+ backend, _spy = _load_backend(monkeypatch)
+ resolved = _load(backend)["resolved"]["transformer_quant"]
+
+ assert resolved["value"] == "fp8"
+ assert resolved["artifact"] == f"prequant:{PREQUANT_REPO}/{PREQUANT_FILE}"
+ assert PREQUANT_FILE in resolved["reason"]
+ # `source` must stay "auto"/"explicit": the frontend branches on it.
+ assert resolved["source"] == "auto"
+
+
+def test_a_seed_that_does_not_land_replans_and_tops_up_the_shards(fake_runtime, monkeypatch):
+ """A failed seed re-plans at bf16 and restores shards from_pretrained cannot re-fetch."""
+ backend, spy = _load_backend(monkeypatch, seeded = False)
+ status = _load(backend)
+
+ assert "transformer" not in _FakePipeline.last
+ assert spy.restored and set(DENOISER_SHARDS) <= set(spy.restored[0])
+ assert spy.quantised == ["auto"]
+ assert status["transformer_quant"] == "fp8"
+ assert status["resolved"]["transformer_quant"].get("artifact") is None
+
+
+def test_a_top_up_that_spans_cache_roots_assembles_from_the_hub_id(fake_runtime, monkeypatch):
+ """A cache-folder change can leave the manifest in the old root and the restored shards in the
+ live one. No snapshot then holds both, so assembly must fall back to the hub id: from_pretrained
+ would treat the staged directory as terminal with no transformer in it."""
+ backend, _spy = _load_backend(monkeypatch, seeded = False)
+ monkeypatch.setattr(DiffusionBackend, "_prefetch_files", lambda *_a, **_k: None)
+ _load(backend, _base_local_dir = "/old/root/snapshots/abc")
+
+ assert _FakePipeline.last["base"] == Z_IMAGE_REPO
+
+
+def test_a_top_up_never_promotes_a_snapshot_the_staging_did_not_hand_back(
+ fake_runtime, monkeypatch
+):
+ """The top-up stages the shards only. A snapshot the staging never returned holds them without
+ the companions, so it must not become the directory the pipeline assembles from."""
+ backend, _spy = _load_backend(monkeypatch, seeded = False)
+ monkeypatch.setattr(
+ DiffusionBackend, "_prefetch_files", lambda *_a, **_k: "/live/root/snapshots/abc"
+ )
+ _load(backend, _base_local_dir = None)
+
+ assert _FakePipeline.last["base"] == Z_IMAGE_REPO
+
+
+def test_a_single_root_top_up_keeps_assembling_from_the_staged_snapshot(fake_runtime, monkeypatch):
+ """The ordinary case: the shards land in the staged snapshot, which keeps from_pretrained off
+ the hub."""
+ backend, _spy = _load_backend(monkeypatch, seeded = False)
+ monkeypatch.setattr(
+ DiffusionBackend, "_prefetch_files", lambda *_a, **_k: "/live/root/snapshots/abc"
+ )
+ _load(backend, _base_local_dir = "/live/root/snapshots/abc")
+
+ assert _FakePipeline.last["base"] == "/live/root/snapshots/abc"
+
+
+def test_an_artifact_sized_plan_that_offloads_at_load_time_drops_the_seed(
+ fake_runtime, monkeypatch
+):
+ """A plan that offloads at load time drops the seed: torchao tensors reject offload hooks."""
+ backend, spy = _load_backend(monkeypatch, offload = "sequential")
+ _load(backend)
+
+ assert spy.seeds == []
+ assert spy.restored and set(DENOISER_SHARDS) <= set(spy.restored[0])
+
+
+def test_the_artifact_label_names_the_file_that_really_loaded():
+ """A repo serving only its fallback filename is labelled with the fallback, not the primary."""
+ # _resolve_checkpoint_path falls back when the primary name is absent, so labelling from
+ # source.filename would publish provenance for a file nobody fetched.
+ import types
+
+ from core.inference.diffusion_denoiser_prequant import prequant_artifact_label
+
+ source = types.SimpleNamespace(
+ kind = "repo", location = "unsloth/Z-Image-Turbo-FP8", filename = "Z-Image-Turbo-FP8.pt"
+ )
+ assert prequant_artifact_label(source) == (
+ "prequant:unsloth/Z-Image-Turbo-FP8/Z-Image-Turbo-FP8.pt"
+ )
+ loaded = types.SimpleNamespace(_unsloth_prequant_path = "/cache/blobs/Z-Image-Turbo-fp8.pt")
+ assert prequant_artifact_label(source, loaded) == (
+ "prequant:unsloth/Z-Image-Turbo-FP8/Z-Image-Turbo-fp8.pt"
+ )
+ local = types.SimpleNamespace(kind = "path", location = "/models/mine.pt", filename = None)
+ assert prequant_artifact_label(local, loaded) == "prequant:/models/mine.pt"
+
+
+def test_a_dropped_seed_replans_once_the_dense_shards_are_back(fake_runtime, monkeypatch):
+ """The plan the dense load runs on is taken AFTER the skipped transformer shards are restored."""
+ # A pipeline plan prices CACHED bytes, so one taken while transformer/ was skipped saw companions
+ # only: left in place it reads 'none' and the load keeps the bf16 denoiser resident.
+ backend, spy = _load_backend(monkeypatch, offload = "sequential")
+ _load(backend)
+
+ assert spy.seeds == []
+ assert spy.restored and set(DENOISER_SHARDS) <= set(spy.restored[0])
+ assert len(spy.plans) == 3
+ assert spy.plans[-1].get("transformer_resident_override_mib") is None
+
+
+def test_nothing_skipped_takes_no_extra_plan(fake_runtime, monkeypatch):
+ """A load that skipped no shards re-plans nothing: there is no under-counted plan to redo."""
+ backend, spy = _load_backend(monkeypatch, offload = "sequential")
+ _load(backend, _pipeline_prequant_skipped = ())
+
+ assert spy.restored == []
+ assert len(spy.plans) == 2
+
+
+def test_a_declined_plan_never_seeds_at_the_load(fake_runtime, monkeypatch):
+ backend, spy = _load_backend(monkeypatch)
+ _load(
+ backend,
+ _pipeline_prequant_planned = PIPELINE_SEED_DECLINED,
+ _pipeline_prequant_skipped = (),
+ )
+
+ assert spy.seeds == []
+ assert spy.restored == []
+ assert spy.quantised == ["auto"]
+
+
+def test_a_direct_load_with_no_plan_phase_keeps_todays_behaviour(fake_runtime, monkeypatch):
+ """A direct call with no plan phase never seeds a decision the pull was not scoped on."""
+ backend, spy = _load_backend(monkeypatch)
+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())
+
+ assert spy.seeds == []
+ assert spy.quantised == ["auto"]
+
+
+def test_an_explicit_scheme_with_a_hosted_artifact_is_seeded_too(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ assert _settle(backend, transformer_quant = "fp8") == "fp8"
+
+
+def test_an_explicit_scheme_with_no_artifact_falls_to_the_in_memory_path(fake_runtime, monkeypatch):
+ """With nothing to seed, an explicit scheme quantises in place and still fails closed."""
+ backend, spy = _load_backend(monkeypatch)
+ status = _load(
+ backend,
+ transformer_quant = "fp8",
+ _pipeline_prequant_planned = None,
+ _pipeline_prequant_skipped = (),
+ )
+ assert spy.seeds == []
+ assert spy.quantised == ["fp8"]
+ assert status["transformer_quant"] == "fp8"
+
+ monkeypatch.setattr(dmod, "quantize_transformer", lambda *_a, **_k: None)
+ monkeypatch.setattr(dmod, "transformer_is_quantised", lambda _m: False)
+ with pytest.raises(RuntimeError, match = "transformer_quant='fp8'"):
+ _load(
+ backend,
+ transformer_quant = "fp8",
+ _pipeline_prequant_planned = None,
+ _pipeline_prequant_skipped = (),
+ )
+
+
+ "family, repo",
+ [
+ ("z-image", "unsloth/Z-Image-Turbo-FP8"),
+ ("flux.1-schnell", "unsloth/FLUX.1-schnell-FP8"),
+ ("qwen-image", "unsloth/Qwen-Image-FP8"),
+ ],
+)
+def test_the_rebuilt_fp8_artifacts_are_listed_for_both_schemes(family, repo):
+ """The hosted repos carry an fp8 and an int8 denoiser file, so an fp8 host seeds instead of quantising in memory."""
+ from core.inference.diffusion_families import detect_family, family_prequant_repo
+
+ fam = detect_family(family)
+ assert fam is not None
+ for scheme in ("fp8", "int8"):
+ assert family_prequant_repo(fam, scheme) == repo
diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py
index f1a9724adbd..efe468d1448 100644
--- a/studio/backend/tests/test_diffusion_routes.py
+++ b/studio/backend/tests/test_diffusion_routes.py
@@ -12,6 +12,23 @@ from __future__ import annotations
import types
+
+def _cuda_target():
+ """A compilable bf16 CUDA target, as ``_resolve_device_target`` returns on a real card.
+
+ The dtype is the real ``torch.bfloat16`` and ``supports_default_torch_compile`` is set, since the
+ precision preflight asks ``compile_eligible`` whether a quantised pipeline could compile at all:
+ a stub missing either reads as a card that cannot, and refuses loads a real one accepts.
+ """
+ import torch
+ return types.SimpleNamespace(
+ device = "cuda",
+ dtype = torch.bfloat16,
+ _cc = (10, 0),
+ supports_default_torch_compile = True,
+ )
+
+
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
@@ -1370,12 +1387,37 @@ def test_status_passes_through_resolved(client, monkeypatch):
# The cpu_offload value stays a real boolean (not coerced to a string).
assert body["resolved"]["cpu_offload"]["value"] is False
# A declined explicit precision keeps BOTH sides across the boundary: ask and outcome.
- assert body["resolved"]["transformer_quant"] == resolved["transformer_quant"]
+ assert body["resolved"]["transformer_quant"] == {
+ **resolved["transformer_quant"],
+ "artifact": None,
+ }
# Entries from an older backend (no requested/status) still parse, defaulted to "applied".
assert body["resolved"]["speed_mode"]["requested"] is None
assert body["resolved"]["speed_mode"]["status"] == "applied"
+def test_status_carries_the_prequant_artifact_through_the_route(client, monkeypatch):
+ # A seeded load records WHICH hosted file the precision came from; dropped at the boundary, the
+ # UI cannot tell a hosted checkpoint from a runtime quantise of the same scheme.
+ backend = diffusion_module.get_diffusion_backend()
+ resolved = {
+ "transformer_quant": {
+ "value": "fp8",
+ "source": "auto",
+ "reason": "seeded",
+ "artifact": "prequant:unsloth/Z-Image-Turbo-FP8/Z-Image-Turbo-FP8.pt",
+ }
+ }
+ monkeypatch.setattr(
+ backend, "status", lambda: {**_unloaded_status(), "loaded": True, "resolved": resolved}
+ )
+ body = client.get("/api/inference/images/status").json()
+ assert (
+ body["resolved"]["transformer_quant"]["artifact"]
+ == "prequant:unsloth/Z-Image-Turbo-FP8/Z-Image-Turbo-FP8.pt"
+ )
+
+
def test_status_resolved_defaults_to_null(client):
# A backend status without a `resolved` key leaves the additive field null (older backends and the unloaded state).
body = client.get("/api/inference/images/status").json()
@@ -1666,6 +1708,7 @@ def test_download_plan_forwards_the_load_time_controls(client, monkeypatch):
transformer_quant = "int8",
memory_mode = "low_vram",
cpu_offload = True,
+ transformer_quant_fast_accum = False,
loras = [{"id": "unsloth/some-lora", "weight": 0.8}],
)
@@ -1675,6 +1718,9 @@ def test_download_plan_forwards_the_load_time_controls(client, monkeypatch):
assert seen["transformer_quant"] == "int8"
assert seen["memory_mode"] == "low_vram"
assert seen["cpu_offload"] is True
+ # A forced accumulate the hosted checkpoint cannot bake declines the seed: dropped here, the plan
+ # stages an artifact the load refuses, then pulls the dense shards outside the staging.
+ assert seen["transformer_quant_fast_accum"] is False
assert len(seen["loras"] or []) == 1
@@ -2118,7 +2164,7 @@ def test_an_offloading_memory_request_refuses_an_explicit_precision(monkeypatch,
monkeypatch.setattr(
DiffusionBackend,
"_resolve_device_target",
- lambda self, fam: types.SimpleNamespace(device = "cuda", dtype = "bfloat16", _cc = (10, 0)),
+ lambda self, fam: _cuda_target(),
)
with pytest.raises(RuntimeError) as excinfo:
backend.assert_precision_available(
@@ -2176,7 +2222,7 @@ def test_an_offloading_memory_request_refuses_a_torchao_text_encoder(monkeypatch
monkeypatch.setattr(
DiffusionBackend,
"_resolve_device_target",
- lambda self, fam: types.SimpleNamespace(device = "cuda", dtype = "bfloat16", _cc = (10, 0)),
+ lambda self, fam: _cuda_target(),
)
# Support and the torchao install are not what is under test here.
monkeypatch.setattr(diffusion_module, "te_quant_supported", lambda target, m: True)
@@ -2202,7 +2248,7 @@ def test_layerwise_fp8_survives_an_offloading_memory_request(monkeypatch):
monkeypatch.setattr(
DiffusionBackend,
"_resolve_device_target",
- lambda self, fam: types.SimpleNamespace(device = "cuda", dtype = "bfloat16", _cc = (10, 0)),
+ lambda self, fam: _cuda_target(),
)
monkeypatch.setattr(diffusion_module, "te_quant_supported", lambda target, m: True)
monkeypatch.setattr(diffusion_module, "torchao_quantize_importable", lambda: True)
@@ -2226,7 +2272,7 @@ def test_a_broken_torchao_refuses_a_torchao_text_encoder_before_the_download(mon
monkeypatch.setattr(
DiffusionBackend,
"_resolve_device_target",
- lambda self, fam: types.SimpleNamespace(device = "cuda", dtype = "bfloat16", _cc = (10, 0)),
+ lambda self, fam: _cuda_target(),
)
monkeypatch.setattr(diffusion_module, "te_quant_supported", lambda target, m: True)
monkeypatch.setattr(diffusion_module, "torchao_quantize_importable", lambda: False)
@@ -2248,7 +2294,7 @@ def test_layerwise_fp8_does_not_need_torchao(monkeypatch):
monkeypatch.setattr(
DiffusionBackend,
"_resolve_device_target",
- lambda self, fam: types.SimpleNamespace(device = "cuda", dtype = "bfloat16", _cc = (10, 0)),
+ lambda self, fam: _cuda_target(),
)
monkeypatch.setattr(diffusion_module, "te_quant_supported", lambda target, m: True)
monkeypatch.setattr(diffusion_module, "torchao_quantize_importable", lambda: False)
@@ -2417,6 +2463,178 @@ def test_download_plan_still_refuses_a_bad_gpu_while_training_holds_the_cards(cl
assert "visible to this process" in resp.json()["detail"]
+def test_a_pipeline_pick_may_pin_a_precision(monkeypatch):
+ from core.inference.diffusion import DiffusionBackend
+
+ backend = DiffusionBackend.__new__(DiffusionBackend)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_resolve_device_target",
+ lambda self, fam: _cuda_target(),
+ )
+ monkeypatch.setattr(diffusion_module, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(diffusion_module, "select_transformer_quant_scheme", lambda *a, **k: "fp8")
+ backend.assert_precision_available(
+ types.SimpleNamespace(name = "z-image"),
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ )
+
+
+def test_a_single_file_pick_still_cannot_pin_a_precision(monkeypatch):
+ from core.inference.diffusion import DiffusionBackend
+
+ backend = DiffusionBackend.__new__(DiffusionBackend)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_resolve_device_target",
+ lambda self, fam: _cuda_target(),
+ )
+ monkeypatch.setattr(diffusion_module, "dense_transformer_supported", lambda target: True)
+ with pytest.raises(RuntimeError) as excinfo:
+ backend.assert_precision_available(
+ types.SimpleNamespace(name = "ltx-2.3"),
+ model_kind = "single_file",
+ transformer_quant = "fp8",
+ )
+ assert "transformer_quant='fp8' could not be used" in str(excinfo.value)
+ assert "single_file" in str(excinfo.value)
+ assert "GGUF and pipeline" in str(excinfo.value)
+
+
+def test_a_pipeline_pick_is_still_refused_on_a_device_that_cannot_quantise(monkeypatch):
+ from core.inference.diffusion import DiffusionBackend
+
+ backend = DiffusionBackend.__new__(DiffusionBackend)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_resolve_device_target",
+ lambda self, fam: _cuda_target(),
+ )
+ monkeypatch.setattr(diffusion_module, "dense_transformer_supported", lambda target: False)
+ with pytest.raises(RuntimeError) as excinfo:
+ backend.assert_precision_available(
+ types.SimpleNamespace(name = "z-image"),
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ )
+ assert "transformer_quant='fp8' could not be used" in str(excinfo.value)
+
+
+def test_a_unet_family_is_refused_before_the_eviction_and_the_download(monkeypatch):
+ """SDXL's family metadata already proves there is no transformer, so say so pre-eviction."""
+ from core.inference.diffusion import DiffusionBackend
+
+ backend = DiffusionBackend.__new__(DiffusionBackend)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_resolve_device_target",
+ lambda self, fam: _cuda_target(),
+ )
+ monkeypatch.setattr(diffusion_module, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(
+ diffusion_module,
+ "select_transformer_quant_scheme",
+ lambda *a, **k: pytest.fail("the UNet refusal must not need a scheme probe"),
+ )
+ with pytest.raises(RuntimeError) as excinfo:
+ backend.assert_precision_available(
+ types.SimpleNamespace(name = "sdxl", denoiser_attr = "unet"),
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ )
+ message = str(excinfo.value)
+ assert "transformer_quant='fp8' could not be used" in message
+ assert "unet" in message and "not a transformer" in message
+
+
+def test_a_transformer_family_still_reaches_the_scheme_check(monkeypatch):
+ """The new branch must not swallow the families that do have a transformer."""
+ from core.inference.diffusion import DiffusionBackend
+
+ backend = DiffusionBackend.__new__(DiffusionBackend)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_resolve_device_target",
+ lambda self, fam: _cuda_target(),
+ )
+ monkeypatch.setattr(diffusion_module, "dense_transformer_supported", lambda target: True)
+ seen: list = []
+ monkeypatch.setattr(
+ diffusion_module,
+ "select_transformer_quant_scheme",
+ lambda *a, **k: (seen.append(a) or "fp8"),
+ )
+ # denoiser_attr defaults to "transformer" on every DiT family.
+ backend.assert_precision_available(
+ types.SimpleNamespace(name = "z-image"),
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ )
+ assert seen
+
+
+def test_an_eager_pipeline_precision_is_refused_before_the_eviction(monkeypatch):
+ """Eager plus an explicit scheme is decidable from the request, so it must not cost a download."""
+ from core.inference.diffusion import DiffusionBackend
+
+ backend = DiffusionBackend.__new__(DiffusionBackend)
+ monkeypatch.setattr(
+ DiffusionBackend, "_resolve_device_target", lambda self, fam: _cuda_target()
+ )
+ monkeypatch.setattr(diffusion_module, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(
+ diffusion_module,
+ "select_transformer_quant_scheme",
+ lambda *a, **k: pytest.fail("the eager refusal must not need a scheme probe"),
+ )
+ with pytest.raises(RuntimeError) as excinfo:
+ backend.assert_precision_available(
+ types.SimpleNamespace(name = "z-image"),
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ speed_mode = "eager",
+ )
+ assert "eager" in str(excinfo.value)
+
+
+def test_a_gguf_pick_still_accepts_an_eager_precision(monkeypatch):
+ """The compile guard is the pipeline path's; GGUF substitutes weights and is left as on main."""
+ from core.inference.diffusion import DiffusionBackend
+
+ backend = DiffusionBackend.__new__(DiffusionBackend)
+ monkeypatch.setattr(
+ DiffusionBackend, "_resolve_device_target", lambda self, fam: _cuda_target()
+ )
+ monkeypatch.setattr(diffusion_module, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(diffusion_module, "select_transformer_quant_scheme", lambda *a, **k: "fp8")
+ backend.assert_precision_available(
+ types.SimpleNamespace(name = "z-image"),
+ model_kind = "gguf",
+ transformer_quant = "fp8",
+ speed_mode = "eager",
+ )
+
+
+def test_a_process_that_cannot_compile_refuses_a_pipeline_precision_up_front(monkeypatch):
+ """A Windows install without Triton is knowable here too, so it refuses before the download."""
+ from core.inference.diffusion import DiffusionBackend
+
+ backend = DiffusionBackend.__new__(DiffusionBackend)
+ monkeypatch.setattr(
+ DiffusionBackend, "_resolve_device_target", lambda self, fam: _cuda_target()
+ )
+ monkeypatch.setattr(diffusion_module, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(diffusion_module, "compile_eligible", lambda target, **kw: False)
+ with pytest.raises(RuntimeError) as excinfo:
+ backend.assert_precision_available(
+ types.SimpleNamespace(name = "z-image"),
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ )
+ assert "compile" in str(excinfo.value)
+
+
def test_the_plan_route_refuses_an_unrecognised_model_before_planning(client):
"""The route validates exactly as /images/load does, on purpose: an unloadable pick has to fail
HERE rather than after a multi-GB download. So an unrecognised repo never reaches the planner,
diff --git a/studio/backend/tests/test_diffusion_transformer_quant.py b/studio/backend/tests/test_diffusion_transformer_quant.py
index 8bcebbe38bb..0c9524a0814 100644
--- a/studio/backend/tests/test_diffusion_transformer_quant.py
+++ b/studio/backend/tests/test_diffusion_transformer_quant.py
@@ -22,6 +22,8 @@ from core.inference.diffusion_transformer_quant import (
TQ_INT8,
TQ_MXFP8,
TQ_NVFP4,
+ dense_quant_supported_kind,
+ dense_quant_unsupported_kind_reason,
dense_transformer_supported,
make_filter_fn,
normalize_transformer_quant,
@@ -90,25 +92,21 @@ def _allow(monkeypatch, allowed):
monkeypatch.setattr(tq, "_scheme_supported", lambda scheme, device, **kw: scheme in allowed)
-def test_auto_blackwell_prefers_fp8_then_falls_back(monkeypatch):
+def test_auto_blackwell_prefers_int8_then_walks_the_ladder(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
- # Even with every scheme available, auto picks fp8 on Blackwell: measured on a B200 it is faster AND more accurate than nvfp4 at DiT shapes.
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_FP8, TQ_INT8})
+ assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
+ _allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_FP8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_FP8
- # fp8 unavailable: auto skips nvfp4 even though the hardware runs it, because nvfp4 is an
- # explicit opt-in only (slower AND less accurate at DiT shapes), and lands on mxfp8.
- _allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_INT8})
+ # auto skips nvfp4 even though the hardware runs it: opt-in only.
+ _allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_MXFP8
- # Only mxfp8 + int8 left -> mxfp8 (still above int8).
- _allow(monkeypatch, {TQ_MXFP8, TQ_INT8})
- assert select_transformer_quant_scheme(_target(), "auto") == TQ_MXFP8
- # Only int8 usable -> int8.
- _allow(monkeypatch, {TQ_INT8})
- assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
+ _allow(monkeypatch, {TQ_NVFP4})
+ assert select_transformer_quant_scheme(_target(), "auto") is None
def test_auto_consumer_blackwell_prefers_int8(monkeypatch):
- # Consumer Blackwell (RTX 50xx): fp8 FP32-accumulate is throughput-halved while int8 is full-rate, so auto prefers int8.
+ # Consumer Blackwell (RTX 50xx): fp8 FP32 accumulate is halved, int8 is full-rate.
_stub_torch(monkeypatch, cc = (10, 0), device_name = "NVIDIA GeForce RTX 5090")
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_FP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
@@ -125,29 +123,29 @@ def test_auto_consumer_ada_prefers_int8(monkeypatch):
def test_auto_workstation_unknown_prefers_int8(monkeypatch):
- # An unknown / workstation name is treated as consumer (the safe default), so int8 first.
_stub_torch(monkeypatch, cc = (8, 9), device_name = "NVIDIA RTX A5000")
_allow(monkeypatch, {TQ_FP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
-def test_auto_professional_rtx_prefers_fp8(monkeypatch):
- # Professional parts (RTX PRO 6000 Blackwell, RTX 6000 Ada) count as datacenter elsewhere in the backend, so auto keeps fp8 first, matching llama_cpp.
+def test_auto_professional_rtx_prefers_int8(monkeypatch):
+ # Professional parts count as data-center for the accumulate gate, but the ladder order does not depend on it.
for device_name, cc in (
("NVIDIA RTX PRO 6000 Blackwell Server Edition", (10, 0)),
("NVIDIA RTX 6000 Ada Generation", (8, 9)),
):
_stub_torch(monkeypatch, cc = cc, device_name = device_name)
_allow(monkeypatch, {TQ_FP8, TQ_INT8})
- assert select_transformer_quant_scheme(_target(), "auto") == TQ_FP8
+ assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
-def test_auto_ada_hopper_prefers_fp8(monkeypatch):
- # Data-center Ada (L40S) / Hopper (H100) are not nerfed, so fp8 comes first.
+def test_auto_ada_hopper_prefers_int8_then_fp8(monkeypatch):
_stub_torch(monkeypatch, cc = (8, 9), device_name = "NVIDIA L40S")
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_FP8, TQ_INT8})
- assert select_transformer_quant_scheme(_target(), "auto") == TQ_FP8
+ assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
_stub_torch(monkeypatch, cc = (9, 0), device_name = "NVIDIA H100 80GB HBM3") # Hopper
+ assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
+ _allow(monkeypatch, {TQ_FP8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_FP8
@@ -1128,16 +1126,15 @@ def test_quantize_transformer_tolerates_failure(monkeypatch):
def test_family_deny_auto_skips_mx_and_nvfp4_for_qwen(monkeypatch):
- # B200 with every scheme available: mxfp8 and nvfp4 still damage the Qwen DiT, so auto skips
- # them. fp8 is no longer denied (activation_value_lb fixed the black frames), so auto now
- # takes fp8 first on a data-center part rather than falling all the way to int8.
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8, TQ_NVFP4, TQ_MXFP8, TQ_INT8})
- assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image") == TQ_FP8
- assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image-edit") == TQ_FP8
- # With fp8 unavailable the deny still bites: mxfp8 / nvfp4 are skipped and int8 is the pick.
- _allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image") == TQ_INT8
+ assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image-edit") == TQ_INT8
+ # fp8 is no longer denied: activation_value_lb fixed the black frames.
+ _allow(monkeypatch, {TQ_FP8, TQ_NVFP4, TQ_MXFP8})
+ assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image") == TQ_FP8
+ _allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8})
+ assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image") is None
def test_family_deny_refuses_explicit_mxfp8_and_nvfp4_for_qwen(monkeypatch):
@@ -1153,11 +1150,10 @@ def test_family_deny_refuses_explicit_mxfp8_and_nvfp4_for_qwen(monkeypatch):
def test_family_deny_no_family_keeps_ladder(monkeypatch):
- # Without a family (or an unknown one) the ladder is unchanged: fp8 first on B200.
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8, TQ_INT8})
- assert select_transformer_quant_scheme(_target(), "auto") == TQ_FP8
- assert select_transformer_quant_scheme(_target(), "auto", family = "sdxl") == TQ_FP8
+ assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
+ assert select_transformer_quant_scheme(_target(), "auto", family = "sdxl") == TQ_INT8
def test_quantize_transformer_threads_family(monkeypatch):
@@ -1414,9 +1410,8 @@ def test_auto_scheme_candidates_lists_the_whole_ladder_not_just_the_winner(monke
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8, TQ_MXFP8, TQ_INT8})
- assert auto_scheme_candidates(_target()) == (TQ_FP8, TQ_MXFP8, TQ_INT8)
- # The deny list still applies: qwen-image keeps mxfp8 out, so fp8 then int8.
- assert auto_scheme_candidates(_target(), "qwen-image") == (TQ_FP8, TQ_INT8)
+ assert auto_scheme_candidates(_target()) == (TQ_INT8, TQ_FP8, TQ_MXFP8)
+ assert auto_scheme_candidates(_target(), "qwen-image") == (TQ_INT8, TQ_FP8)
# Whatever the probe refuses is absent, so the list can never offer an unusable scheme.
_allow(monkeypatch, {TQ_INT8})
assert auto_scheme_candidates(_target(), "qwen-image") == (TQ_INT8,)
@@ -1598,3 +1593,313 @@ def test_real_torchao_configs_carry_set_inductor_config_false():
assert cfg.set_inductor_config is False, scheme
if ic is not None:
assert getattr(ic, "coordinate_descent_tuning", None) == before
+
+
+# Load-kind eligibility.
+
+
+def test_the_dense_quant_kinds_are_gguf_and_pipeline():
+ assert tq.DENSE_QUANT_KINDS == ("gguf", "pipeline")
+ assert dense_quant_supported_kind("gguf") is True
+ assert dense_quant_supported_kind("pipeline") is True
+ assert dense_quant_supported_kind("single_file") is False
+ assert dense_quant_supported_kind(" PIPELINE ") is True
+ assert dense_quant_supported_kind(None) is False
+ assert dense_quant_supported_kind("") is False
+
+
+def test_the_unsupported_kind_reason_names_the_kind_and_the_two_that_work():
+ reason = dense_quant_unsupported_kind_reason("single_file")
+ assert "single_file" in reason
+ assert "GGUF and pipeline" in reason
+ assert "the precision its checkpoint carries" in reason
+
+
+# Built-pipeline eligibility.
+
+
+class _Denoiser:
+ """A module as the gate reads one: parameters that report a dtype."""
+
+ def __init__(
+ self,
+ dtype = "torch.bfloat16",
+ **attrs,
+ ) -> None:
+ self._params = [types.SimpleNamespace(dtype = dtype)]
+ for key, value in attrs.items():
+ setattr(self, key, value)
+
+ def parameters(self, recurse = True):
+ return iter(self._params)
+
+
+def test_a_dense_bf16_pipeline_is_the_one_shape_that_passes():
+ pipe = types.SimpleNamespace(transformer = _Denoiser())
+ assert tq.dense_quant_blocker(pipe) is None
+ assert [attr for attr, _m in tq.denoiser_modules(pipe)] == ["transformer"]
+
+
+def test_a_unet_pipeline_is_blocked_by_having_no_transformer():
+ blocker = tq.dense_quant_blocker(types.SimpleNamespace(unet = _Denoiser()))
+ assert blocker is not None and "UNet" in blocker
+
+
+ "dtype", ["torch.uint8", "torch.float8_e4m3fn", "torch.float16", "torch.int8"]
+)
+def test_a_pre_quantised_pipeline_is_blocked_by_its_parameter_dtypes(dtype):
+ blocker = tq.dense_quant_blocker(types.SimpleNamespace(transformer = _Denoiser(dtype)))
+ assert blocker is not None and dtype.split(".")[-1] in blocker
+
+
+ ("attrs", "expected"),
+ [
+ ({"_unsloth_runtime_quant": "int8"}, "already quantised"),
+ ({"is_loaded_in_4bit": True}, "4-bit"),
+ ({"is_loaded_in_8bit": True}, "8-bit"),
+ ({"config": types.SimpleNamespace(quantization_config = object())}, "quantization_config"),
+ ],
+)
+def test_a_declared_quantisation_blocks_it_too(attrs, expected):
+ blocker = tq.dense_quant_blocker(types.SimpleNamespace(transformer = _Denoiser(**attrs)))
+ assert blocker is not None and expected in blocker
+
+
+def test_the_second_denoiser_is_enumerated_and_judged():
+ pipe = types.SimpleNamespace(
+ transformer = _Denoiser(), unconditional_transformer = _Denoiser("torch.float8_e4m3fn")
+ )
+ assert [attr for attr, _m in tq.denoiser_modules(pipe)] == [
+ "transformer",
+ "unconditional_transformer",
+ ]
+ blocker = tq.dense_quant_blocker(pipe)
+ assert blocker is not None and "unconditional_transformer" in blocker
+
+
+def test_the_denoiser_view_presents_an_arbitrary_attribute_as_the_transformer():
+ second = _Denoiser()
+ pipe = types.SimpleNamespace(transformer = _Denoiser(), unconditional_transformer = second, vae = "v")
+ view = tq.DenoiserView(pipe, "unconditional_transformer")
+ assert view.transformer is second
+ assert view.vae == "v" # everything else reads through
+
+
+def test_a_pipeline_that_cannot_be_walked_is_not_called_quantised():
+ class _Unwalkable:
+ def parameters(self, recurse = True):
+ raise RuntimeError("no")
+
+ assert tq.dense_quant_blocker(types.SimpleNamespace(transformer = _Unwalkable())) is None
+
+
+def test_a_dequantised_source_blocks_the_quant_even_though_its_tensors_are_bf16():
+ widened = _Denoiser() # bf16 tensors, exactly as the loader leaves them
+ assert tq.dense_quant_blocker(types.SimpleNamespace(transformer = widened)) is None
+ tq.mark_source_precision(widened, "fp8")
+ blocker = tq.dense_quant_blocker(types.SimpleNamespace(transformer = widened))
+ assert blocker is not None
+ assert "fp8" in blocker and "widened to bf16" in blocker
+
+
+def test_the_source_marker_is_best_effort_and_returns_the_module():
+ module = _Denoiser()
+ assert tq.mark_source_precision(module, "fp8") is module
+
+ class _Frozen:
+ __slots__ = ()
+
+ frozen = _Frozen()
+ assert tq.mark_source_precision(frozen, "fp8") is frozen
+ assert getattr(frozen, tq.SOURCE_PRECISION_ATTR, None) is None
+
+
+def test_the_ideogram_fp8_loader_stamps_what_it_widened():
+ import pathlib
+
+ import core.inference.diffusion_ideogram4 as ideo
+
+ source = pathlib.Path(ideo.__file__).read_text(encoding = "utf-8")
+ assert 'mark_source_precision(model, "fp8")' in source
+
+
+# Advertised host capability.
+
+
+def _capable_host(
+ monkeypatch,
+ *,
+ torchao_reason = None,
+ cap = (8, 9),
+):
+ """A CUDA host past the arch floor, with torchao, the probe cache and the compiler pinned.
+
+ ``compile_eligible`` is stubbed true because these targets are namespaces without a card's real
+ ``supports_default_torch_compile`` and dtype; the test that cares about it overrides this."""
+ from core.inference import diffusion_speed
+
+ monkeypatch.setattr(diffusion_speed, "compile_eligible", lambda target, **kw: True)
+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(tq, "_capability", lambda: cap)
+ monkeypatch.setattr(tq, "_TORCHAO_UNAVAILABLE", (torchao_reason,))
+ monkeypatch.setattr(tq, "_smoke_cache_device_key", lambda device: "cuda:0")
+ monkeypatch.setattr(tq, "_SMOKE_CACHE", {})
+
+
+def test_a_capable_host_advertises_dense_quant(monkeypatch):
+ _capable_host(monkeypatch)
+ assert tq.dense_quant_host_capable(_target()) is True
+
+
+def test_a_host_whose_torchao_cannot_import_advertises_nothing(monkeypatch):
+ """An unimportable torchao makes every scheme decline, so the capability must be false.
+
+ `dense_transformer_supported` only catches the Windows-ROCm stub and `_capability` reads the
+ card, so without this the picker calls rows fast while every load falls back to bf16.
+ """
+ _capable_host(monkeypatch, torchao_reason = "ImportError: cannot import name 'ScalingType'")
+ assert tq.dense_quant_host_capable(_target()) is False
+
+
+def test_a_host_below_the_arch_floor_advertises_nothing(monkeypatch):
+ _capable_host(monkeypatch, cap = (7, 0))
+ assert tq.dense_quant_host_capable(_target()) is False
+
+
+def test_an_unsupported_device_advertises_nothing(monkeypatch):
+ _capable_host(monkeypatch)
+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)
+ assert tq.dense_quant_host_capable(_target(device = "cpu")) is False
+
+
+def test_a_probed_failure_withdraws_the_capability(monkeypatch):
+ """The arch floor is necessary, not sufficient: a probed failure must not stay advertised."""
+ _capable_host(monkeypatch)
+ assert tq.dense_quant_host_capable(_target()) is True
+ # The load path has since proved this tier's schemes do not run on this card.
+ tq._SMOKE_CACHE.update({(TQ_FP8, "cuda:0"): False, (TQ_INT8, "cuda:0"): False})
+ assert tq.dense_quant_host_capable(_target()) is False
+ # One survivor is enough for auto to have something to pick.
+ tq._SMOKE_CACHE[(TQ_INT8, "cuda:0")] = True
+ assert tq.dense_quant_host_capable(_target()) is True
+
+
+def test_an_explicit_only_scheme_does_not_make_auto_capable(monkeypatch):
+ """`auto` cannot pick nvfp4, so a host that runs only nvfp4 has nothing automatic to offer."""
+ _capable_host(monkeypatch, cap = (10, 0))
+ tq._SMOKE_CACHE.update(
+ {
+ (TQ_FP8, "cuda:0"): False,
+ (TQ_MXFP8, "cuda:0"): False,
+ (TQ_INT8, "cuda:0"): False,
+ (TQ_NVFP4, "cuda:0"): True,
+ }
+ )
+ assert tq.dense_quant_host_capable(_target()) is False
+ assert not any(TQ_NVFP4 in schemes for _floor, schemes in tq._AUTO_LADDER)
+
+
+def test_the_capability_never_probes(monkeypatch):
+ """A polled status route must not buy the CUDA context the in-process probe leaks."""
+ _capable_host(monkeypatch)
+ monkeypatch.setattr(
+ tq, "_smoke_probe", lambda *a, **k: pytest.fail("the status path probed in-process")
+ )
+ monkeypatch.setattr(
+ tq, "_child_probe_table", lambda device: pytest.fail("the status path spawned a child")
+ )
+ assert tq.dense_quant_host_capable(_target()) is True
+
+
+# Source precision recovered from the shard header.
+
+
+def _write_shard(tmp_path, attr, dtype):
+ import torch
+ from safetensors.torch import save_file
+
+ sub = tmp_path / attr
+ sub.mkdir(parents = True, exist_ok = True)
+ save_file(
+ {"proj.weight": torch.zeros(4, 4, dtype = dtype)},
+ str(sub / "diffusion_pytorch_model.safetensors"),
+ )
+
+
+def test_a_raw_fp8_checkpoint_is_recognised_from_its_header(tmp_path):
+ """from_pretrained widens fp8 to bf16, so the header is the only surviving evidence."""
+ import torch
+
+ _write_shard(tmp_path, "transformer", torch.float8_e4m3fn)
+ assert tq.stored_denoiser_precision(str(tmp_path)) == "fp8"
+
+
+def test_a_bf16_checkpoint_reports_nothing(tmp_path):
+ import torch
+ _write_shard(tmp_path, "transformer", torch.bfloat16)
+ assert tq.stored_denoiser_precision(str(tmp_path)) is None
+
+
+def test_the_second_denoiser_is_read_too(tmp_path):
+ import torch
+
+ _write_shard(tmp_path, "transformer", torch.bfloat16)
+ _write_shard(tmp_path, "unconditional_transformer", torch.float8_e5m2)
+ assert tq.stored_denoiser_precision(str(tmp_path)) == "fp8"
+
+
+def test_an_int8_checkpoint_is_recognised(tmp_path):
+ import torch
+ _write_shard(tmp_path, "transformer", torch.int8)
+ assert tq.stored_denoiser_precision(str(tmp_path)) == "int8"
+
+
[email protected]("local_dir", [None, "", "/nonexistent/path"])
+def test_no_snapshot_is_no_verdict(local_dir):
+ """Without a local snapshot this must not guess, and must never reach the network."""
+ assert tq.stored_denoiser_precision(local_dir) is None
+
+
+def test_a_recovered_source_precision_blocks_the_quant(tmp_path):
+ """The recovered marker feeds the same refusal the Ideogram loader's marker does."""
+ import torch
+
+ _write_shard(tmp_path, "transformer", torch.float8_e4m3fn)
+ widened = _Denoiser() # bf16 tensors, exactly as the loader leaves them
+ pipe = types.SimpleNamespace(transformer = widened)
+ assert tq.dense_quant_blocker(pipe) is None
+ tq.mark_source_precision(widened, tq.stored_denoiser_precision(str(tmp_path)))
+ blocker = tq.dense_quant_blocker(pipe)
+ assert blocker is not None and "fp8" in blocker and "widened to bf16" in blocker
+
+
+def test_a_host_that_cannot_compile_advertises_nothing(monkeypatch):
+ """The loader keeps a pipeline dense when nothing can compile it, so there is no path to sell.
+ Reachable on a Windows CUDA install with no Triton wheel, and under TORCHDYNAMO_DISABLE."""
+ from core.inference import diffusion_speed
+
+ _capable_host(monkeypatch)
+ monkeypatch.setattr(diffusion_speed, "compile_eligible", lambda target, **kw: False)
+ assert tq.dense_quant_host_capable(_target()) is False
+ monkeypatch.setattr(diffusion_speed, "compile_eligible", lambda target, **kw: True)
+ assert tq.dense_quant_host_capable(_target()) is True
+
+
+def test_a_later_shard_still_reveals_a_narrow_source(tmp_path):
+ """Narrow checkpoints keep norms and embeddings wide, so shard one can be entirely bf16."""
+ import torch
+ from safetensors.torch import save_file
+
+ sub = tmp_path / "transformer"
+ sub.mkdir(parents = True)
+ save_file(
+ {"norm.weight": torch.zeros(4, dtype = torch.bfloat16)},
+ str(sub / "diffusion_pytorch_model-00001-of-00002.safetensors"),
+ )
+ save_file(
+ {"proj.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fn)},
+ str(sub / "diffusion_pytorch_model-00002-of-00002.safetensors"),
+ )
+ assert tq.stored_denoiser_precision(str(tmp_path)) == "fp8"
diff --git a/studio/backend/tests/test_video_routes.py b/studio/backend/tests/test_video_routes.py
index 35967bcdaea..556cb4c0f27 100644
--- a/studio/backend/tests/test_video_routes.py
+++ b/studio/backend/tests/test_video_routes.py
@@ -842,8 +842,15 @@ def test_status_passthrough(client, monkeypatch):
)
body = client.get("/api/inference/video/status").json()
assert body["loaded"] is True and body["family"] == "ltx-2"
- assert body["resolved"]["transformer_quant"] == resolved["transformer_quant"]
- assert body["resolved"]["text_encoder_quant"] == resolved["text_encoder_quant"]
+ # ``artifact`` is additive on the response model: a record naming no hosted checkpoint is null.
+ assert body["resolved"]["transformer_quant"] == {
+ **resolved["transformer_quant"],
+ "artifact": None,
+ }
+ assert body["resolved"]["text_encoder_quant"] == {
+ **resolved["text_encoder_quant"],
+ "artifact": None,
+ }
# Entries from an older backend (no requested/status) still parse, defaulted to "applied".
assert body["resolved"]["speed_mode"]["requested"] is None
assert body["resolved"]["speed_mode"]["status"] == "applied"
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts
index f40d3917b0f..f1da1eefdd8 100644
--- a/studio/frontend/src/features/images/api.ts
+++ b/studio/frontend/src/features/images/api.ts
@@ -16,6 +16,8 @@ export interface DiffusionResolvedControl {
// "applied" (honored, or nothing was asked) | "fell_back" | "unsupported". Absent on older backends.
status?: "applied" | "fell_back" | "unsupported";
reason: string;
+ // "prequant:<repo>/<file>" when a hosted checkpoint was seeded; absent on a runtime quantise.
+ artifact?: string | null;
}
export interface DiffusionStatus {
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index a45fbbc57c4..534b30e3c1e 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -59,13 +59,17 @@ import { useScrollFades } from "@/hooks/use-scroll-fades";
import { ModelSelector } from "@/features/model-picker/components/model-selector";
import { IMAGE_GEN_TASKS } from "@/features/model-picker/components/model-selector/pickers";
import { PillTabs } from "@/features/model-picker/components/model-selector/pill-tabs";
-import type { HostClass } from "@/features/model-picker/components/model-selector/host-artifact-policy";
+import {
+ type HostClass,
+ hostOffersDensePrecision,
+} from "@/features/model-picker/components/model-selector/host-artifact-policy";
import {
IMAGE_CATALOG,
catalogToModelOptions,
+ curatedArtifactTakesDenseQuant,
loadSpecFor,
} from "@/features/model-picker/components/model-selector/model-catalog";
-import { useHostClass } from "@/hooks/use-host-class";
+import { useDenseQuantSchemes, useHostClass } from "@/hooks/use-host-class";
import type {
ModelOption,
ModelSelectorChangeMeta,
@@ -121,6 +125,7 @@ import {
denseTransformerBuildLabel,
isNativeEngineStatus,
formatResolvedValue,
+ isDenseQuantKind,
isPrecisionRefusal,
memoryRecipeValue,
resolvedBadge,
@@ -176,11 +181,25 @@ import {
type TrainFamilyOption,
} from "./train/train-base-selector";
+/** Whether this pick may receive a transformer precision request. Unknown repos defer to the backend. */
+function sendsTransformerQuant(kind: string | null | undefined, repoId: string): boolean {
+ return (
+ isDenseQuantKind(kind) &&
+ curatedArtifactTakesDenseQuant(repoId, IMAGE_CATALOG) !== false
+ );
+}
+
// Curated models come from the shared catalog, one group per model with its artifacts as data and
// the load kind per artifact from loadSpecFor. Built per render, since a host that can only run
// the native engine is not offered pipeline rows.
-function useImageModels(host: HostClass): ModelOption[] {
- return useMemo(() => catalogToModelOptions(IMAGE_CATALOG, host), [host]);
+function useImageModels(
+ host: HostClass,
+ denseQuantSchemes: readonly string[],
+): ModelOption[] {
+ return useMemo(
+ () => catalogToModelOptions(IMAGE_CATALOG, host, denseQuantSchemes),
+ [host, denseQuantSchemes],
+ );
}
// Workflow tabs. `requires` is the backend workflow id (status.workflows) the model must
@@ -1168,7 +1187,8 @@ export function ImagesPage({
const pendingRecalledGeneration = useRef<{ model: RememberedImageModel; load: number; workflow: WorkflowId } | null>(null);
const { isMobile, pinned } = useSidebar();
const hostClass = useHostClass();
- const imageModels = useImageModels(hostClass);
+ const denseQuantSchemes = useDenseQuantSchemes();
+ const imageModels = useImageModels(hostClass, denseQuantSchemes);
const [quant, setQuant] = useState<string | null>(galleryCache.quant);
const [prompt, setPrompt] = useState(
"Cinematic wide shot of a whimsical Alice in Wonderland tea party in an overgrown Victorian garden. Exactly three figures at a long white lace-draped table: a tall eccentric gentleman in an oversized emerald velvet top hat pouring tea from a silver pot mid-motion; a young woman in a pale blue Victorian dress seated left, holding a porcelain teacup with both hands, looking up and laughing; an older woman in deep burgundy seated right in profile, reaching for a tiered cake stand. Detailed embroidered fabrics, realistic skin texture, natural expressions. The table holds mismatched porcelain, antique silverware, towering pastel cakes, and wildflowers. Giant red-capped mushrooms rise behind the table, with ancient trees overhead and golden sunlight streaming through leaves. Shot on 85mm, f/2.8, focus on the gentleman, soft background falloff. Photorealistic, saturated storybook color, warm amber and deep green palette.",
@@ -2479,10 +2499,10 @@ export function ImagesPage({
hf_token: hfApiToken(getHfToken()),
cpu_offload: advanced.cpu_offload,
speed_mode: advanced.speed_mode,
- // GGUF picks only: the dense fast path replaces a GGUF transformer, and every other kind runs
- // its checkpoint's own precision. The control is hidden there but the state persists across
- // picks, so a stale scheme would reach a load that can only decline it.
- transformer_quant: opts.kind === "gguf" ? advanced.transformer_quant : undefined,
+ // Do not carry a saved precision into a known incompatible artifact.
+ transformer_quant: sendsTransformerQuant(opts.kind, repoId)
+ ? advanced.transformer_quant
+ : undefined,
text_encoder_quant: advanced.text_encoder_quant,
attention_backend: advanced.attention_backend,
memory_mode: advanced.memory_mode,
@@ -2682,8 +2702,10 @@ export function ImagesPage({
hf_token: hfApiToken(getHfToken()),
cpu_offload: advanced.cpu_offload,
speed_mode: advanced.speed_mode,
- // Non-GGUF loads ignore this control; the plan must describe the same request as handleLoad.
- transformer_quant: opts.kind === "gguf" ? advanced.transformer_quant : undefined,
+ // Keep the planned precision identical to the load request.
+ transformer_quant: sendsTransformerQuant(opts.kind, repoId)
+ ? advanced.transformer_quant
+ : undefined,
text_encoder_quant: advanced.text_encoder_quant,
memory_mode: advanced.memory_mode,
// The backend prefetch decision reads the adapter selection too: a baked LoRA always runs the
@@ -3714,25 +3736,31 @@ export function ImagesPage({
["max", "Max"],
]}
/>
- {/* The dense transformer_quant fast path engages only on the GGUF kind, so gate the control to
- GGUF (or nothing loaded) and otherwise say why it is unavailable. Native sd.cpp reports
- model_kind "gguf" as well, to be recallable by exact checkpoint, but runs no torchao path:
- gate it out by ENGINE or it offers FP8/INT8/NVFP4 with no badge and snaps back on load. */}
+ {/* Use the same precision eligibility rule as the load request. Native sd.cpp reports
+ model_kind "gguf" as well, to be recallable by exact checkpoint, but runs no torchao
+ path: gate it out by ENGINE or it offers FP8/INT8/NVFP4 with no badge and snaps back. */}
{!status?.loaded
- || (status.model_kind === "gguf" && !isNativeEngineStatus(status)) ? (
+ || (sendsTransformerQuant(status.model_kind, status.repo_id ?? "")
+ && !isNativeEngineStatus(status)) ? (
<AdvancedSelect
label="Precision"
- hint="How the model computes. Auto picks the fastest precision the hardware supports (at least INT8 on a capable GPU; FP8 on data-center cards) by loading the FULL base model and quantising its transformer onto low-precision tensor cores, and falls back to running the GGUF as-is when the device, VRAM or disk can't take it. Off always runs the GGUF as-is."
+ hint="How the model computes. Auto picks the fastest precision the hardware supports (INT8 on every capable GPU, then FP8 where the card has it) and quantises the transformer onto low-precision tensor cores. A GGUF pick reaches it by loading the FULL base model instead of the GGUF, and falls back to the GGUF as-is when the device, VRAM or disk can't take it; an official pipeline is already dense and is quantised in place, falling back to plain BF16. Off runs the checkpoint as-is."
badge={<ResolvedBadge status={status} controlKey="transformer_quant" />}
value={transformerQuant}
onValueChange={(v) => setTransformerQuant(v as typeof transformerQuant)}
options={[
["auto", "Auto (fastest for GPU)"],
- ["none", "Off (run the GGUF)"],
- ["fp8", "FP8"],
- ["int8", "INT8"],
- ["nvfp4", "NVFP4 (Blackwell)"],
- ["mxfp8", "MXFP8 (Blackwell)"],
+ ["none", "Off (run the checkpoint as-is)"],
+ // The explicit low-precision schemes need the dense tensor-core path, which a Mac or
+ // CPU-only host cannot run, so the picker does not list what the loader would refuse.
+ ...(hostOffersDensePrecision(hostClass)
+ ? ([
+ ["fp8", "FP8"],
+ ["int8", "INT8"],
+ ["nvfp4", "NVFP4 (Blackwell)"],
+ ["mxfp8", "MXFP8 (Blackwell)"],
+ ] as [string, string][])
+ : []),
]}
/>
) : (
@@ -3740,7 +3768,9 @@ export function ImagesPage({
<span className="flex items-center gap-1 text-xs font-medium text-muted-foreground">
Precision
</span>
- <span className="text-xs text-muted-foreground/60">GGUF models only</span>
+ <span className="text-xs text-muted-foreground/60">
+ Runs this checkpoint's own precision
+ </span>
</div>
)}
<AdvancedSelect
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/host-artifact-policy.ts b/studio/frontend/src/features/model-picker/components/model-selector/host-artifact-policy.ts
index 065964eba0b..612b4461f18 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/host-artifact-policy.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/host-artifact-policy.ts
@@ -1,12 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-/** What the host can actually run, as opposed to what fits on it. Capability, not size:
- * `curatedArtifactFitsDevice` already answers "is there room", and the two must stay apart. A
- * 128 GB Mac has room for the H3 pipeline and still cannot load it. `unknown` is not a
- * formality: the GPU hook starts at `available: false, budgetKnown: false`, so a boolean would
- * read every first render as CPU-only and blink the non-GGUF rows out on a real GPU host. */
-export type HostClass = "unknown" | "gguf-only" | "accelerated";
+import { normalizeDenseQuantSchemes } from "../../../../lib/dense-quant-schemes.ts";
+
+/** Media-picker runtime capability. The backend reports dense quant, since its name alone cannot
+ * distinguish unsupported accelerators. */
+export type HostClass = "unknown" | "gguf-only" | "accelerated" | "dense-quant";
/** Backends whose diffusion pipelines the media pages can place. */
const ACCELERATED_BACKENDS = new Set(["cuda", "rocm", "xpu"]);
@@ -18,10 +17,13 @@ export function classifyHost({
deviceType,
deviceBackend,
budgetKnown,
+ denseQuantSupported,
}: {
deviceType?: string | null;
deviceBackend?: string | null;
budgetKnown?: boolean;
+ /** Backend-reported dense quant capability. */
+ denseQuantSupported?: boolean;
}): HostClass {
// Mac outranks the backend string. Apple GPUs report as available and Unsloth may name the
// backend mlx or cpu, but no Mac can place a Modular Diffusers workflow: it needs
@@ -30,11 +32,23 @@ export function classifyHost({
const backend = (deviceBackend ?? "").trim().toLowerCase();
if (!(backend && budgetKnown)) return "unknown";
if (GGUF_ONLY_BACKENDS.has(backend)) return "gguf-only";
- if (ACCELERATED_BACKENDS.has(backend)) return "accelerated";
+ if (ACCELERATED_BACKENDS.has(backend)) {
+ return denseQuantSupported ? "dense-quant" : "accelerated";
+ }
// An unrecognised backend is a new accelerator, not a CPU. Show what we show today.
return "unknown";
}
+/** Whether the host can place a diffusion pipeline. */
+export function hostIsAccelerated(host: HostClass): boolean {
+ return host === "accelerated" || host === "dense-quant";
+}
+
+/** Whether the dense torchao transformer quant can engage on this host. */
+export function hostRunsDenseQuant(host: HostClass): boolean {
+ return host === "dense-quant";
+}
+
/** The H3 group, whose two rows differ by roughly 10x in throughput. */
const H3_PIPELINE_ID = "minimaxai/minimax-h3";
@@ -57,13 +71,46 @@ export function curatedArtifactIsOfferable(
const H3_GGUF_ID = "unsloth/minimax-h3-gguf";
-/** The speed qualifier for a curated row, or null when it earns none. Deliberately keyed on the
- * two H3 ids rather than on `format !== "gguf"`: most non-GGUF rows are plain bf16 or
- * bnb-4bit, and only H3 has the measured gap this wording claims. */
-export function h3PerfSuffix(repoId: string, host: HostClass): string | null {
- if (host !== "accelerated") return null;
+/** The speed qualifier for a dense-quant row, naming the precision that will run. An empty list (an
+ * older backend) falls back to a bare "Fast". Only the first, best entry is read. */
+/** Dense 8-bit schemes that share the one user-facing label; see ``densePerfSuffix``. */
+const DENSE_EIGHT_BIT = new Set(["int8", "fp8", "mxfp8"]);
+
+export function densePerfSuffix(
+ denseQuantSchemes?: readonly string[] | null,
+): string {
+ const scheme = normalizeDenseQuantSchemes(denseQuantSchemes)[0];
+ if (!scheme) return "Fast";
+ // The row names the fast TIER, not the scheme the load settles on: int8 and mxfp8 render as FP8,
+ // the name users know for the 8-bit fast path, so reordering the ladder does not rename the model.
+ // The loaded-models card's resolved record still names the real scheme. A 4-bit scheme names
+ // itself, being a different quality tier.
+ return DENSE_EIGHT_BIT.has(scheme) ? "Fast FP8" : `Fast ${scheme.toUpperCase()}`;
+}
+
+/** Speed qualifier for a GGUF diffusion row. A GGUF runs the native engine, which has no
+ * low-precision tensor-core path and no compiled dense transformer, so it is the slow row wherever
+ * a dense row sits beside it. Null off an accelerator, where it is the only row that runs. */
+export function ggufPerfSuffix(host: HostClass): string | null {
+ return hostIsAccelerated(host) ? "Slow" : null;
+}
+
+/** Whether the Precision control should offer the dense low-precision schemes. A Mac or CPU-only host
+ * is refused them at load. An "unknown" host keeps the full list, since an older backend reporting
+ * no capability must not lose controls it can honour. */
+export function hostOffersDensePrecision(host: HostClass): boolean {
+ return host !== "gguf-only";
+}
+
+/** Speed qualifier for H3 artifacts, naming its precision from the same host scheme list. */
+export function h3PerfSuffix(
+ repoId: string,
+ host: HostClass,
+ denseQuantSchemes?: readonly string[] | null,
+): string | null {
+ if (!hostIsAccelerated(host)) return null;
const id = repoId.trim().toLowerCase();
- if (id === H3_PIPELINE_ID) return "Fast FP8";
+ if (id === H3_PIPELINE_ID) return densePerfSuffix(denseQuantSchemes);
if (id === H3_GGUF_ID) return "Slow";
return null;
}
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts
index ab32248175a..0ab648b0ada 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts
@@ -1,16 +1,17 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-// Assertions over the model catalog: keys, aliases, integrity, quant ladders, search. `--network` adds an OPT-IN
-// Hub reachability/gated-flag pass, kept out of `npm run catalog:check` so a Hub hiccup cannot fail an unrelated
-// PR. Run it on a schedule (.github/workflows/model-catalog-network-check.yml) or by hand with
-// `npm run catalog:check:network`.
+// Assertions over the model catalog: keys, aliases, integrity, quant ladders, search.
+// `--network` adds an OPT-IN Hub reachability/gated-flag pass, kept out of `npm run catalog:check`
+// so a Hub hiccup cannot fail an unrelated PR. Run it on a schedule
+// (.github/workflows/model-catalog-network-check.yml) or by hand with `npm run catalog:check:network`.
import assert from "node:assert/strict";
import type { CatalogGroup, ModelArtifact } from "./model-catalog.ts";
import {
AUDIO_CATALOG,
+ artifactForRepoId,
IMAGE_CATALOG,
VIDEO_CATALOG,
canonicalKeyFor,
@@ -20,6 +21,7 @@ import {
classifyMediaGgufFit,
curatedArtifactFitsDevice,
curatedDisplayNameFor,
+ curatedArtifactTakesDenseQuant,
curatedRowLabelFor,
ggufFitRuns,
groupForRepoId,
@@ -257,7 +259,7 @@ assert.deepEqual(curatedRowLabelFor("unsloth/Z-Image-Turbo-GGUF", IMAGE_CATALOG)
// On a host that can place a diffusion pipeline the two H3 rows say which is which: the
// gap is roughly 10x and the names alone gave nothing to choose on.
assert.deepEqual(curatedRowLabelFor("MiniMaxAI/MiniMax-H3", VIDEO_CATALOG, "accelerated"), {
- name: "MiniMax H3 (Fast FP8)",
+ name: "MiniMax H3 (Fast)",
tags: ["BF16"],
});
assert.deepEqual(
@@ -272,7 +274,7 @@ assert.deepEqual(
// The trigger and the row must agree, or the model renames itself as the popover opens.
assert.equal(
curatedDisplayNameFor("MiniMaxAI/MiniMax-H3", VIDEO_CATALOG, "accelerated"),
- "MiniMax H3 (Fast FP8)",
+ "MiniMax H3 (Fast)",
);
assert.equal(
curatedDisplayNameFor("unsloth/MiniMax-H3-GGUF", VIDEO_CATALOG, "accelerated"),
@@ -283,6 +285,162 @@ assert.deepEqual(
curatedRowLabelFor("Lightricks/LTX-2", VIDEO_CATALOG, "accelerated"),
curatedRowLabelFor("Lightricks/LTX-2", VIDEO_CATALOG),
);
+// H3's measured qualifier applies to every accelerated host.
+assert.deepEqual(
+ curatedRowLabelFor("MiniMaxAI/MiniMax-H3", VIDEO_CATALOG, "dense-quant"),
+ curatedRowLabelFor("MiniMaxAI/MiniMax-H3", VIDEO_CATALOG, "accelerated"),
+);
+
+// A dense-quant row earns the qualifier; its chip stays the stored precision.
+assert.deepEqual(curatedRowLabelFor("Tongyi-MAI/Z-Image-Turbo", IMAGE_CATALOG, "dense-quant"), {
+ name: "Z-Image-Turbo (Fast)",
+ tags: ["BF16"],
+});
+assert.equal(
+ curatedDisplayNameFor("Tongyi-MAI/Z-Image-Turbo", IMAGE_CATALOG, "dense-quant"),
+ "Z-Image-Turbo (Fast)",
+);
+// Pre-quantised and GGUF siblings keep their artifact labels.
+assert.deepEqual(
+ curatedRowLabelFor("unsloth/Z-Image-Turbo-unsloth-bnb-4bit", IMAGE_CATALOG, "dense-quant"),
+ { name: "Z-Image-Turbo", tags: ["bnb-4bit"] },
+);
+// Every diffusion GGUF is the slow row on a host that can also place the dense pipeline, not just
+// H3's: the native engine has no low-precision tensor-core path.
+assert.deepEqual(curatedRowLabelFor("unsloth/Z-Image-Turbo-GGUF", IMAGE_CATALOG, "dense-quant"), {
+ name: "Z-Image-Turbo-GGUF (Slow)",
+ tags: [],
+});
+// Hosts without dense quant read exactly the same, minus the qualifier.
+for (const host of ["accelerated", "gguf-only", "unknown"] as const) {
+ assert.deepEqual(
+ curatedRowLabelFor("Tongyi-MAI/Z-Image-Turbo", IMAGE_CATALOG, host),
+ { name: "Z-Image-Turbo", tags: ["BF16"] },
+ host,
+ );
+}
+// SDXL uses a UNet and never receives the transformer-quant label.
+for (const id of ["stabilityai/sdxl-turbo", "stabilityai/stable-diffusion-xl-base-1.0"]) {
+ const row = curatedRowLabelFor(id, IMAGE_CATALOG, "dense-quant");
+ assert.ok(row && !row.name.includes("(Fast)"), `${id} reads "${row?.name}"`);
+}
+
+// Only eligible bf16 image pipelines claim the path.
+for (const [label, catalog] of [
+ ["image", IMAGE_CATALOG],
+ ["audio", AUDIO_CATALOG],
+] as const) {
+ for (const group of catalog) {
+ for (const artifact of group.artifacts) {
+ if (artifact.format !== "bf16" || artifact.loadKind !== "pipeline") continue;
+ const row = curatedRowLabelFor(artifact.repoId, catalog, "dense-quant");
+ // Word boundary, not "(Fast)": `qualify` skips the bracket when the variant name already
+ // carries the word, as HiDream I1 (Fast (distilled)) does.
+ const claims = /\bFast\b/.test(row?.name ?? "");
+ assert.equal(
+ Boolean(claims),
+ label === "image" && artifact.denseQuantable === true,
+ `${label}: ${artifact.repoId} reads "${row?.name}" ${JSON.stringify(row?.tags)}`,
+ );
+ }
+ }
+}
+
+// Only compatible artifacts may receive a precision request.
+for (const [id, expected] of [
+ ["unsloth/Z-Image-Turbo-GGUF", true],
+ ["Tongyi-MAI/Z-Image-Turbo", true],
+ ["krea/Krea-2-Turbo", true],
+ ["stabilityai/sdxl-turbo", false],
+ ["stabilityai/stable-diffusion-xl-base-1.0", false],
+ ["unsloth/Z-Image-Turbo-unsloth-bnb-4bit", false],
+ ["ideogram-ai/ideogram-4-fp8", false],
+ ["ideogram-ai/ideogram-4-nf4-diffusers", false],
+] as const) {
+ assert.equal(curatedArtifactTakesDenseQuant(id, IMAGE_CATALOG), expected, id);
+}
+// Unknown ids defer to the loader.
+assert.equal(curatedArtifactTakesDenseQuant("someone/pasted", IMAGE_CATALOG), undefined);
+
+// A row that claims the fast path must be one the load request may actually send a precision for.
+for (const group of IMAGE_CATALOG) {
+ for (const artifact of group.artifacts) {
+ const row = curatedRowLabelFor(artifact.repoId, IMAGE_CATALOG, "dense-quant");
+ if (!/\bFast\b/.test(row?.name ?? "")) continue;
+ assert.notEqual(
+ curatedArtifactTakesDenseQuant(artifact.repoId, IMAGE_CATALOG),
+ false,
+ artifact.repoId,
+ );
+ }
+}
+
+// Single-artifact groups have nothing to tell apart, so they stay bare; only the qualifier moves.
+for (const id of [
+ "krea/Krea-2-Turbo",
+ "Alpha-VLLM/Lumina-Image-2.0",
+ "hunyuanvideo-community/HunyuanImage-2.1-Diffusers",
+]) {
+ const group = groupForRepoId(id, IMAGE_CATALOG);
+ assert.equal(group?.artifacts.length, 1, id);
+ for (const host of ["dense-quant", "accelerated"] as const) {
+ assert.deepEqual(curatedRowLabelFor(id, IMAGE_CATALOG, host)?.tags, [], `${id} ${host}`);
+ }
+ assert.ok(curatedRowLabelFor(id, IMAGE_CATALOG, "dense-quant")?.name.includes("(Fast)"), id);
+ assert.equal(curatedRowLabelFor(id, IMAGE_CATALOG, "accelerated")?.name.includes("(Fast)"), false, id);
+}
+
+// The row states a CAPABILITY of the artifact on this host, nothing about the request. Which
+// precision a load lands on also depends on Precision/Speed/Memory, the family deny list and a
+// per-card kernel probe; `resolved` reports that after the load. Predicting it here would mean
+// mirroring the backend selector's inputs, which is unbounded.
+{
+ const id = "Tongyi-MAI/Z-Image-Turbo";
+ // The chip is the STORED precision, exactly as on a host without the fast path.
+ assert.deepEqual(curatedRowLabelFor(id, IMAGE_CATALOG, "dense-quant"), {
+ name: "Z-Image-Turbo (Fast)",
+ tags: ["BF16"],
+ });
+ assert.deepEqual(curatedRowLabelFor(id, IMAGE_CATALOG, "accelerated"), {
+ name: "Z-Image-Turbo",
+ tags: ["BF16"],
+ });
+ // The host capability never rewrites a chip: chips describe the artifact, so they are identical on
+ // every host and the invented "FP8 / INT8" runtime pair appears nowhere. A stored-fp8 row shows
+ // FP8 as its own label, not as a claim about what will run.
+ for (const catalog of [IMAGE_CATALOG, VIDEO_CATALOG]) {
+ for (const group of catalog) {
+ for (const artifact of group.artifacts) {
+ const dense = curatedRowLabelFor(artifact.repoId, catalog, "dense-quant")?.tags ?? [];
+ const plain = curatedRowLabelFor(artifact.repoId, catalog, "accelerated")?.tags ?? [];
+ assert.deepEqual(dense, plain, artifact.repoId);
+ assert.equal(dense.includes("FP8 / INT8"), false, artifact.repoId);
+ }
+ }
+ }
+ // Closed and open agree, which is the invariant the qualifier has to keep.
+ assert.equal(
+ curatedDisplayNameFor(id, IMAGE_CATALOG, "dense-quant"),
+ curatedRowLabelFor(id, IMAGE_CATALOG, "dense-quant")?.name,
+ );
+ const rows = catalogToModelOptions(IMAGE_CATALOG, "dense-quant");
+ assert.equal(rows.find((o) => o.id === id)?.name, "Z-Image-Turbo (Fast)");
+}
+
+// Do not duplicate a qualifier already present in the variant name.
+assert.deepEqual(
+ curatedRowLabelFor("HiDream-ai/HiDream-I1-Fast", IMAGE_CATALOG, "dense-quant"),
+ { name: "HiDream I1 (Fast (distilled))", tags: ["BF16"] },
+);
+
+// Pre-quantised pipelines never claim dense quantisation.
+for (const id of [
+ "unsloth/Z-Image-Turbo-unsloth-bnb-4bit",
+ "ideogram-ai/ideogram-4-fp8",
+]) {
+ const row = curatedRowLabelFor(id, IMAGE_CATALOG, "dense-quant");
+ assert.ok(row && !row.name.includes("(Fast)"), `${id} reads "${row?.name}"`);
+}
// A gguf-only host loses exactly the artifacts the backend refuses there. Non-GGUF is NOT
// the test: diffusion runs on MPS and audio STT runs through the whisper.cpp sidecar.
@@ -370,8 +528,30 @@ for (const catalog of [IMAGE_CATALOG, VIDEO_CATALOG]) {
}
}
-// A non-GGUF artifact stating a parameter count must state its size, or the VRAM badge falls back to the QLoRA
-// estimator, which reads a pipeline as a language model (5B says 5.9 GB where Wan 2.2 TI2V is 30).
+// Publication source is not part of a precision label.
+for (const catalog of [IMAGE_CATALOG, VIDEO_CATALOG, AUDIO_CATALOG]) {
+ for (const group of catalog) {
+ for (const artifact of group.artifacts) {
+ for (const text of [
+ artifact.label,
+ curatedDisplayNameFor(artifact.repoId, catalog) ?? "",
+ ...catalogToModelOptions(catalog)
+ .filter((o) => o.id === artifact.repoId)
+ .map((o) => o.description ?? ""),
+ ]) {
+ assert.equal(
+ /official/i.test(text),
+ false,
+ `${artifact.repoId} still reads "${text}"`,
+ );
+ }
+ }
+ }
+}
+
+// A non-GGUF artifact stating a parameter count must state its size, or the VRAM badge falls back
+// to the QLoRA estimator, which reads a pipeline as a language model (5B says 5.9 GB where Wan 2.2
+// TI2V is 30).
for (const catalog of [IMAGE_CATALOG, VIDEO_CATALOG, AUDIO_CATALOG]) {
for (const group of catalog) {
for (const artifact of group.artifacts) {
@@ -390,8 +570,8 @@ const H3 = "MiniMaxAI/MiniMax-H3"; // 145 GB, tiers at 74/140 and 123/80
const fitsCurated = (id: string, gpuGb: number, systemRamGb: number) =>
curatedArtifactFitsDevice(id, VIDEO_CATALOG, { gpuGb, systemRamGb });
-// The resident 70% rule on the card alone. RAM is not a discrete GPU's budget: a pipeline with no measured tier
-// is placed wholly on the card, so 64 GB of RAM does not rescue a 12 GB card.
+// The resident 70% rule on the card alone. RAM is not a discrete GPU's budget: a pipeline with no
+// measured tier goes wholly on the card, so 64 GB of RAM does not rescue a 12 GB one.
assert.equal(fitsCurated(WAN, 48, 0), true);
assert.equal(fitsCurated(WAN, 40, 0), false);
assert.equal(fitsCurated(WAN, 12, 64), false);
@@ -407,9 +587,9 @@ assert.equal(fitsCurated(H3, 74, 100), false);
// A GGUF ladder self-fits via pickDefaultQuant, and an unknown id is not ours to judge.
assert.equal(fitsCurated("unsloth/MiniMax-H3-GGUF", 12, 64), undefined);
assert.equal(fitsCurated("someone/not-in-the-catalog", 12, 64), undefined);
-// Transcription retries a failed device load on CPU (stt_sidecar.py), so RAM is a real budget for stt: Whisper
-// Large runs on a card too small to hold it. A tts load rejects CPU offload (inference.py raise_if_offloaded), so
-// Orpheus is judged on the card.
+// Transcription retries a failed device load on CPU (stt_sidecar.py), so RAM is a real budget for
+// stt: Whisper Large runs on a card too small to hold it. A tts load rejects CPU offload
+// (inference.py raise_if_offloaded), so Orpheus is judged on the card.
assert.equal(
curatedArtifactFitsDevice("unsloth/whisper-large-v3", AUDIO_CATALOG, {
gpuGb: 4,
@@ -450,8 +630,8 @@ assert.equal(
const GB = 1024 ** 3;
-// Delegated to lib/gguf-fit, the Hub badge formula: 0.97 of the card (or saved VRAM budget) against weights +
-// KV, then RAM offload at 0.5. The old 0.7-of-each rule on raw file size matched neither the Hub nor the loader.
+// Delegated to lib/gguf-fit, the Hub badge formula: 0.97 of the card (or saved VRAM budget) against
+// weights + KV, then RAM offload at 0.5. The old 0.7-of-each rule on raw file size matched neither.
assert.equal(classifyGgufFit(10 * GB, { gpuGb: 24, systemRamGb: 64 }), "fits");
// 20 GiB needs 24.0: past the 23.28 budget, still inside the 24 GiB card.
assert.equal(classifyGgufFit(20 * GB, { gpuGb: 24, systemRamGb: 64 }), "marginal");
@@ -512,9 +692,9 @@ for (const gpuCount of [1, 2, 4]) {
);
}
-// Images / Video place a GGUF through the diffusion backend, whose 64 GiB unified budget is (total - 20%) * 0.85
-// = 43.5 GiB, per diffusion_memory.py. This rule allows 44.8; the llama.cpp one allows 62.1 and would promise
-// loads the planner refuses.
+// Images / Video place a GGUF through the diffusion backend, whose 64 GiB unified budget is
+// (total - 20% reserve) * 0.85 = 43.5 GiB (diffusion_memory.py). classifyMediaGgufFit allows 44.8;
+// the llama.cpp rule allows 62.1 and would promise loads the planner refuses.
assert.equal(classifyMediaGgufFit(40 * GB, 64, 0), "fits"); // 40 <= 44.8
assert.equal(classifyMediaGgufFit(50 * GB, 64, 0), "oom"); // past 44.8, no RAM tier
// The same 50 GiB file reads as fitting under the llama.cpp rule, the regression this guard
@@ -586,8 +766,8 @@ assert.equal(
.format,
"bnb-4bit",
);
-// 48 GB: still bnb-4bit. fp8 is family-denied for qwen-image (renders black) and the -FP8 repo ships prequant
-// .pt rather than single-file safetensors, so the old winner auto-routed to a download that 404s.
+// 48 GB: still bnb-4bit. fp8 is family-denied for qwen-image (renders black) and the -FP8 repo ships
+// prequant .pt rather than single-file safetensors, so the old winner auto-routed to a 404.
assert.equal(
pickDefaultArtifact(qwenGroup, { gpuGb: 48, systemRamGb: 64, isDownloaded: notDownloaded })
.format,
@@ -672,8 +852,8 @@ const hyimage = groupForRepoId(
IMAGE_CATALOG,
);
assert.ok(hyimage);
-// bf16 at both ends now: the QuantStack GGUF was unpublished, so the group has no quant ladder left. 50 GB still
-// fits a 24 GB card's 61.6 GB budget, so this asserts the group did not vanish with its GGUF.
+// bf16 at both ends now: the QuantStack GGUF was unpublished, so no quant ladder is left. 50 GB
+// still fits a 24 GB card's 61.6 GB budget, so this asserts the group did not vanish with it.
assert.equal(
pickDefaultArtifact(hyimage, { gpuGb: 24, systemRamGb: 64, isDownloaded: notDownloaded })
.repoId,
@@ -755,9 +935,9 @@ assert.equal(
.format,
"bf16",
);
-// The upper tier in the picker's units: 132 GiB VRAM is 141.7 decimal GB, past the 132 GB where the estimator
-// drops its host-RAM floor to 85 GB, and 85 GiB RAM is 91.3 GB. A decimal-GB tier table would wrongly send this
-// host to GGUF.
+// The upper tier in the picker's units: 132 GiB VRAM is 141.7 decimal GB, past the 132 GB where the
+// estimator drops its host-RAM floor to 85 GB, and 85 GiB RAM is 91.3 GB. A decimal-GB tier table
+// would wrongly send this host to GGUF.
assert.equal(
pickDefaultArtifact(h3, { gpuGb: 132, systemRamGb: 85, isDownloaded: notDownloaded })
.format,
@@ -812,9 +992,9 @@ assert.equal(
.format,
"gguf",
);
-// LTX-2.3 video carries the official BF16 single file (no FP8: the loader refuses its scaled-fp8 one), keeping
-// the ~50 GB Gemma3 encoder resident, so B200-class only. Looked up by the retired unsloth/LTX-2.3 id on purpose:
-// a pasted or persisted copy must still land on this group through the GGUF artifact's suffix-stripped key.
+// LTX-2.3 video carries the official BF16 single file (no FP8: the loader refuses its scaled-fp8
+// one), keeping the ~50 GB Gemma3 encoder resident, so B200-class only. Looked up by the retired
+// unsloth/LTX-2.3 id on purpose: a pasted copy must still land here via the suffix-stripped key.
const ltxGroup = groupForRepoId("unsloth/LTX-2.3", VIDEO_CATALOG);
assert.ok(ltxGroup);
assert.equal(ltxGroup.canonicalId, "Lightricks/LTX-2.3");
@@ -945,20 +1125,259 @@ assert.ok(!groupMatchesQuery(qwenGroup, "ideogram"));
assert.ok(groupMatchesQuery(ltx23, "ltx"));
assert.ok(groupMatchesQuery(ltx23, "lightricks/ltx-2.3"));
+const PREQUANT_ROWS = [
+ ["Tongyi-MAI/Z-Image-Turbo", "unsloth/Z-Image-Turbo-FP8"],
+ ["Qwen/Qwen-Image", "unsloth/Qwen-Image-FP8"],
+ ["Qwen/Qwen-Image-2512", "unsloth/Qwen-Image-2512-FP8"],
+ ["black-forest-labs/FLUX.1-schnell", "unsloth/FLUX.1-schnell-FP8"],
+ ["krea/Krea-2-Turbo", "unsloth/Krea-2-Turbo-FP8"],
+] as const;
+
+for (const [id, repo] of PREQUANT_ROWS) {
+ const hit = artifactForRepoId(id, IMAGE_CATALOG);
+ assert.ok(hit, id);
+ assert.equal(hit.artifact.prequantRepo, repo, id);
+ // Only a row the quantiser can take may name a checkpoint, else the fit rule sizes a load by an
+ // artifact the backend never fetches.
+ assert.equal(curatedArtifactTakesDenseQuant(id, IMAGE_CATALOG), true, id);
+ for (const scheme of ["fp8", "int8"] as const) {
+ const size = hit.artifact.prequantSizeGb?.[scheme];
+ assert.ok(typeof size === "number" && size > 0, `${id} ${scheme}`);
+ assert.ok((size as number) < (hit.artifact.approxSizeGb ?? 0), `${id} ${scheme}`);
+ }
+ assert.ok(hit.artifact.totalParams, id);
+ assert.ok(hit.artifact.approxSizeGb, id);
+}
+
+for (const catalog of [IMAGE_CATALOG, VIDEO_CATALOG, AUDIO_CATALOG]) {
+ for (const group of catalog) {
+ for (const artifact of group.artifacts) {
+ if (!artifact.prequantRepo && !artifact.prequantSizeGb) continue;
+ assert.equal(group.scope, "image", artifact.repoId);
+ assert.equal(artifact.format, "bf16", artifact.repoId);
+ assert.equal(artifact.loadKind, "pipeline", artifact.repoId);
+ assert.equal(artifact.denseQuantable, true, artifact.repoId);
+ assert.ok(artifact.prequantRepo, artifact.repoId);
+ assert.ok(artifact.prequantSizeGb, artifact.repoId);
+ }
+ }
+}
+
+// Z-Image-Turbo needs 42.9 GB of card dense and 34.9 GB pre-quantised under the 70% rule.
+const zTurboId = "Tongyi-MAI/Z-Image-Turbo";
+assert.equal(
+ curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, { gpuGb: 40, systemRamGb: 128 }),
+ false,
+);
+for (const schemes of [["fp8"], ["int8"]]) {
+ assert.equal(
+ curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, {
+ gpuGb: 40,
+ systemRamGb: 128,
+ denseQuantSchemes: schemes,
+ }),
+ true,
+ schemes[0],
+ );
+}
+assert.equal(
+ curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, {
+ gpuGb: 40,
+ systemRamGb: 128,
+ denseQuantSchemes: [],
+ }),
+ false,
+);
+assert.equal(
+ curatedArtifactFitsDevice("Qwen/Qwen-Image", IMAGE_CATALOG, {
+ gpuGb: 64,
+ systemRamGb: 128,
+ denseQuantSchemes: ["fp8"],
+ }),
+ true,
+);
+assert.equal(
+ curatedArtifactFitsDevice("Qwen/Qwen-Image", IMAGE_CATALOG, {
+ gpuGb: 64,
+ systemRamGb: 128,
+ denseQuantSchemes: ["int8"],
+ }),
+ false,
+);
+// The reported schemes are a LADDER. `_pipeline_planned_denoiser_scheme` walks past an int8 rung
+// whose artifact-sized plan still offloads and seeds fp8 instead, so sizing by schemes[0] alone
+// refuses a card the pipeline really runs on (Qwen-Image int8 is 25.4 GB against fp8's 19.06).
+const qwenImageGroup = groupForRepoId("Qwen/Qwen-Image", IMAGE_CATALOG);
+assert.ok(qwenImageGroup);
+assert.equal(
+ curatedArtifactFitsDevice("Qwen/Qwen-Image", IMAGE_CATALOG, {
+ gpuGb: 64,
+ systemRamGb: 128,
+ denseQuantSchemes: ["int8", "fp8"],
+ }),
+ true,
+);
+assert.equal(
+ pickDefaultArtifact(qwenImageGroup, {
+ gpuGb: 64,
+ systemRamGb: 128,
+ denseQuantSchemes: ["int8", "fp8"],
+ isDownloaded: notDownloaded,
+ }).repoId,
+ "Qwen/Qwen-Image",
+);
+// A ladder no rung of which fits is still refused, and still routes to the bnb row.
+assert.equal(
+ curatedArtifactFitsDevice("Qwen/Qwen-Image", IMAGE_CATALOG, {
+ gpuGb: 24,
+ systemRamGb: 128,
+ denseQuantSchemes: ["int8", "fp8"],
+ }),
+ false,
+);
+for (const id of ["black-forest-labs/FLUX.1-dev", "stabilityai/sdxl-turbo"]) {
+ assert.equal(
+ curatedArtifactFitsDevice(id, IMAGE_CATALOG, {
+ gpuGb: 40,
+ systemRamGb: 128,
+ denseQuantSchemes: ["fp8"],
+ }),
+ curatedArtifactFitsDevice(id, IMAGE_CATALOG, { gpuGb: 40, systemRamGb: 128 }),
+ id,
+ );
+}
+
+// The "Fits on device" group filter sizes a row like the badge and the router do: Krea-2-Turbo is
+// 18 GB dense and ~12 GB as its hosted int8 / fp8 artifact, with no GGUF fallback row, so the dense
+// figure hid the whole group on a card the backend loads it on.
+const kreaTurboGroup = groupForRepoId("krea/Krea-2-Turbo", IMAGE_CATALOG);
+assert.ok(kreaTurboGroup);
+assert.equal(
+ catalogGroupFitsDevice(kreaTurboGroup, { gpuGb: 24, systemRamGb: 0 }, notDownloaded),
+ false,
+);
+assert.equal(
+ catalogGroupFitsDevice(
+ kreaTurboGroup,
+ { gpuGb: 24, systemRamGb: 0, denseQuantSchemes: ["int8", "fp8"] },
+ notDownloaded,
+ ),
+ true,
+);
+// A card too small for the quantised artifact too is still refused.
+assert.equal(
+ catalogGroupFitsDevice(
+ kreaTurboGroup,
+ { gpuGb: 16, systemRamGb: 0, denseQuantSchemes: ["int8", "fp8"] },
+ notDownloaded,
+ ),
+ false,
+);
+
+// The router follows the same rule as the fit verdict.
+const zTurboGroup = groupForRepoId(zTurboId, IMAGE_CATALOG);
+assert.ok(zTurboGroup);
+assert.equal(
+ pickDefaultArtifact(zTurboGroup, {
+ gpuGb: 40,
+ systemRamGb: 128,
+ isDownloaded: notDownloaded,
+ }).format,
+ "bnb-4bit",
+);
+assert.equal(
+ pickDefaultArtifact(zTurboGroup, {
+ gpuGb: 40,
+ systemRamGb: 128,
+ denseQuantSchemes: ["fp8"],
+ isDownloaded: notDownloaded,
+ }).repoId,
+ zTurboId,
+);
+assert.equal(
+ pickDefaultArtifact(zTurboGroup, {
+ gpuGb: 24,
+ systemRamGb: 128,
+ denseQuantSchemes: ["fp8"],
+ isDownloaded: notDownloaded,
+ }).format,
+ "bnb-4bit",
+);
+
+assert.deepEqual(curatedRowLabelFor(zTurboId, IMAGE_CATALOG, "dense-quant", ["fp8"]), {
+ name: "Z-Image-Turbo (Fast FP8)",
+ tags: ["BF16"],
+});
+assert.deepEqual(curatedRowLabelFor(zTurboId, IMAGE_CATALOG, "dense-quant", ["int8"]), {
+ name: "Z-Image-Turbo (Fast FP8)",
+ tags: ["BF16"],
+});
+assert.equal(
+ curatedRowLabelFor(zTurboId, IMAGE_CATALOG, "dense-quant", ["fp8", "int8"])?.name,
+ "Z-Image-Turbo (Fast FP8)",
+);
+assert.equal(
+ curatedRowLabelFor(zTurboId, IMAGE_CATALOG, "dense-quant", [])?.name,
+ "Z-Image-Turbo (Fast)",
+);
+assert.equal(
+ curatedDisplayNameFor(zTurboId, IMAGE_CATALOG, "dense-quant", ["fp8"]),
+ "Z-Image-Turbo (Fast FP8)",
+);
+assert.equal(
+ catalogToModelOptions(IMAGE_CATALOG, "dense-quant", ["int8"]).find((o) => o.id === zTurboId)
+ ?.name,
+ "Z-Image-Turbo (Fast FP8)",
+);
+assert.deepEqual(
+ curatedRowLabelFor("HiDream-ai/HiDream-I1-Fast", IMAGE_CATALOG, "dense-quant", ["fp8"]),
+ { name: "HiDream I1 (Fast (distilled))", tags: ["BF16"] },
+);
+assert.equal(
+ curatedRowLabelFor("HiDream-ai/HiDream-I1-Dev", IMAGE_CATALOG, "dense-quant", ["fp8"])?.name,
+ "HiDream I1 (Dev (distilled)) (Fast FP8)",
+);
+
+assert.deepEqual(
+ curatedRowLabelFor("MiniMaxAI/MiniMax-H3", VIDEO_CATALOG, "dense-quant", ["fp8"]),
+ { name: "MiniMax H3 (Fast FP8)", tags: ["BF16"] },
+);
+assert.deepEqual(
+ curatedRowLabelFor("MiniMaxAI/MiniMax-H3", VIDEO_CATALOG, "dense-quant", ["int8"]),
+ { name: "MiniMax H3 (Fast FP8)", tags: ["BF16"] },
+);
+assert.equal(
+ curatedRowLabelFor("unsloth/MiniMax-H3-GGUF", VIDEO_CATALOG, "dense-quant", ["fp8"])?.name,
+ "MiniMax-H3-GGUF (Slow)",
+);
+for (const schemes of [[], ["fp8"], ["int8"]]) {
+ for (const catalog of [IMAGE_CATALOG, VIDEO_CATALOG]) {
+ for (const group of catalog) {
+ for (const artifact of group.artifacts) {
+ assert.deepEqual(
+ curatedRowLabelFor(artifact.repoId, catalog, "dense-quant", schemes)?.tags ?? [],
+ curatedRowLabelFor(artifact.repoId, catalog, "accelerated")?.tags ?? [],
+ `${artifact.repoId} ${schemes.join(",") || "none"}`,
+ );
+ }
+ }
+ }
+}
+
console.log("model-catalog check: all assertions passed");
-// Opt-in `--network` pass: every failure it catches was reported by hand, a link that 401s on an undeclared
-// gated repo or 404s after a rename. Anonymous on purpose, since that is what a fresh install sees; only a
-// definitive verdict fails the run.
+// Opt-in `--network` pass: every failure it catches was reported by hand, a link that 401s on an
+// undeclared gated repo or 404s after a rename. Anonymous on purpose, since that is what a fresh
+// install sees; only a definitive verdict fails the run.
const HF_API = "https://huggingface.co/api/models";
const HF_RESOLVE = "https://huggingface.co";
const NETWORK_ATTEMPTS = 3;
/** Per-attempt wall clock, headers and body together. */
const NETWORK_TIMEOUT_MS = 20_000;
-/** Wall clock for the whole network pass, under the workflow's 10-minute timeout. Bounding each attempt is not
- * enough: ~53 repos at NETWORK_BATCH 4 is 14 serial batches, so a stalling peer would be killed at 10 as a red
- * run. Past the deadline requests short-circuit to "no opinion" and the check exits 0. */
+/** Wall clock for the whole network pass, under the workflow's 10-minute timeout. Bounding each
+ * attempt is not enough: ~53 repos at NETWORK_BATCH 4 is 14 serial batches, so a stalling peer
+ * would be killed at 10 as a red run. Past the deadline requests are "no opinion" and we exit 0. */
const NETWORK_DEADLINE_MS = 7 * 60 * 1000;
/** Set when the network pass starts; Infinity keeps the offline assertions unbounded. */
let networkDeadlineAt = Number.POSITIVE_INFINITY;
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
index 6411384a638..616694ccf26 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
@@ -1,10 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-// One canonical name per diffusion model, its published artifacts (GGUF quants, prequant FP8 / bnb-4bit repos,
-// official BF16 pipelines), and a deterministic router picking the best artifact for the device. Pure helpers, no
-// React/DOM deps. See model-catalog.check.ts.
+// One canonical name per diffusion model, its published artifacts (GGUF quants, prequant FP8 /
+// bnb-4bit repos, official BF16 pipelines), and a deterministic router picking the best artifact
+// for the device. Pure helpers, no React/DOM deps. See model-catalog.check.ts.
+import { normalizeDenseQuantSchemes } from "../../../../lib/dense-quant-schemes.ts";
import {
type GgufFitClass,
classifyGgufFit as classifyGgufFitForDevice,
@@ -12,12 +13,17 @@ import {
import {
type HostClass,
curatedArtifactIsOfferable,
+ densePerfSuffix,
+ ggufPerfSuffix,
h3PerfSuffix,
+ hostRunsDenseQuant,
} from "./host-artifact-policy.ts";
import type { ModelCapabilities } from "./model-capabilities";
import type { ModelOption } from "./types";
export type ArtifactFormat = "gguf" | "fp8" | "bnb-4bit" | "bf16";
+/** The dense torchao schemes a host reports in `/api/system.dense_quant_schemes`. */
+export type DenseQuantScheme = "fp8" | "int8";
export type LoadKind = "gguf" | "single_file" | "pipeline";
export interface ModelArtifact {
@@ -27,8 +33,14 @@ export interface ModelArtifact {
loadKind: LoadKind;
/** single_file loads name their exact checkpoint inside the repo. */
filename?: string;
- /** Second-level row label ("GGUF", "FP8", "BF16 (official)", "BF16 - 720p"). */
+ /** Second-level row label ("GGUF", "FP8", "BF16", "BF16 - 720p"). */
label: string;
+ /** Whether this bf16 pipeline exposes a transformer eligible for dense quantisation. */
+ denseQuantable?: boolean;
+ /** Hosted pre-quantised checkpoint an auto load fetches instead of the dense bf16 shards. */
+ prequantRepo?: string;
+ /** Size (GB) per scheme; int8 stores scales fp8 does not. TRANSFORMER ONLY. */
+ prequantSizeGb?: Readonly<Partial<Record<DenseQuantScheme, number>>>;
/** Curated resident-size estimate for routing. Omitted = unknown: never auto-picked unless
* downloaded. GGUF omits it too, since its quant ladder self-fits via pickDefaultQuant. */
approxSizeGb?: number;
@@ -61,9 +73,9 @@ export interface CatalogGroup {
artifacts: ModelArtifact[];
/** Cross-owner ids that resolve to this group. Suffix stripping never merges two owners on its own. */
aliases?: readonly string[];
- /** What the model can do, for the row's capability glyphs. Same fallback rule as `ModelArtifact.totalParams`:
- * the Hub listing's tags win where it returns the row, since `detectCapabilities` reads tags then repo-name
- * keywords and a name like "MiniMax-H3-GGUF" says nothing about the audio track the model emits. */
+ /** What the model can do, for the row's capability glyphs. Same fallback rule as
+ * `ModelArtifact.totalParams`: the Hub listing's tags win, since a name like "MiniMax-H3-GGUF"
+ * says nothing about the audio track the model emits. */
capabilities?: Partial<ModelCapabilities>;
}
@@ -113,9 +125,10 @@ const bf16Pipeline = (
repoId,
format: "bf16",
loadKind: "pipeline",
- label: "BF16 (official)",
+ label: "BF16",
approxSizeGb,
keywords: ["bf16", "safetensors", "full precision"],
+ denseQuantable: true,
...extra,
});
@@ -131,9 +144,10 @@ const bf16Single = (
format: "bf16",
loadKind: "single_file",
filename,
- label: "BF16 (official)",
+ label: "BF16",
approxSizeGb,
keywords: ["bf16", "safetensors", "full precision"],
+ denseQuantable: true,
...extra,
});
@@ -147,7 +161,11 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
description: "Text-to-image",
scope: "image",
artifacts: [
- bf16Pipeline("Tongyi-MAI/Z-Image-Turbo", 30, { totalParams: 6154908736 }),
+ bf16Pipeline("Tongyi-MAI/Z-Image-Turbo", 30, {
+ totalParams: 6154908736,
+ prequantRepo: "unsloth/Z-Image-Turbo-FP8",
+ prequantSizeGb: { fp8: 5.86, int8: 5.86 },
+ }),
bnb4bit("unsloth/Z-Image-Turbo-unsloth-bnb-4bit", 8, { totalParams: 3210823936 }),
gguf("unsloth/Z-Image-Turbo-GGUF"),
],
@@ -164,14 +182,17 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
displayName: "Qwen-Image 2512",
description: "Text-to-image",
scope: "image",
- // The prequant repo is real and public and the backend reaches its int8 half through
- // prequant_repos. It has no artifact row here, so alias it to keep a pasted id finding it.
+ // The backend reaches the prequant repo's int8 half through prequant_variant_repos. It has no
+ // artifact row here, so alias it to keep a pasted id finding it.
aliases: ["unsloth/Qwen-Image-2512-FP8"],
artifacts: [
- bf16Pipeline("Qwen/Qwen-Image-2512", 54, { totalParams: 20430401088 }),
- // No FP8 row: unsloth/Qwen-Image-2512-FP8 holds torch prequant .pt checkpoints, not a single-file
- // .safetensors, and fp8 is denied for this family anyway (_FAMILY_SCHEME_DENY: qwen-image renders every frame
- // black under fp8).
+ bf16Pipeline("Qwen/Qwen-Image-2512", 54, {
+ totalParams: 20430401088,
+ prequantRepo: "unsloth/Qwen-Image-2512-FP8",
+ prequantSizeGb: { fp8: 19.06, int8: 25.4 },
+ }),
+ // No FP8 row: that repo holds torch prequant .pt checkpoints, not a single-file .safetensors;
+ // the backend seeds them through the row above.
bnb4bit("unsloth/Qwen-Image-2512-unsloth-bnb-4bit", 14, { totalParams: 10850871408 }),
gguf("unsloth/Qwen-Image-2512-GGUF"),
],
@@ -182,7 +203,11 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
description: "Text-to-image",
scope: "image",
artifacts: [
- bf16Pipeline("Qwen/Qwen-Image", 54, { totalParams: 20430401088 }),
+ bf16Pipeline("Qwen/Qwen-Image", 54, {
+ totalParams: 20430401088,
+ prequantRepo: "unsloth/Qwen-Image-FP8",
+ prequantSizeGb: { fp8: 19.06, int8: 31.73 },
+ }),
gguf("unsloth/Qwen-Image-GGUF"),
],
},
@@ -194,7 +219,12 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
artifacts: [
// Apache-2.0 but still gated on the Hub (gated: "auto", a contact-info form), so an anonymous
// download 401s exactly like dev. The licence and the gate are independent.
- bf16Pipeline("black-forest-labs/FLUX.1-schnell", 32, { gated: true, totalParams: 11891178560 }),
+ bf16Pipeline("black-forest-labs/FLUX.1-schnell", 32, {
+ gated: true,
+ totalParams: 11891178560,
+ prequantRepo: "unsloth/FLUX.1-schnell-FP8",
+ prequantSizeGb: { fp8: 11.09, int8: 14.13 },
+ }),
gguf("unsloth/FLUX.1-schnell-GGUF"),
],
},
@@ -263,7 +293,14 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
scope: "image",
// Gated on the Hub, and the group's only artifact, so a bare click has nothing open to fall
// through to: the picker must show the gate rather than start a download that 401s.
- artifacts: [bf16Pipeline("krea/Krea-2-Turbo", 18, { gated: true, totalParams: 12820073036 })],
+ artifacts: [
+ bf16Pipeline("krea/Krea-2-Turbo", 18, {
+ gated: true,
+ totalParams: 12820073036,
+ prequantRepo: "unsloth/Krea-2-Turbo-FP8",
+ prequantSizeGb: { fp8: 11.95, int8: 12.19 },
+ }),
+ ],
},
{
// 2.6B DiT + Gemma2-2B encoder, ~11 GB bf16-resident (ships fp32, cast on load). Apache-2.0,
@@ -275,14 +312,12 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
artifacts: [bf16Pipeline("Alpha-VLLM/Lumina-Image-2.0", 11, { totalParams: 2609769152 })],
},
{
- // 17B dual-stream 2K-native DiT with a Qwen2.5-VL encoder. bf16 only: this used to carry
- // gguf("QuantStack/HunyuanImage-2.1-GGUF") as the consumer route, and that repo was unpublished. An entry the
- // Hub cannot serve renders as a one-click download that fails partway through, the exact case
- // model-catalog-network-check exists to find. No vetted replacement: unsloth has an FP8 mirror but no GGUF, and
- // the third-party GGUF repos are a different lineage. At 24 GB the bf16 still fits the 61.6 GB budget, so the
- // group stays visible; the quant ladder is what is gone. The mirror guider components load natively on
- // diffusers 0.39. The retired GGUF repo 404s to an authed request and 401s anonymously; QuantStack itself is
- // alive and still ships its other GGUF repos, so this is one model withdrawn rather than a publisher going away.
+ // 17B dual-stream 2K-native DiT with a Qwen2.5-VL encoder; the mirror's guider components load
+ // natively on diffusers 0.39. bf16 only: the consumer route used to
+ // be gguf("QuantStack/HunyuanImage-2.1-GGUF"), which was unpublished (404 authed, 401 anonymous)
+ // and has no vetted replacement, and an unservable entry renders as a download that fails partway
+ // through. QuantStack still ships its other GGUF repos, so this is one model withdrawn. The bf16
+ // still fits the 61.6 GB budget, so the group stays visible; only the quant ladder is gone.
canonicalId: "hunyuanvideo-community/HunyuanImage-2.1-Diffusers",
displayName: "HunyuanImage 2.1",
description: "Text-to-image",
@@ -292,9 +327,9 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
],
},
{
- // 17B MoE DiT + four text encoders. The MIT repos ship no Llama text_encoder_4, so the backend assembles it
- // from the open unsloth mirror at load time (+16 GB): ~63 GB bf16-resident, a datacenter pick. Full is the
- // undistilled base, Dev and Fast its distillations.
+ // 17B MoE DiT + four text encoders. The MIT repos ship no Llama text_encoder_4, so the backend
+ // assembles it from the open unsloth mirror at load time (+16 GB): ~63 GB bf16-resident, a
+ // datacenter pick. Full is the undistilled base; Dev and Fast are its distillations.
canonicalId: "HiDream-ai/HiDream-I1-Full",
displayName: "HiDream I1",
description: "Text-to-image",
@@ -332,7 +367,14 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
displayName: "SDXL Turbo",
description: "Text-to-image",
scope: "image",
- artifacts: [bf16Pipeline("stabilityai/sdxl-turbo", 8, { label: "Safetensors", totalParams: 2567463684 })],
+ artifacts: [
+ // SDXL uses a UNet rather than a transformer.
+ bf16Pipeline("stabilityai/sdxl-turbo", 8, {
+ label: "Safetensors",
+ totalParams: 2567463684,
+ denseQuantable: false,
+ }),
+ ],
},
{
canonicalId: "stabilityai/stable-diffusion-xl-base-1.0",
@@ -340,9 +382,11 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
description: "Text-to-image",
scope: "image",
artifacts: [
+ // SDXL uses a UNet rather than a transformer.
bf16Pipeline("stabilityai/stable-diffusion-xl-base-1.0", 8, {
label: "Safetensors",
totalParams: 2567463684,
+ denseQuantable: false,
}),
],
},
@@ -358,12 +402,12 @@ export const VIDEO_CATALOG: CatalogGroup[] = [
capabilities: { audio: true },
artifacts: [
bf16Pipeline("MiniMaxAI/MiniMax-H3", 145, {
- // Cluster measurements at the default 1344x768, 124-frame preset: the lower-GPU tier holds one 66 GB
- // component at a time and keeps the full model in RAM. THESE ARE GiB and the backend estimators they mirror
- // are decimal GB, so the two sets must never be copied across: nvidia.py memory_total_gb and main.py
- // available_gb divide by 1024, while video.py divides runtime bytes by 1_000_000_000. Converted, these are
- // 79.5 / 150.3 and 132.1 / 85.9 GB, matching the estimators' 78.74 / 150 and 132 / 85. Copying the decimal
- // figures across applies the conversion twice and sends capable hosts to GGUF.
+ // Cluster measurements at the default 1344x768, 124-frame preset: the lower-GPU tier holds one
+ // 66 GB component at a time and keeps the full model in RAM. THESE ARE GiB (nvidia.py
+ // memory_total_gb and main.py available_gb divide by 1024) while the backend estimators they
+ // mirror are decimal GB (video.py divides by 1_000_000_000), so never copy figures across:
+ // converted, these are 79.5 / 150.3 and 132.1 / 85.9 GB, matching the estimators' 78.74 / 150
+ // and 132 / 85. Copying applies the conversion twice and sends capable hosts to GGUF.
offloadFitTiers: [
{ gpuGb: 74, systemRamGb: 140 },
{ gpuGb: 123, systemRamGb: 80 },
@@ -386,12 +430,12 @@ export const VIDEO_CATALOG: CatalogGroup[] = [
],
},
{
- // The distilled 2.3 release: Lightricks' own bf16/fp8 single-file DiT checkpoints (loaded against the
- // already-trusted LTX-2 base for the VAE / Gemma3 encoder) plus the GGUF quants. The single-file ones keep the
- // ~50 GB encoder in bf16, so consumer GPUs route to GGUF. Keyed on the artifact that exists: unsloth/LTX-2.3
- // was never published (404), and an `unsloth/*` id that is not an artifact clears both the picker's owner guard
- // and the backend's, so a pick that reached the fall-through was loaded as a pipeline and only died at the Hub.
- // Lightricks/LTX-2.3 IS an artifact below, so that fall-through cannot fire.
+ // The distilled 2.3 release: Lightricks' own bf16/fp8 single-file DiT checkpoints (loaded against
+ // the already-trusted LTX-2 base for the VAE / Gemma3 encoder) plus the GGUF quants. The
+ // single-file ones keep the ~50 GB encoder in bf16, so consumer GPUs route to GGUF.
+ // Keyed on an artifact that exists: unsloth/LTX-2.3 was never published (404), and an `unsloth/*`
+ // id that is not an artifact clears both owner guards, so such a pick reached the fall-through,
+ // loaded as a pipeline and died at the Hub. Lightricks/LTX-2.3 IS an artifact below.
canonicalId: "Lightricks/LTX-2.3",
displayName: "LTX 2.3 distilled",
description: "Text-to-video with audio",
@@ -404,8 +448,8 @@ export const VIDEO_CATALOG: CatalogGroup[] = [
90,
),
// No FP8 artifact: the LTX-2.3 loader refuses the official scaled-FP8 single file (it carries
- // .weight_scale/.input_scale), so a click would start a ~76 GB download that always fails. 21.0B is what the
- // Hub reports, and carrying it keeps the row identical when offline.
+ // .weight_scale/.input_scale), so a click would start a ~76 GB download that always fails.
+ // 21.0B is what the Hub reports, and carrying it keeps the row identical when offline.
gguf("unsloth/LTX-2.3-GGUF", { totalParams: 21_005_004_544 }),
],
},
@@ -437,9 +481,8 @@ export const VIDEO_CATALOG: CatalogGroup[] = [
description: "Text-to-video",
scope: "video",
artifacts: [
- // Highest-quality first: pickDefaultArtifact sorts only by FORMAT, so these two bf16 artifacts keep catalog
- // order and the fit loop returns the first that fits. 720p (52 GB) precedes 480p (40 GB), so an 80 GB card
- // picks 720p.
+ // Highest-quality first: pickDefaultArtifact sorts only by FORMAT, so these two bf16 artifacts
+ // keep catalog order and the fit loop takes the first that fits (an 80 GB card gets 720p).
bf16Pipeline("hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v", 52, {
label: "BF16 - 720p",
keywords: ["bf16", "720p"],
@@ -454,10 +497,10 @@ export const VIDEO_CATALOG: CatalogGroup[] = [
},
];
-// The Audio page's curated list. tts groups load into the main slot via /api/inference/load (Orpheus is the only
-// family the llama.cpp TTS path also serves as GGUF); native TTS families use their official
-// Transformers/Diffusers interfaces. stt groups map to the dictation sidecar models in stt-model-catalog.ts, so
-// their sizes are informational only.
+// The Audio page's curated list. tts groups load into the main slot via /api/inference/load (Orpheus
+// is the only family the llama.cpp TTS path also serves as GGUF); native TTS families use their
+// official Transformers/Diffusers interfaces. stt groups map to the dictation sidecar models in
+// stt-model-catalog.ts, so their sizes are informational only.
export const AUDIO_CATALOG: CatalogGroup[] = [
{
canonicalId: "unsloth/orpheus-3b-0.1-ft",
@@ -561,9 +604,8 @@ export const AUDIO_CATALOG: CatalogGroup[] = [
],
},
// Llasa is deliberately absent: it speaks XCodec2 (65,536 <|s_N|> tokens), which is in neither
- // _AUDIO_TOKEN_PATTERNS nor AudioCodecManager, so a curated row loaded and then failed at generation. Unsloth
- // can still TRAIN Llasa (unsloth_Llasa-3B.yaml); this catalog only feeds the Generate picker. Re-add both rows
- // together with an xcodec2 decoder.
+ // _AUDIO_TOKEN_PATTERNS nor AudioCodecManager, so a curated row loaded then failed at generation.
+ // Training still works (unsloth_Llasa-3B.yaml). Re-add both rows with an xcodec2 decoder.
{
canonicalId: "unslothai/Qwen3-ASR-0.6B-GGUF",
displayName: "Qwen3-ASR 0.6B",
@@ -776,9 +818,9 @@ export function curatedTotalParamsFor(
return params && params > 0 ? params : undefined;
}
-/** Curated capabilities for any id belonging to a group that declares them. Same fallback rule as
- * `curatedTotalParamsFor`: the listing's tags win. Group-level, since every artifact of a model can do what the
- * model can do. */
+/** Curated capabilities for any id in a group that declares them. Same fallback rule as
+ * `curatedTotalParamsFor`: the listing's tags win. Group-level, since every artifact of a model
+ * can do what the model can do. */
export function curatedCapabilitiesFor(
repoId: string,
catalog: CatalogGroup[],
@@ -790,8 +832,8 @@ export function curatedCapabilitiesFor(
vision: declared?.vision ?? false,
reasoning: declared?.reasoning ?? false,
audio: declared?.audio ?? false,
- // From the group's scope rather than a declaration: every catalog entry is a generator of one or the other, and
- // the scope already says which. Beats the name heuristic, which has to guess a family from a repo id.
+ // From the group's scope, not a declaration: every entry generates one or the other and the
+ // scope already says which. Beats the name heuristic, which guesses a family from a repo id.
imageGen: declared?.imageGen ?? group.scope === "image",
videoGen: declared?.videoGen ?? group.scope === "video",
};
@@ -803,40 +845,114 @@ export function curatedDisplayNameFor(
repoId: string,
catalog: CatalogGroup[],
host: HostClass = "unknown",
+ denseQuantSchemes: readonly string[] = [],
): string | null {
const hit = artifactForRepoId(repoId, catalog);
if (!hit) return null;
- // A row that earns a speed qualifier must read the same closed as open: this helper names the trigger and
- // curatedRowLabelFor names the row, so a divergence would rename the model as the popover opens.
- if (h3PerfSuffix(repoId, host)) {
- return curatedRowLabelFor(repoId, catalog, host)?.name ?? hit.group.displayName;
+ // A row with a speed qualifier must read the same closed as open: this names the trigger and
+ // curatedRowLabelFor names the row, so a divergence renames the model as the popover opens.
+ if (curatedPerfSuffix(hit, host, denseQuantSchemes)) {
+ return (
+ curatedRowLabelFor(repoId, catalog, host, denseQuantSchemes)?.name ??
+ hit.group.displayName
+ );
}
return hit.group.artifacts.length > 1
? `${hit.group.displayName} (${hit.artifact.label})`
: hit.group.displayName;
}
-// Artifact labels are written as "FORMAT" or "FORMAT - QUALIFIER". The format head and a resolution qualifier are
-// chips; anything else stays in the name, since it is the only thing telling two rows of one group apart.
+// Artifact labels are "FORMAT" or "FORMAT - QUALIFIER". The format head and a resolution qualifier
+// become chips; anything else stays in the name, as it is what tells two rows of a group apart.
const LABEL_PART_SEPARATOR = " - ";
-const OFFICIAL_SUFFIX_RE = /\s*\(official\)$/i;
const GGUF_SUFFIX_RE = /-gguf$/i;
const RESOLUTION_RE = /^\d{3,4}p$/i;
-/** A curated row as name plus chips. The name used to carry the artifact inside brackets ("MiniMax H3 (BF16
- * (official))"), which pushed the part a user scans for behind the part they do not. Null for ids outside the
- * catalog. */
+/** Whether a known artifact can accept transformer quantisation. Unknown ids defer to the backend. */
+export function curatedArtifactTakesDenseQuant(
+ repoId: string,
+ catalog: CatalogGroup[],
+): boolean | undefined {
+ const hit = artifactForRepoId(repoId, catalog);
+ if (!hit) return undefined;
+ // GGUF reaches quantisation through dense base-weight substitution.
+ if (hit.artifact.format === "gguf") return true;
+ // Other artifacts must contain a dense bf16 transformer.
+ return (
+ hit.artifact.format === "bf16" &&
+ hit.artifact.loadKind === "pipeline" &&
+ hit.artifact.denseQuantable === true
+ );
+}
+
+/** Whether this row runs the dense quant path on this host.
+ *
+ * Artifact and host only, deliberately. Which precision a load ENDS UP at also depends on the
+ * request (Precision, Speed, Memory), the family deny list and a per-card kernel probe, none of
+ * which an unclicked row can see. So the row states the capability it is sure of, and `resolved`
+ * reports the precision that actually ran. */
+function artifactUsesDenseQuant(
+ group: CatalogGroup,
+ artifact: ModelArtifact,
+ host: HostClass,
+): boolean {
+ return hostRunsDenseQuant(host) && artifactTakesDenseQuant(group, artifact);
+}
+
+/** The artifact half of the rule above, asked from a device budget rather than a host class. */
+function artifactTakesDenseQuant(
+ group: CatalogGroup,
+ artifact: ModelArtifact,
+): boolean {
+ return (
+ group.scope === "image" &&
+ artifact.format === "bf16" &&
+ artifact.loadKind === "pipeline" &&
+ artifact.denseQuantable === true
+ );
+}
+
+/** Speed qualifier from the dense-quant path or an artifact-specific rule, naming the scheme the
+ * host reported. */
+function curatedPerfSuffix(
+ hit: { group: CatalogGroup; artifact: ModelArtifact },
+ host: HostClass,
+ denseQuantSchemes: readonly string[],
+): string | null {
+ // Every diffusion GGUF is the slow row, not just H3's. Audio is out: its GGUF rows run the
+ // whisper.cpp sidecar, which has no dense sibling to be slow against.
+ if (
+ hit.artifact.format === "gguf" &&
+ (hit.group.scope === "image" || hit.group.scope === "video")
+ ) {
+ return ggufPerfSuffix(host);
+ }
+ if (artifactUsesDenseQuant(hit.group, hit.artifact, host)) {
+ return densePerfSuffix(denseQuantSchemes);
+ }
+ return h3PerfSuffix(hit.artifact.repoId, host, denseQuantSchemes);
+}
+
+/** A curated row as name plus chips. The name used to carry the artifact in brackets ("MiniMax H3
+ * (BF16)"), pushing the part a user scans for behind the part they do not. Null for unknown ids. */
export function curatedRowLabelFor(
repoId: string,
catalog: CatalogGroup[],
host: HostClass = "unknown",
+ denseQuantSchemes: readonly string[] = [],
): { name: string; tags: string[] } | null {
const hit = artifactForRepoId(repoId, catalog);
if (!hit) return null;
// Only where the host can run both rows, so the qualifier compares things the user can pick
// between rather than advertising a speed they cannot have.
- const perf = h3PerfSuffix(repoId, host);
- const qualify = (name: string) => (perf ? `${name} (${perf})` : name);
+ const perf = curatedPerfSuffix(hit, host, denseQuantSchemes);
+ // Matched on the qualifier's first word: testing "Fast FP8" in full would miss the "Fast" already
+ // in the name and read "HiDream I1 (Fast (distilled)) (Fast FP8)".
+ const perfWord = perf?.split(" ")[0];
+ const qualify = (name: string) =>
+ perf && perfWord && !new RegExp(`\\b${perfWord}\\b`, "i").test(name)
+ ? `${name} (${perf})`
+ : name;
// GGUF reads like a text model's row: the repo name already ends in -GGUF, so a chip would only repeat the suffix.
if (hit.artifact.format === "gguf") {
const leaf = hit.artifact.repoId.split("/").pop() ?? hit.artifact.repoId;
@@ -845,7 +961,9 @@ export function curatedRowLabelFor(
// A group with one artifact has nothing to distinguish, so it stays bare.
if (hit.group.artifacts.length <= 1) return { name: qualify(hit.group.displayName), tags: [] };
const [format, ...rest] = hit.artifact.label.split(LABEL_PART_SEPARATOR);
- const tags = [format.replace(OFFICIAL_SUFFIX_RE, "").trim()].filter(Boolean);
+ // The chip is the precision the artifact is STORED at, which is what tells two rows apart; what it
+ // RUNS at is the loader's answer, reported by `resolved` after the load.
+ const tags = [format.trim()].filter(Boolean);
const kept: string[] = [];
for (const part of rest) {
if (RESOLUTION_RE.test(part.trim())) tags.push(part.trim());
@@ -862,17 +980,19 @@ export function curatedRowLabelFor(
export function catalogToModelOptions(
catalog: CatalogGroup[],
host: HostClass = "unknown",
+ denseQuantSchemes: readonly string[] = [],
): ModelOption[] {
const options: ModelOption[] = [];
for (const group of catalog) {
for (const artifact of group.artifacts) {
- // A host that can only run the native engine is not offered the pipeline rows it would be refused at load.
- // This is the one place the `models` prop is built, so filtering here covers the trigger name and the picker's
- // seed ids together.
+ // A host that can only run the native engine is not offered pipeline rows it would be refused at
+ // load. The `models` prop is built only here, so this covers the trigger name and seed ids both.
if (!curatedArtifactIsOfferable(artifact.repoId, host)) continue;
options.push({
id: artifact.repoId,
- name: curatedDisplayNameFor(artifact.repoId, catalog, host) ?? group.displayName,
+ name:
+ curatedDisplayNameFor(artifact.repoId, catalog, host, denseQuantSchemes) ??
+ group.displayName,
description: `${group.description} - ${artifact.label}`,
isGguf: artifact.format === "gguf",
deviceQuant: artifact.deviceQuant,
@@ -943,11 +1063,13 @@ export interface DeviceBudget {
budgetFraction?: number;
/** GPUs gpuGb sums, for the loader's per-card VRAM reserve. Absent means one. */
gpuCount?: number;
+ /** Dense quant schemes this host runs, best first; empty keeps the bf16 sizing rule. */
+ denseQuantSchemes?: readonly string[];
}
-/** GGUF fit, delegated to the one formula the Hub badge already uses. This used to carry its own rule (0.7 * GPU
- * + 0.7 * RAM, raw file size) whose comment claimed to match `_select_gpus`; it did not, so chat hid quants the
- * loader would have taken and ignored the VRAM Budget. */
+/** GGUF fit, delegated to the one formula the Hub badge already uses. Its old private rule
+ * (0.7 * GPU + 0.7 * RAM) did not match `_select_gpus` as claimed, so chat hid quants the loader
+ * would have taken and ignored the VRAM Budget. */
export function classifyGgufFit(
sizeBytes: number,
budget: DeviceBudget,
@@ -959,11 +1081,10 @@ export function classifyGgufFit(
return classifyGgufFitForDevice(sizeBytes, budget);
}
-/** Fit rule for a GGUF the IMAGES / VIDEO / AUDIO pickers offer, the one case the shared llama.cpp formula must
- * not judge: those loads go through the diffusion backend, whose budget is free memory minus a reserve at a 0.85
- * margin (`diffusion_memory.py`) and which cannot offload. On a 64 GiB Mac that planner allows about 43.5 GiB
- * where llama.cpp allows 62.1; this rule allows 44.8. It is not the diffusion planner either, only the closer of
- * the two. */
+/** Fit rule for a GGUF the IMAGES / VIDEO / AUDIO pickers offer, the one case the shared llama.cpp
+ * formula must not judge: those loads go through the diffusion backend, which cannot offload and
+ * budgets free memory minus a reserve at a 0.85 margin (`diffusion_memory.py`). On a 64 GiB Mac
+ * that planner allows ~43.5 GiB, llama.cpp 62.1 and this rule 44.8: not the planner, just closer. */
export function classifyMediaGgufFit(
sizeBytes: number,
gpuGb: number,
@@ -1035,14 +1156,55 @@ const FORMAT_QUALITY: Record<ArtifactFormat, number> = {
gguf: 3,
};
-function fitsArtifactBudget(artifact: ModelArtifact, budget: DeviceBudget): boolean {
+const BF16_BYTES_PER_PARAM = 2;
+
+/** What a row is sized by here: the pre-quantised transformer plus the companions, which are
+ * `approxSizeGb` minus the bf16 transformer. Clamped to the dense figure.
+ *
+ * The reported schemes are a LADDER: `_pipeline_planned_denoiser_scheme` walks every rung below
+ * its winner and seeds the first whose hosted artifact stays resident, so the first rung fitting
+ * `allowanceGb` is what the load uses. Sizing by `schemes[0]` alone would refuse rows the backend
+ * would run (Qwen-Image hosts int8 at 25.4 GB against fp8's 19.06). With no allowance, or when no
+ * rung fits, the first hosted rung. */
+function residentSizeGb(
+ group: CatalogGroup,
+ artifact: ModelArtifact,
+ budget: DeviceBudget,
+ allowanceGb?: number,
+): number | undefined {
+ const dense = artifact.approxSizeGb;
+ const schemes = normalizeDenseQuantSchemes(budget.denseQuantSchemes);
+ if (schemes.length === 0 || dense === undefined) return dense;
+ if (!artifactTakesDenseQuant(group, artifact)) return dense;
+ const denseTransformerGb = artifact.totalParams
+ ? (artifact.totalParams * BF16_BYTES_PER_PARAM) / BYTES_PER_GB
+ : 0;
+ const companionsGb = Math.max(0, dense - denseTransformerGb);
+ let firstHosted: number | undefined;
+ for (const scheme of schemes) {
+ const quantised = artifact.prequantSizeGb?.[scheme as DenseQuantScheme];
+ if (quantised === undefined) continue;
+ const sizeGb = Math.min(dense, quantised + companionsGb);
+ if (firstHosted === undefined) firstHosted = sizeGb;
+ if (allowanceGb === undefined || sizeGb <= allowanceGb) return sizeGb;
+ }
+ return firstHosted ?? dense;
+}
+
+function fitsArtifactBudget(
+ group: CatalogGroup,
+ artifact: ModelArtifact,
+ budget: DeviceBudget,
+): boolean {
if (artifact.offloadFitTiers?.length) {
return artifact.offloadFitTiers.some(
(tier) => budget.gpuGb >= tier.gpuGb && budget.systemRamGb >= tier.systemRamGb,
);
}
- if (artifact.approxSizeGb === undefined) return false;
- return artifact.approxSizeGb <= budget.gpuGb * 0.7;
+ const allowanceGb = budget.gpuGb * 0.7;
+ const sizeGb = residentSizeGb(group, artifact, budget, allowanceGb);
+ if (sizeGb === undefined) return false;
+ return sizeGb <= allowanceGb;
}
/** The artifact a bare group click loads. Sized artifacts normally use the 0.7 * GPU budget;
@@ -1058,7 +1220,7 @@ export function pickDefaultArtifact(
const downloaded = artifacts.filter((a) => input.isDownloaded(a.repoId));
if (downloaded.length > 0) {
const fitting = downloaded.find(
- (a) => a.format !== "gguf" && fitsArtifactBudget(a, input),
+ (a) => a.format !== "gguf" && fitsArtifactBudget(group, a, input),
);
if (fitting) return fitting;
const downloadedGguf = downloaded.find((a) => a.format === "gguf");
@@ -1073,7 +1235,11 @@ export function pickDefaultArtifact(
for (const artifact of artifacts) {
// Skip a gated, NOT-downloaded artifact: auto-routing there fails the download without
// license/token access. The downloaded branch above still returns gated artifacts.
- if (artifact.format !== "gguf" && !artifact.gated && fitsArtifactBudget(artifact, input)) {
+ if (
+ artifact.format !== "gguf" &&
+ !artifact.gated &&
+ fitsArtifactBudget(group, artifact, input)
+ ) {
return artifact;
}
}
@@ -1083,11 +1249,11 @@ export function pickDefaultArtifact(
)[0];
}
-/** Whether ONE curated artifact loads on this device, by the rule `pickDefaultArtifact` routes with, since a row
- * click loads that exact artifact. System RAM is not part of a discrete-GPU budget: a pipeline goes wholly on the
- * card unless the catalog states a measured offload tier or the loader falls back to CPU, which only
- * transcription does. A unified-memory host reports RAM and no GPU, and there the RAM is the card. Undefined
- * where nothing can be judged, so the caller shows no verdict rather than a wrong one. */
+/** Whether ONE curated artifact loads on this device, by the rule `pickDefaultArtifact` routes with,
+ * since a row click loads that exact artifact. System RAM is not part of a discrete-GPU budget: a
+ * pipeline goes wholly on the card unless the catalog states a measured offload tier or the loader
+ * falls back to CPU, which only transcription does. A unified-memory host reports RAM and no GPU,
+ * and there the RAM is the card. Undefined where nothing can be judged. */
export function curatedArtifactFitsDevice(
repoId: string,
catalog: CatalogGroup[],
@@ -1097,18 +1263,20 @@ export function curatedArtifactFitsDevice(
if (!hit || hit.artifact.format === "gguf") return undefined;
const { group, artifact } = hit;
if (budget.gpuGb <= 0 && budget.systemRamGb <= 0) return undefined;
- if (artifact.offloadFitTiers?.length) return fitsArtifactBudget(artifact, budget);
- if (artifact.approxSizeGb === undefined) return undefined;
- // Transcription retries a failed device load on CPU (stt_sidecar.py), so RAM is a real budget there, but the
- // WHOLE model goes to whichever device it lands on, so it is the larger of the two and not their sum. An image,
- // video or TTS load rejects CPU offload.
+ if (artifact.offloadFitTiers?.length) return fitsArtifactBudget(group, artifact, budget);
+ // Transcription retries a failed device load on CPU (stt_sidecar.py), so RAM is a real budget
+ // there, but the WHOLE model lands on one device: the larger of the two, not their sum. An
+ // image, video or TTS load rejects CPU offload.
const deviceGb =
group.task === "stt"
? Math.max(budget.gpuGb, budget.systemRamGb)
: budget.gpuGb > 0
? budget.gpuGb
: budget.systemRamGb;
- return artifact.approxSizeGb <= deviceGb * 0.7;
+ const allowanceGb = deviceGb * 0.7;
+ const sizeGb = residentSizeGb(group, artifact, budget, allowanceGb);
+ if (sizeGb === undefined) return undefined;
+ return sizeGb <= allowanceGb;
}
/** Whether the "fit on device" toggle keeps a group, including measured offload tiers when an
@@ -1132,6 +1300,9 @@ export function catalogGroupFitsDevice(
(tier) => budget.gpuGb >= tier.gpuGb && budget.systemRamGb >= tier.systemRamGb,
);
}
- return a.approxSizeGb !== undefined && a.approxSizeGb <= budgetGb;
+ // The same quantised sizing the row badge and pickDefaultArtifact use: the dense figure would
+ // drop a group whose hosted int8 / fp8 form is what the backend would load.
+ const sizeGb = residentSizeGb(group, a, budget, budgetGb);
+ return sizeGb !== undefined && sizeGb <= budgetGb;
});
}
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
index b16d5f4288a..060d9395749 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
@@ -67,6 +67,7 @@ import {
import type { HfTaskFilter } from "@/features/hub/hooks/use-hub-model-search";
import {
useDebouncedValue,
+ useDenseQuantSchemes,
useGpuInfo,
useHostClass,
useInferenceGpuInfo,
@@ -891,8 +892,14 @@ function isRuntimeLoadedModel(
function artifactBudget(gpu: {
memoryTotalGb: number;
systemRamAvailableGb: number;
+ denseQuantSchemes?: readonly string[];
}): DeviceBudget {
- return { gpuGb: gpu.memoryTotalGb, systemRamGb: gpu.systemRamAvailableGb };
+ return {
+ gpuGb: gpu.memoryTotalGb,
+ systemRamGb: gpu.systemRamAvailableGb,
+ // Judges a pre-quantised row by that checkpoint's size, not the bf16 shards it replaces.
+ denseQuantSchemes: gpu.denseQuantSchemes,
+ };
}
const META_COLUMN = {
@@ -3138,6 +3145,7 @@ export function HubModelPicker({
const deviceType = usePlatformStore((s) => s.deviceType);
const isMac = deviceType === "mac";
const hostClass = useHostClass();
+ const denseQuantSchemes = useDenseQuantSchemes();
// Drop models Unsloth cannot run for chat. A task-scoped picker wants exactly the tasks the
// chat classifier calls unsupported, so it gates on the task.
@@ -3270,11 +3278,11 @@ export function HubModelPicker({
// A curated row's name and its chips; ids outside the catalog have neither and show the raw repo id.
const curatedRow = useCallback(
(id: string) =>
- (catalog && curatedRowLabelFor(id, catalog, hostClass)) ?? {
+ (catalog && curatedRowLabelFor(id, catalog, hostClass, denseQuantSchemes)) ?? {
name: id,
tags: [] as string[],
},
- [catalog, hostClass],
+ [catalog, hostClass, denseQuantSchemes],
);
/** Whether this host can run a curated id at all, as opposed to whether it has room for it. Browse rows only. */
diff --git a/studio/frontend/src/features/video/api.ts b/studio/frontend/src/features/video/api.ts
index aa10a6e268e..b68ba7fce24 100644
--- a/studio/frontend/src/features/video/api.ts
+++ b/studio/frontend/src/features/video/api.ts
@@ -20,6 +20,8 @@ export interface VideoResolvedControl {
// "applied" (honored, or nothing was asked) | "fell_back" | "unsupported". Absent on older backends.
status?: "applied" | "fell_back" | "unsupported";
reason: string;
+ // "prequant:<repo>/<file>" when a hosted checkpoint was seeded; absent on a runtime quantise.
+ artifact?: string | null;
}
// Per-family generation defaults + shape constraints, from status.defaults when loaded.
diff --git a/studio/frontend/src/features/video/video-page.tsx b/studio/frontend/src/features/video/video-page.tsx
index a3583c96097..53528f767b6 100644
--- a/studio/frontend/src/features/video/video-page.tsx
+++ b/studio/frontend/src/features/video/video-page.tsx
@@ -87,13 +87,16 @@ import { usePersistedChoice } from "@/hooks/use-persisted-choice";
import { useScrollFades } from "@/hooks/use-scroll-fades";
import { ModelSelector } from "@/features/model-picker/components/model-selector";
import { VIDEO_GEN_TASKS } from "@/features/model-picker/components/model-selector/pickers";
-import type { HostClass } from "@/features/model-picker/components/model-selector/host-artifact-policy";
+import {
+ type HostClass,
+ hostOffersDensePrecision,
+} from "@/features/model-picker/components/model-selector/host-artifact-policy";
import {
VIDEO_CATALOG,
catalogToModelOptions,
loadSpecFor,
} from "@/features/model-picker/components/model-selector/model-catalog";
-import { useHostClass } from "@/hooks/use-host-class";
+import { useDenseQuantSchemes, useHostClass } from "@/hooks/use-host-class";
import type {
ModelOption,
ModelSelectorChangeMeta,
@@ -184,8 +187,14 @@ import { videoThumbnailQueue, withThumbnailRetries } from "./thumbnail-request-q
// Curated models come from the shared catalog, one group per model with a format second level,
// which also surfaces LTX-2.3 in Recommended since its HF pipeline_tag is image-to-video.
// Host-dependent: a Mac gets only GGUF rows. The load kind per artifact comes from loadSpecFor.
-function useVideoModels(host: HostClass): ModelOption[] {
- return useMemo(() => catalogToModelOptions(VIDEO_CATALOG, host), [host]);
+function useVideoModels(
+ host: HostClass,
+ denseQuantSchemes: readonly string[],
+): ModelOption[] {
+ return useMemo(
+ () => catalogToModelOptions(VIDEO_CATALOG, host, denseQuantSchemes),
+ [host, denseQuantSchemes],
+ );
}
// Per-model generation defaults (steps + guidance), matched by repo-id substring, most specific first.
@@ -879,7 +888,8 @@ function VideoGenerator({
}) {
const initialReadySent = useRef(false);
const hostClass = useHostClass();
- const videoModels = useVideoModels(hostClass);
+ const denseQuantSchemes = useDenseQuantSchemes();
+ const videoModels = useVideoModels(hostClass, denseQuantSchemes);
const [quant, setQuant] = useState<string | null>(galleryCache.quant);
const [prompt, setPrompt] = useState(
"Ultra-realistic cinematic documentary footage of a quiet Kyoto neighborhood at sunrise. An elderly Japanese man opens his traditional wooden shop while a young woman wearing a simple kimono walks past carrying a small basket. Cherry blossom petals gently fall through the air, bicycles pass by, warm sunlight enters between narrow streets, distant temple bells echo. The camera slowly moves forward like a professional travel documentary, realistic human movements, natural expressions, authentic Japanese architecture, subtle wind movement in clothing and trees, realistic colors, 35mm film photography style.",
@@ -3269,17 +3279,23 @@ function VideoGenerator({
{!status?.loaded || status.model_kind === "pipeline" ? (
<AdvancedSelect
label="Precision"
- hint="How the model computes. Auto picks the fastest precision the hardware supports (at least INT8 on a capable GPU; FP8 on data-center cards) by quantising the transformer onto low-precision tensor cores, and keeps plain bf16 when the device or memory plan can't take it. Off always runs bf16."
+ hint="How the model computes. Auto picks the fastest precision the hardware supports (INT8 on every capable GPU, then FP8 where the card has it) by quantising the transformer onto low-precision tensor cores, and keeps plain bf16 when the device or memory plan can't take it. Off always runs bf16."
badge={<ResolvedBadge status={status} controlKey="transformer_quant" />}
value={transformerQuant}
onValueChange={(v) => setTransformerQuant(v as typeof transformerQuant)}
options={[
["auto", "Auto (fastest for GPU)"],
["none", "Off (bf16)"],
- ["fp8", "FP8"],
- ["int8", "INT8"],
- ["nvfp4", "NVFP4 (Blackwell)"],
- ["mxfp8", "MXFP8 (Blackwell)"],
+ // The explicit low-precision schemes need the dense tensor-core path, which a Mac or
+ // CPU-only host cannot run, so the picker does not list what the loader would refuse.
+ ...(hostOffersDensePrecision(hostClass)
+ ? ([
+ ["fp8", "FP8"],
+ ["int8", "INT8"],
+ ["nvfp4", "NVFP4 (Blackwell)"],
+ ["mxfp8", "MXFP8 (Blackwell)"],
+ ] as [string, string][])
+ : []),
]}
/>
) : (
diff --git a/studio/frontend/src/hooks/index.ts b/studio/frontend/src/hooks/index.ts
index bf36b6b49f9..1946336d139 100644
--- a/studio/frontend/src/hooks/index.ts
+++ b/studio/frontend/src/hooks/index.ts
@@ -3,7 +3,7 @@
export { useDebouncedValue } from "./use-debounced-value";
export { useGpuInfo, useInferenceGpuInfo } from "./use-gpu-info";
-export { useHostClass } from "./use-host-class";
+export { useDenseQuantSchemes, useHostClass } from "./use-host-class";
export { useGpuUtilization } from "./use-gpu-utilization";
export { useHardwareInfo } from "./use-hardware-info";
export { useHfDatasetSplits } from "./use-hf-dataset-splits";
diff --git a/studio/frontend/src/hooks/use-gpu-info.ts b/studio/frontend/src/hooks/use-gpu-info.ts
index 9889d74118f..af5ad1f618f 100644
--- a/studio/frontend/src/hooks/use-gpu-info.ts
+++ b/studio/frontend/src/hooks/use-gpu-info.ts
@@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { useEffect, useMemo, useState } from "react";
+import { normalizeDenseQuantSchemes } from "@/lib/dense-quant-schemes";
import {
type GpuIndexKind,
type PinnableGpuContext,
@@ -46,6 +47,11 @@ export interface GpuInfo {
* GPU-less one, because "which runtimes can this host place" is exactly the question a host
* with no usable GPU has to answer. Empty until system info arrives. */
backend: string;
+ /** Backend-reported dense quant capability. False until system info arrives. */
+ denseQuantSupported: boolean;
+ /** The dense quant schemes the backend says this host can run, best first ("fp8", "int8"). Empty
+ * until system info arrives, and on a backend too old to report the field. */
+ denseQuantSchemes: readonly string[];
name: string;
memoryTotalGb: number;
memorySharedGb: number;
@@ -81,6 +87,8 @@ const DEFAULT_GPU: GpuInfo = {
sharedMemory: false,
unifiedMemory: false,
backend: "",
+ denseQuantSupported: false,
+ denseQuantSchemes: [],
name: "Unknown",
memoryTotalGb: 0,
memorySharedGb: 0,
@@ -106,6 +114,8 @@ function toGpuInfo(
// path: unified-memory math still needs a RAM budget to work with.
const base = {
backend: data?.device_backend ?? "",
+ denseQuantSupported: data?.dense_quant_supported === true,
+ denseQuantSchemes: normalizeDenseQuantSchemes(data?.dense_quant_schemes),
cpuCore: data?.cpu?.physical_count ?? 0,
cpuThread: data?.cpu?.logical_count ?? 0,
systemRamAvailableGb: data?.memory?.available_gb ?? 0,
@@ -240,6 +250,27 @@ function toGpuDevices(
}));
}
+/**
+ * Carry the previous `denseQuantSchemes` array forward when its contents are unchanged.
+ *
+ * `refresh_memory=true` re-probes `memory.available_gb`, so nearly every poll yields a new snapshot,
+ * but this list is a hardware capability that does not move. Consumers memoise the media picker's
+ * option list on it (`useImageModels`, `curatedRowLabelFor`), so an equal-but-fresh array detached
+ * and re-created every row on every probe.
+ */
+export function withStableSchemes(current: GpuInfo, next: GpuInfo): GpuInfo {
+ const held = current.denseQuantSchemes;
+ const fresh = next.denseQuantSchemes;
+ if (held === fresh) return next;
+ if (
+ held.length === fresh.length &&
+ held.every((scheme, index) => scheme === fresh[index])
+ ) {
+ return { ...next, denseQuantSchemes: held };
+ }
+ return next;
+}
+
/** Aggregate GPU info from /api/system; shares one module-level fetch across all GPU hooks. */
function useGpuInfoSource(source: "gpu" | "inference_gpu"): GpuInfo {
const cachedSystem = getCachedSystemInfo();
@@ -254,7 +285,9 @@ function useGpuInfoSource(source: "gpu" | "inference_gpu"): GpuInfo {
if (cancelled) return;
const next = toGpuInfo(data, source);
setGpu((current) =>
- JSON.stringify(current) === JSON.stringify(next) ? current : next,
+ JSON.stringify(current) === JSON.stringify(next)
+ ? current
+ : withStableSchemes(current, next),
);
};
const update = () => {
diff --git a/studio/frontend/src/hooks/use-host-class.ts b/studio/frontend/src/hooks/use-host-class.ts
index 2f9b3f216a6..cdd05d91297 100644
--- a/studio/frontend/src/hooks/use-host-class.ts
+++ b/studio/frontend/src/hooks/use-host-class.ts
@@ -9,10 +9,7 @@ import {
import { useMemo } from "react";
import { useGpuInfo } from "./use-gpu-info";
-/** What this host can run, for the media pickers.
- *
- * The two inputs live in different places -- the OS in the platform store, the resolved torch
- * backend in system info -- so every caller reads them the same way from here. */
+/** Combine platform and backend state into a media-picker host class. */
export function useHostClass(): HostClass {
const gpu = useGpuInfo();
const deviceType = usePlatformStore((s) => s.deviceType);
@@ -22,7 +19,13 @@ export function useHostClass(): HostClass {
deviceType,
deviceBackend: gpu.backend,
budgetKnown: gpu.budgetKnown,
+ denseQuantSupported: gpu.denseQuantSupported,
}),
- [deviceType, gpu.backend, gpu.budgetKnown],
+ [deviceType, gpu.backend, gpu.budgetKnown, gpu.denseQuantSupported],
);
}
+
+/** The dense quant schemes this host can run, best first. */
+export function useDenseQuantSchemes(): readonly string[] {
+ return useGpuInfo().denseQuantSchemes;
+}
diff --git a/studio/frontend/src/hooks/use-system.ts b/studio/frontend/src/hooks/use-system.ts
index e49ba9c6392..a3e0c5e6db2 100644
--- a/studio/frontend/src/hooks/use-system.ts
+++ b/studio/frontend/src/hooks/use-system.ts
@@ -54,6 +54,11 @@ export interface SystemInfoResponse {
platform: string;
python_version: string;
device_backend: "cuda" | "rocm" | "cpu" | "mlx" | "xpu";
+ /** Backend-reported dense quant capability. Absent on older backends. */
+ dense_quant_supported?: boolean;
+ /** The dense quant schemes this host can run, best first. Absent on older backends, where readers
+ * default to [] and name no precision. */
+ dense_quant_schemes?: string[];
uptime_seconds: number | null;
cpu: {
logical_count: number;
diff --git a/studio/frontend/src/lib/dense-quant-schemes.ts b/studio/frontend/src/lib/dense-quant-schemes.ts
new file mode 100644
index 00000000000..4842f71aaad
--- /dev/null
+++ b/studio/frontend/src/lib/dense-quant-schemes.ts
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// `/api/system.dense_quant_schemes` sits beside the older `dense_quant_supported` boolean, so a
+// reader must survive a backend sending only the boolean.
+
+/** The reported list, lower-cased and emptied of blanks. An absent or malformed field is [], never a
+ * guess: inventing "fp8" would name a precision an Ampere host never runs. */
+export function normalizeDenseQuantSchemes(
+ schemes: readonly unknown[] | undefined | null,
+): readonly string[] {
+ if (!Array.isArray(schemes)) return [];
+ return schemes
+ .filter((scheme): scheme is string => typeof scheme === "string")
+ .map((scheme) => scheme.trim().toLowerCase())
+ .filter((scheme) => scheme.length > 0);
+}
diff --git a/studio/frontend/src/lib/resolved-precision.ts b/studio/frontend/src/lib/resolved-precision.ts
index 5eb02cd6788..1e6342c5911 100644
--- a/studio/frontend/src/lib/resolved-precision.ts
+++ b/studio/frontend/src/lib/resolved-precision.ts
@@ -41,6 +41,16 @@ export interface ResolvedBadgeInfo {
// toast DESCRIPTION under this title instead of as an unreadable single line.
export const PRECISION_REFUSAL_TITLE = "Requested precision is not available";
+/** Load kinds that can reach the dense transformer-quant path. Mirrors the backend constant. */
+export const DENSE_QUANT_KINDS = ["gguf", "pipeline"] as const;
+
+/** Whether this load kind can reach the dense transformer-quant path at all. */
+export function isDenseQuantKind(kind: string | null | undefined): boolean {
+ return (DENSE_QUANT_KINDS as readonly string[]).includes(
+ (kind ?? "").trim().toLowerCase(),
+ );
+}
+
/** Whether a load failure is that refusal, so it can be presented as an actionable choice. */
export function isPrecisionRefusal(message: string): boolean {
return /_quant='[^']*' could not be used/.test(message);
diff --git a/studio/frontend/tests/dense-quant-prequant-routing.test.ts b/studio/frontend/tests/dense-quant-prequant-routing.test.ts
new file mode 100644
index 00000000000..e4660e1019c
--- /dev/null
+++ b/studio/frontend/tests/dense-quant-prequant-routing.test.ts
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// A dense-quant host loads an official image pipeline from the hosted pre-quantised transformer, so
+// the picker must name that precision AND judge fit against the quantised resident size. Both read
+// `dense_quant_schemes`; older backends omit it, and absent is [] with nothing changed.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ AUDIO_CATALOG,
+ IMAGE_CATALOG,
+ VIDEO_CATALOG,
+ artifactForRepoId,
+ catalogToModelOptions,
+ curatedArtifactFitsDevice,
+ curatedDisplayNameFor,
+ curatedRowLabelFor,
+ groupForRepoId,
+ pickDefaultArtifact,
+} from "../src/features/model-picker/components/model-selector/model-catalog.ts";
+
+const Z_TURBO = "Tongyi-MAI/Z-Image-Turbo";
+const QWEN_IMAGE = "Qwen/Qwen-Image";
+const QWEN_2512 = "Qwen/Qwen-Image-2512";
+const H3 = "MiniMaxAI/MiniMax-H3";
+const notDownloaded = () => false;
+
+/** A card with plenty of host RAM: only the GPU figure varies between cases. */
+const onCard = (gpuGb: number, denseQuantSchemes?: readonly string[]) => ({
+ gpuGb,
+ systemRamGb: 128,
+ ...(denseQuantSchemes ? { denseQuantSchemes } : {}),
+});
+
+test("the catalog states the hosted checkpoint each official row would be fetched as", () => {
+ for (const [id, repo, fp8, int8] of [
+ [Z_TURBO, "unsloth/Z-Image-Turbo-FP8", 5.86, 5.86],
+ [QWEN_IMAGE, "unsloth/Qwen-Image-FP8", 19.06, 31.73],
+ [
+ "black-forest-labs/FLUX.1-schnell",
+ "unsloth/FLUX.1-schnell-FP8",
+ 11.09,
+ 14.13,
+ ],
+ ["krea/Krea-2-Turbo", "unsloth/Krea-2-Turbo-FP8", 11.95, 12.19],
+ ] as const) {
+ const hit = artifactForRepoId(id, IMAGE_CATALOG);
+ assert.ok(hit, id);
+ assert.equal(hit.artifact.prequantRepo, repo, id);
+ assert.equal(hit.artifact.prequantSizeGb?.fp8, fp8, id);
+ assert.equal(hit.artifact.prequantSizeGb?.int8, int8, id);
+ }
+});
+
+test("the fit verdict is the quantised resident size on a host that runs a scheme", () => {
+ assert.equal(
+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40)),
+ false,
+ );
+ assert.equal(
+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40, ["fp8"])),
+ true,
+ );
+ assert.equal(
+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40, ["int8"])),
+ true,
+ );
+ assert.equal(
+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24, ["fp8"])),
+ false,
+ );
+});
+
+test("the two schemes are sized apart, since they are different artifacts", () => {
+ assert.equal(
+ curatedArtifactFitsDevice(QWEN_IMAGE, IMAGE_CATALOG, onCard(64, ["fp8"])),
+ true,
+ );
+ assert.equal(
+ curatedArtifactFitsDevice(QWEN_IMAGE, IMAGE_CATALOG, onCard(64, ["int8"])),
+ false,
+ );
+});
+
+test("the 2512 pick is judged by the checkpoint the backend seeds for it, not by bf16", () => {
+ assert.equal(
+ curatedArtifactFitsDevice(QWEN_2512, IMAGE_CATALOG, onCard(64)),
+ false,
+ );
+ assert.equal(
+ curatedArtifactFitsDevice(QWEN_2512, IMAGE_CATALOG, onCard(64, ["int8"])),
+ true,
+ );
+ const group = groupForRepoId(QWEN_2512, IMAGE_CATALOG);
+ assert.ok(group);
+ assert.equal(
+ pickDefaultArtifact(group, {
+ ...onCard(64, ["int8"]),
+ isDownloaded: notDownloaded,
+ }).repoId,
+ QWEN_2512,
+ );
+ assert.equal(
+ pickDefaultArtifact(group, { ...onCard(64), isDownloaded: notDownloaded })
+ .format,
+ "bnb-4bit",
+ );
+});
+
+test("a host with no scheme, and a row with no hosted checkpoint, are unchanged", () => {
+ assert.equal(
+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40, [])),
+ false,
+ );
+ for (const id of ["black-forest-labs/FLUX.1-dev", "stabilityai/sdxl-turbo"]) {
+ assert.equal(
+ curatedArtifactFitsDevice(id, IMAGE_CATALOG, onCard(40, ["fp8"])),
+ curatedArtifactFitsDevice(id, IMAGE_CATALOG, onCard(40)),
+ id,
+ );
+ }
+});
+
+test("the router sends a card that only fits the quantised form to the official row", () => {
+ const group = groupForRepoId(Z_TURBO, IMAGE_CATALOG);
+ assert.ok(group);
+ assert.equal(
+ pickDefaultArtifact(group, { ...onCard(40), isDownloaded: notDownloaded })
+ .format,
+ "bnb-4bit",
+ );
+ assert.equal(
+ pickDefaultArtifact(group, {
+ ...onCard(40, ["fp8"]),
+ isDownloaded: notDownloaded,
+ }).repoId,
+ Z_TURBO,
+ );
+ assert.equal(
+ pickDefaultArtifact(group, {
+ ...onCard(24, ["fp8"]),
+ isDownloaded: notDownloaded,
+ }).format,
+ "bnb-4bit",
+ );
+});
+
+test("the row names the scheme the host runs", () => {
+ assert.equal(
+ curatedRowLabelFor(Z_TURBO, IMAGE_CATALOG, "dense-quant", ["fp8"])?.name,
+ "Z-Image-Turbo (Fast FP8)",
+ );
+ assert.equal(
+ curatedRowLabelFor(Z_TURBO, IMAGE_CATALOG, "dense-quant", ["int8"])?.name,
+ "Z-Image-Turbo (Fast FP8)",
+ );
+ assert.equal(
+ curatedRowLabelFor(Z_TURBO, IMAGE_CATALOG, "dense-quant", [])?.name,
+ "Z-Image-Turbo (Fast)",
+ );
+ assert.equal(
+ curatedRowLabelFor(Z_TURBO, IMAGE_CATALOG, "dense-quant")?.name,
+ "Z-Image-Turbo (Fast)",
+ );
+ assert.equal(
+ curatedDisplayNameFor(Z_TURBO, IMAGE_CATALOG, "dense-quant", ["fp8"]),
+ curatedRowLabelFor(Z_TURBO, IMAGE_CATALOG, "dense-quant", ["fp8"])?.name,
+ );
+ assert.equal(
+ catalogToModelOptions(IMAGE_CATALOG, "dense-quant", ["int8"]).find(
+ (option) => option.id === Z_TURBO,
+ )?.name,
+ "Z-Image-Turbo (Fast FP8)",
+ );
+});
+
+test("the H3 pipeline row names its precision again", () => {
+ assert.deepEqual(
+ curatedRowLabelFor(H3, VIDEO_CATALOG, "dense-quant", ["fp8"]),
+ {
+ name: "MiniMax H3 (Fast FP8)",
+ tags: ["BF16"],
+ },
+ );
+ assert.deepEqual(
+ curatedRowLabelFor(H3, VIDEO_CATALOG, "dense-quant", ["int8"]),
+ {
+ name: "MiniMax H3 (Fast FP8)",
+ tags: ["BF16"],
+ },
+ );
+ assert.equal(
+ curatedRowLabelFor(
+ "unsloth/MiniMax-H3-GGUF",
+ VIDEO_CATALOG,
+ "dense-quant",
+ ["fp8"],
+ )?.name,
+ "MiniMax-H3-GGUF (Slow)",
+ );
+});
+
+test("every diffusion GGUF row is tagged Slow, not only H3's", () => {
+ for (const [repoId, catalog, name] of [
+ ["unsloth/Z-Image-Turbo-GGUF", IMAGE_CATALOG, "Z-Image-Turbo-GGUF (Slow)"],
+ ["unsloth/FLUX.1-schnell-GGUF", IMAGE_CATALOG, "FLUX.1-schnell-GGUF (Slow)"],
+ ["unsloth/LTX-2.3-GGUF", VIDEO_CATALOG, "LTX-2.3-GGUF (Slow)"],
+ ] as const) {
+ assert.equal(
+ curatedRowLabelFor(repoId, catalog, "dense-quant", ["fp8"])?.name,
+ name,
+ repoId,
+ );
+ assert.equal(
+ curatedRowLabelFor(repoId, catalog, "accelerated")?.name,
+ name,
+ repoId,
+ );
+ assert.equal(
+ curatedDisplayNameFor(repoId, catalog, "dense-quant", ["fp8"]),
+ name,
+ repoId,
+ );
+ }
+});
+
+test("a host with no accelerator is not told which row is slow", () => {
+ // Off an accelerator the GGUF is the only row that runs, so the qualifier would read as a
+ // warning about the user's one option rather than a comparison.
+ for (const host of ["gguf-only", "unknown"] as const) {
+ for (const [repoId, catalog] of [
+ ["unsloth/Z-Image-Turbo-GGUF", IMAGE_CATALOG],
+ ["unsloth/LTX-2.3-GGUF", VIDEO_CATALOG],
+ ["unsloth/MiniMax-H3-GGUF", VIDEO_CATALOG],
+ ] as const) {
+ const name = curatedRowLabelFor(repoId, catalog, host, ["fp8"])?.name;
+ assert.equal(name?.includes("Slow"), false, `${repoId} ${host}`);
+ }
+ }
+});
+
+test("an audio GGUF keeps its plain name, since it has no dense sibling", () => {
+ for (const repoId of [
+ "unsloth/orpheus-3b-0.1-ft-GGUF",
+ "unslothai/Qwen3-ASR-0.6B-GGUF",
+ ]) {
+ for (const group of AUDIO_CATALOG) {
+ if (!group.artifacts.some((a) => a.repoId === repoId)) continue;
+ const name = curatedRowLabelFor(repoId, AUDIO_CATALOG, "dense-quant", [
+ "fp8",
+ ])?.name;
+ assert.equal(name?.includes("Slow"), false, repoId);
+ }
+ }
+});
+
+test("the scheme reaches the name and never the chip", () => {
+ for (const schemes of [[], ["fp8"], ["int8"]]) {
+ for (const catalog of [IMAGE_CATALOG, VIDEO_CATALOG]) {
+ for (const group of catalog) {
+ for (const artifact of group.artifacts) {
+ assert.deepEqual(
+ curatedRowLabelFor(artifact.repoId, catalog, "dense-quant", schemes)
+ ?.tags ?? [],
+ curatedRowLabelFor(artifact.repoId, catalog, "accelerated")?.tags ??
+ [],
+ `${artifact.repoId} ${schemes.join(",") || "none"}`,
+ );
+ }
+ }
+ }
+ }
+});
diff --git a/studio/frontend/tests/dense-quant-schemes-identity.test.ts b/studio/frontend/tests/dense-quant-schemes-identity.test.ts
new file mode 100644
index 00000000000..0678de71a7e
--- /dev/null
+++ b/studio/frontend/tests/dense-quant-schemes-identity.test.ts
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// The media picker memoises its option list on `denseQuantSchemes` (useImageModels,
+// curatedRowLabelFor), and the GGUF rows carry a host-dependent "(Slow)" suffix. So a snapshot
+// differing only in free host RAM (re-probed by `refresh_memory=true`) must NOT hand back a fresh
+// array, or every probe rebuilds the list and detaches every row it had just drawn.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import type * as GpuHooks from "../src/hooks/use-gpu-info.ts";
+import { normalizeDenseQuantSchemes } from "../src/lib/dense-quant-schemes.ts";
+import { loadWithStubs } from "./helpers/module-stubs.ts";
+
+const hooks = loadWithStubs<typeof GpuHooks>(
+ new URL("../src/hooks/use-gpu-info.ts", import.meta.url),
+ {
+ react: {
+ useState: (initial: unknown) => [
+ typeof initial === "function" ? initial() : initial,
+ () => {},
+ ],
+ useEffect: () => {},
+ useMemo: (read: () => unknown) => read(),
+ },
+ "./use-system": { getCachedSystemInfo: () => null },
+ "@/lib/dense-quant-schemes": { normalizeDenseQuantSchemes },
+ },
+ { relativePassthrough: true },
+);
+
+/** A GpuInfo carrying the given schemes and free-RAM reading; every other field is fixed. */
+function snapshot(
+ schemes: readonly string[],
+ systemRamAvailableGb: number,
+): GpuHooks.GpuInfo {
+ return {
+ available: true,
+ budgetKnown: true,
+ sharedMemory: false,
+ unifiedMemory: false,
+ backend: "cuda",
+ denseQuantSupported: schemes.length > 0,
+ denseQuantSchemes: schemes,
+ name: "NVIDIA GeForce RTX 4090",
+ memoryTotalGb: 24,
+ memorySharedGb: 0,
+ dedicatedMemoryTotalGb: 24,
+ maxDeviceMemoryGb: 24,
+ loadDeviceMemoryGb: 24,
+ loadDeviceSharedMemory: false,
+ loadDeviceSharesHostMemory: false,
+ deviceCount: 1,
+ cpuCore: 8,
+ cpuThread: 16,
+ systemRamAvailableGb,
+ systemRamAvailableHostGb: systemRamAvailableGb,
+ systemRamAvailableKnown: true,
+ systemRamTotalGb: 64,
+ };
+}
+
+test("the premise: each probe normalises a fresh schemes array", () => {
+ const reported = ["fp8", "int8"];
+ const first = normalizeDenseQuantSchemes(reported);
+ const second = normalizeDenseQuantSchemes(reported);
+ assert.deepEqual(first, second);
+ assert.notEqual(first, second, "a fresh array per probe is what defeats the memo");
+});
+
+test("a probe that only moved free RAM keeps the schemes array identity", () => {
+ const current = snapshot(normalizeDenseQuantSchemes(["fp8", "int8"]), 31.2);
+ const next = snapshot(normalizeDenseQuantSchemes(["fp8", "int8"]), 28.7);
+ assert.notEqual(
+ current.denseQuantSchemes,
+ next.denseQuantSchemes,
+ "the probe really did build a new array",
+ );
+
+ const held = hooks.withStableSchemes(current, next);
+
+ assert.equal(
+ held.denseQuantSchemes,
+ current.denseQuantSchemes,
+ "the option list memo must not be invalidated by a RAM-only probe",
+ );
+});
+
+test("the RAM reading the probe refreshed is still published", () => {
+ const current = snapshot(normalizeDenseQuantSchemes(["fp8"]), 31.2);
+ const next = snapshot(normalizeDenseQuantSchemes(["fp8"]), 28.7);
+
+ const held = hooks.withStableSchemes(current, next);
+
+ // Holding the array must not hold the snapshot: the footprint panel reads these.
+ assert.equal(held.systemRamAvailableGb, 28.7);
+ assert.equal(held.systemRamAvailableHostGb, 28.7);
+});
+
+test("a host that really changed its schemes gets the new array", () => {
+ const current = snapshot(normalizeDenseQuantSchemes(["fp8", "int8"]), 31.2);
+ const next = snapshot(normalizeDenseQuantSchemes(["int8"]), 31.2);
+
+ const held = hooks.withStableSchemes(current, next);
+
+ assert.equal(held.denseQuantSchemes, next.denseQuantSchemes);
+ assert.deepEqual(held.denseQuantSchemes, ["int8"]);
+});
+
+test("order is part of the identity: schemes are ranked best first", () => {
+ const current = snapshot(normalizeDenseQuantSchemes(["fp8", "int8"]), 31.2);
+ const next = snapshot(normalizeDenseQuantSchemes(["int8", "fp8"]), 31.2);
+
+ const held = hooks.withStableSchemes(current, next);
+
+ assert.equal(
+ held.denseQuantSchemes,
+ next.denseQuantSchemes,
+ "a reordered ladder names a different best scheme, so it is a real change",
+ );
+});
+
+test("the first snapshot to arrive is published as-is", () => {
+ const cold = snapshot([], 0);
+ const warm = snapshot(normalizeDenseQuantSchemes(["fp8"]), 31.2);
+
+ const held = hooks.withStableSchemes(cold, warm);
+
+ assert.equal(held.denseQuantSchemes, warm.denseQuantSchemes);
+ assert.deepEqual(held.denseQuantSchemes, ["fp8"]);
+});
diff --git a/studio/frontend/tests/host-artifact-policy.test.ts b/studio/frontend/tests/host-artifact-policy.test.ts
index 702d7a39b9c..457951cdffd 100644
--- a/studio/frontend/tests/host-artifact-policy.test.ts
+++ b/studio/frontend/tests/host-artifact-policy.test.ts
@@ -7,19 +7,61 @@ import test from "node:test";
import {
classifyHost,
curatedArtifactIsOfferable,
+ densePerfSuffix,
+ ggufPerfSuffix,
h3PerfSuffix,
+ hostIsAccelerated,
+ hostOffersDensePrecision,
+ hostRunsDenseQuant,
} from "../src/features/model-picker/components/model-selector/host-artifact-policy.ts";
+import { normalizeDenseQuantSchemes } from "../src/lib/dense-quant-schemes.ts";
test("the backends that can place a diffusion pipeline are accelerated", () => {
for (const deviceBackend of ["cuda", "rocm", "xpu"]) {
assert.equal(
- classifyHost({ deviceBackend, budgetKnown: true }),
- "accelerated",
+ hostIsAccelerated(classifyHost({ deviceBackend, budgetKnown: true })),
+ true,
deviceBackend,
);
}
});
+// Dense quant follows backend capability rather than the backend name.
+test("the dense-quant class follows the backend's capability answer, not its name", () => {
+ assert.equal(
+ classifyHost({ deviceBackend: "cuda", budgetKnown: true, denseQuantSupported: true }),
+ "dense-quant",
+ );
+ assert.equal(hostRunsDenseQuant("dense-quant"), true);
+ const preAmpere = classifyHost({
+ deviceBackend: "cuda",
+ budgetKnown: true,
+ denseQuantSupported: false,
+ });
+ assert.equal(preAmpere, "accelerated");
+ assert.equal(hostIsAccelerated(preAmpere), true);
+ assert.equal(hostRunsDenseQuant(preAmpere), false);
+ // Missing capability is conservative for older or unresolved backends.
+ assert.equal(classifyHost({ deviceBackend: "cuda", budgetKnown: true }), "accelerated");
+ for (const deviceBackend of ["rocm", "xpu"]) {
+ const host = classifyHost({ deviceBackend, budgetKnown: true });
+ assert.equal(host, "accelerated", deviceBackend);
+ assert.equal(hostRunsDenseQuant(host), false, deviceBackend);
+ }
+ for (const host of ["gguf-only", "unknown"] as const) {
+ assert.equal(hostRunsDenseQuant(host), false, host);
+ }
+ assert.equal(
+ classifyHost({
+ deviceType: "mac",
+ deviceBackend: "cuda",
+ budgetKnown: true,
+ denseQuantSupported: true,
+ }),
+ "gguf-only",
+ );
+});
+
test("the backends that only run the native engine are gguf-only", () => {
for (const deviceBackend of ["mlx", "cpu"]) {
assert.equal(
@@ -98,10 +140,45 @@ test("a gguf-only host keeps every non-GGUF row the backend can still load", ()
});
test("the speed suffixes name the two H3 rows on an accelerated host", () => {
- assert.equal(h3PerfSuffix("MiniMaxAI/MiniMax-H3", "accelerated"), "Fast FP8");
+ assert.equal(h3PerfSuffix("MiniMaxAI/MiniMax-H3", "accelerated"), "Fast");
assert.equal(h3PerfSuffix("unsloth/MiniMax-H3-GGUF", "accelerated"), "Slow");
});
+test("every dense 8-bit scheme shares the one FP8 label", () => {
+ assert.equal(densePerfSuffix(["fp8"]), "Fast FP8");
+ assert.equal(densePerfSuffix(["int8"]), "Fast FP8");
+ assert.equal(densePerfSuffix(["mxfp8"]), "Fast FP8");
+ // Reordering the ladder must not rename the row: the label is the tier, and the resolved record
+ // names the scheme that actually loaded.
+ assert.equal(densePerfSuffix(["fp8", "int8"]), "Fast FP8");
+ assert.equal(densePerfSuffix(["int8", "fp8"]), "Fast FP8");
+ // A 4-bit scheme is a different quality tier and keeps its own name.
+ assert.equal(densePerfSuffix(["nvfp4"]), "Fast NVFP4");
+});
+
+test("a host that names no scheme keeps the bare qualifier", () => {
+ assert.equal(densePerfSuffix([]), "Fast");
+ assert.equal(densePerfSuffix(undefined), "Fast");
+ assert.equal(densePerfSuffix(null), "Fast");
+ assert.equal(densePerfSuffix(["", " "]), "Fast");
+ assert.equal(densePerfSuffix([" fp8 "]), "Fast FP8");
+});
+
+test("the H3 pipeline row names its precision from the same scheme list", () => {
+ assert.equal(h3PerfSuffix("MiniMaxAI/MiniMax-H3", "dense-quant", ["fp8"]), "Fast FP8");
+ assert.equal(h3PerfSuffix("MiniMaxAI/MiniMax-H3", "dense-quant", ["int8"]), "Fast FP8");
+ assert.equal(h3PerfSuffix("MiniMaxAI/MiniMax-H3", "accelerated", ["fp8"]), "Fast FP8");
+ for (const schemes of [["fp8"], ["int8"], []]) {
+ assert.equal(
+ h3PerfSuffix("unsloth/MiniMax-H3-GGUF", "dense-quant", schemes),
+ "Slow",
+ schemes.join(",") || "none",
+ );
+ }
+ assert.equal(h3PerfSuffix("MiniMaxAI/MiniMax-H3", "dense-quant"), "Fast");
+ assert.equal(h3PerfSuffix("MiniMaxAI/MiniMax-H3", "dense-quant", []), "Fast");
+});
+
test("no other model claims a speed it was never measured at", () => {
for (const id of [
"Lightricks/LTX-2.3",
@@ -112,9 +189,34 @@ test("no other model claims a speed it was never measured at", () => {
}
});
+test("an absent or malformed scheme list reads as no schemes, never as a guess", () => {
+ assert.deepEqual(normalizeDenseQuantSchemes(undefined), []);
+ assert.deepEqual(normalizeDenseQuantSchemes(null), []);
+ assert.deepEqual(normalizeDenseQuantSchemes("fp8" as unknown as string[]), []);
+ assert.deepEqual(normalizeDenseQuantSchemes([]), []);
+ assert.deepEqual(normalizeDenseQuantSchemes([" FP8 ", "INT8"]), ["fp8", "int8"]);
+ assert.deepEqual(normalizeDenseQuantSchemes(["int8", "fp8"]), ["int8", "fp8"]);
+ assert.deepEqual(normalizeDenseQuantSchemes(["", " ", 8, null, "fp8"]), ["fp8"]);
+});
+
test("a gguf-only or undiscovered host gets no suffix at all", () => {
for (const host of ["gguf-only", "unknown"] as const) {
assert.equal(h3PerfSuffix("MiniMaxAI/MiniMax-H3", host), null, host);
assert.equal(h3PerfSuffix("unsloth/MiniMax-H3-GGUF", host), null, host);
+ assert.equal(h3PerfSuffix("MiniMaxAI/MiniMax-H3", host, ["fp8"]), null, host);
+ assert.equal(ggufPerfSuffix(host), null, host);
}
});
+
+test("a GGUF row is the slow one wherever a dense row can run beside it", () => {
+ assert.equal(ggufPerfSuffix("accelerated"), "Slow");
+ assert.equal(ggufPerfSuffix("dense-quant"), "Slow");
+});
+
+test("the dense precisions are offered exactly where they can run", () => {
+ assert.equal(hostOffersDensePrecision("dense-quant"), true);
+ assert.equal(hostOffersDensePrecision("accelerated"), true);
+ // An older backend reports nothing; keep the controls rather than remove ones it can honour.
+ assert.equal(hostOffersDensePrecision("unknown"), true);
+ assert.equal(hostOffersDensePrecision("gguf-only"), false);
+});
diff --git a/studio/frontend/tests/images-precision-gate-native.test.ts b/studio/frontend/tests/images-precision-gate-native.test.ts
index 7192d69a25f..c210c01fd29 100644
--- a/studio/frontend/tests/images-precision-gate-native.test.ts
+++ b/studio/frontend/tests/images-precision-gate-native.test.ts
@@ -5,7 +5,7 @@ import assert from "node:assert/strict";
import test from "node:test";
import ts from "typescript";
-import { isNativeEngineStatus } from "../src/lib/resolved-precision.ts";
+import { isDenseQuantKind, isNativeEngineStatus } from "../src/lib/resolved-precision.ts";
import { readSrc } from "./helpers/kit.ts";
/** The Images page's Precision gate, evaluated the way the component evaluates it. */
@@ -17,7 +17,7 @@ function precisionControlShown(status: {
} | null): boolean {
return (
!status?.loaded
- || (status.model_kind === "gguf" && !isNativeEngineStatus(status))
+ || (isDenseQuantKind(status.model_kind) && !isNativeEngineStatus(status))
);
}
@@ -43,10 +43,12 @@ test("the Precision control is offered for a diffusers GGUF and withheld from na
true,
"a diffusers GGUF load is exactly what transformer_quant is for",
);
+ // An official pipeline pick now resolves to the hosted FP8 or INT8 checkpoint, so it takes a
+ // precision request too; it is quantised in place rather than swapped for a GGUF.
assert.equal(
precisionControlShown({ ...DIFFUSERS_GGUF, model_kind: "pipeline" }),
- false,
- "a full pipeline has no GGUF to quantise",
+ true,
+ "a pipeline pick is quantised in place, so Precision applies to it",
);
// The regression this guards: sd.cpp reports transformer_quant null and no `resolved`
// map, so an offered control has no badge and snaps back on the next load.
@@ -68,9 +70,10 @@ test("the Images page gates Precision on the engine, not on model_kind alone", (
ts.ScriptKind.TSX,
);
void tree;
- const gate = source.includes(
- 'status.model_kind === "gguf" && !isNativeEngineStatus(status)',
- );
+ // Pinned on the native-engine exclusion, not on the kind test beside it: the kind test widened
+ // from a bare GGUF check to sendsTransformerQuant when pipeline picks became quantisable, and
+ // this guard is about the ENGINE.
+ const gate = source.includes("&& !isNativeEngineStatus(status)");
assert.ok(
gate,
"the Precision gate must exclude the native engine; a bare "
diff --git a/studio/frontend/tests/memory-probe-known.test.ts b/studio/frontend/tests/memory-probe-known.test.ts
index 8c1526c22a1..ea1a8730f26 100644
--- a/studio/frontend/tests/memory-probe-known.test.ts
+++ b/studio/frontend/tests/memory-probe-known.test.ts
@@ -5,6 +5,7 @@ import assert from "node:assert/strict";
import test from "node:test";
import type * as GpuHooks from "../src/hooks/use-gpu-info.ts";
import type * as SystemHooks from "../src/hooks/use-system.ts";
+import { normalizeDenseQuantSchemes } from "../src/lib/dense-quant-schemes.ts";
import { loadWithStubs } from "./helpers/module-stubs.ts";
let snapshot: unknown = null;
@@ -20,6 +21,9 @@ const hooks = loadWithStubs<typeof GpuHooks>(
useMemo: (read: () => unknown) => read(),
},
"./use-system": { getCachedSystemInfo: () => snapshot },
+ // Aliased, so the passthrough above cannot resolve it. The real implementation, since a
+ // hand-written normaliser would answer for the source rather than from it.
+ "@/lib/dense-quant-schemes": { normalizeDenseQuantSchemes },
},
{ relativePassthrough: true },
);
@@ -53,6 +57,30 @@ test("GPU and RAM probes distinguish zero from missing or invalid readings", ()
}
});
+test("the dense quant schemes are read off the route, and default to none", () => {
+ const withSchemes = (schemes?: unknown) => {
+ snapshot = {
+ status: "ready",
+ device_backend: "cuda",
+ dense_quant_supported: true,
+ ...(schemes === undefined ? {} : { dense_quant_schemes: schemes }),
+ gpu: {
+ available: true,
+ backend: "cuda",
+ devices: [{ index: 0, index_kind: "physical", memory_total_gb: 24 }],
+ },
+ memory: { total_gb: 32, available_gb: 16 },
+ };
+ return hooks.useGpuInfo();
+ };
+ assert.deepEqual(withSchemes(["fp8"]).denseQuantSchemes, ["fp8"]);
+ assert.deepEqual(withSchemes(["INT8"]).denseQuantSchemes, ["int8"]);
+ const older = withSchemes();
+ assert.equal(older.denseQuantSupported, true);
+ assert.deepEqual(older.denseQuantSchemes, []);
+ assert.deepEqual(withSchemes([]).denseQuantSchemes, []);
+});
+
test("resident refresh bypasses cached and in-flight pre-load memory", async () => {
const requests: { url: string; resolve: (response: Response) => void }[] = [];
const system = loadWithStubs<typeof SystemHooks>(
diff --git a/studio/frontend/tests/resolved-precision.test.ts b/studio/frontend/tests/resolved-precision.test.ts
index 4d4e7f702c9..0c1b9aa728d 100644
--- a/studio/frontend/tests/resolved-precision.test.ts
+++ b/studio/frontend/tests/resolved-precision.test.ts
@@ -5,8 +5,10 @@ import assert from "node:assert/strict";
import test from "node:test";
import {
+ DENSE_QUANT_KINDS,
PRECISION_REFUSAL_TITLE,
type ResolvedControl,
+ isDenseQuantKind,
isPrecisionRefusal,
isResolvedHonored,
resolvedBadge,
@@ -314,6 +316,18 @@ test("a precision refusal is recognised so it can be shown as an actionable toas
assert.equal(PRECISION_REFUSAL_TITLE, "Requested precision is not available");
});
+// The supported kinds mirror the backend constant.
+test("the dense-quant kinds are the two the backend quantises", () => {
+ assert.deepEqual([...DENSE_QUANT_KINDS], ["gguf", "pipeline"]);
+ assert.equal(isDenseQuantKind("gguf"), true);
+ assert.equal(isDenseQuantKind("pipeline"), true);
+ assert.equal(isDenseQuantKind("single_file"), false);
+ assert.equal(isDenseQuantKind(" Pipeline "), true);
+ assert.equal(isDenseQuantKind(null), false);
+ assert.equal(isDenseQuantKind(undefined), false);
+ assert.equal(isDenseQuantKind(""), false);
+});
+
const ENCODER_OPTIONS = ["auto", "fp8", "fp8_dynamic", "int8", "nvfp4"] as const;
const toEncoderOption = (v: string) =>
ENCODER_OPTIONS.find((o) => o === v || (o === "auto" && (v === "none" || v === "off"))) ?? null;