Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep int8 / fp8 image transformers quantised when the load has to offload (#12287)
Daniel Hane477badf8125
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 2c1cbfa45d3..3faf0475f48 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -127,6 +127,12 @@ from .diffusion_memory import (refine_memory_plan_for_components,settled_snapshot_device_memory,snapshot_device_memory,+ _torchao_stream_pinnable,+ pipeline_host_mib,+ torchao_offload_plan,+ torchao_scheme_streams,+ torchao_streaming_plan,+ torchao_survives_plan,unified_memory_shortfall_message,vae_tile_side,vae_can_slice,@@ -1626,6 +1632,66 @@ def _activation_guard_batch(chunks: Sequence[Sequence[Any]]) -> int:return max((len(chunk) for chunk in chunks), default = 1)+def _inplace_torchao_placement(+ plan: Any,+ scheme: Optional[str],+ estimate: Any,+ largest_companion_mib: Callable[[], Optional[int]],+) -> tuple[Any, Optional[str]]:+ """(placement, None) for the in-place torchao quant, or (``plan``, why not). Model offload checks LOADED encoders."""+ if plan_keeps_transformer_resident(plan):+ return plan, None+ if plan.offload_policy == OFFLOAD_MODEL:+ estimates = getattr(plan, "estimates", None) or {}+ budget = int(estimates.get("safe_device_budget_mib") or 0)+ overhead = int(estimates.get("runtime_headroom_mib") or 0) + int(+ estimates.get("base_overhead_mib") or 0+ )+ if (+ estimate is not None+ and estimate.steady_transformer_mib + overhead <= budget+ and (largest_companion_mib() or 0) <= budget+ ):+ return plan, None+ if torchao_scheme_streams(scheme) and _torchao_stream_pinnable(plan):+ return torchao_streaming_plan(plan), None+ return plan, (+ "whole-module offload onloads each component whole, and the quantised transformer or a text "+ "encoder is not known to fit the device budget"+ )+ placed = torchao_offload_plan(plan, scheme)+ if placed is not None:+ return placed, None+ return plan, (+ f"'{plan.offload_policy}' offload streams the transformer through hooks torchao weights do not survive "+ f"({scheme} on this torchao). Pin low_vram or a resident memory mode to combine the two"+ )+++def _denoiser_hooked(pipe: Any) -> bool:+ """Whether an offload hook moves a denoiser of ``pipe``; a CUDA graph over it would replay stale pointers."""+ for name in ("transformer", "transformer_2", "unconditional_transformer", "unet"):+ module = getattr(pipe, name, None)+ if module is None:+ continue+ if getattr(module, "_hf_hook", None) is not None:+ return True+ registry = getattr(module, "_diffusers_hook", None)+ hooks = getattr(registry, "hooks", None) or {}+ if any("offload" in str(key) for key in hooks):+ return True+ return False+++def _torchao_render_needs_no_grad(state: Any) -> bool:+ """inference_mode rejects moving torchao weights between devices, so offloaded quant renders use no_grad."""+ if getattr(state, "offload_policy", OFFLOAD_NONE) == OFFLOAD_NONE:+ return False+ return bool(+ getattr(state, "transformer_quant", None) or getattr(state, "text_encoder_quant", None)+ )++def _memory_request_forces_offload(memory_mode: Optional[str], cpu_offload: bool) -> bool:"""Whether this memory request offloads the transformer no matter what the weights measure.``balanced`` and ``low_vram`` name their policy outright in ``resolve_offload_policy``, and@@ -3410,7 +3476,13 @@ class DiffusionBackend:except Exception: # noqa: BLE001 -- no lower rungs is just "no retry"passdeclined = False+ # An offloaded quantised rung beats bf16 shards; a later resident rung beats both.+ offloaded_rung: Optional[str] = Nonememory = snapshot_device_memory(target)+ # Like the VRAM override below: the outgoing pipeline's host weights are gone before this one pins.+ reclaimable_host_mib = pipeline_host_mib(+ getattr(getattr(self, "_state", None), "pipe", None)+ )for rung in rungs:source = denoiser_prequant_source(fam,@@ -3453,16 +3525,24 @@ class DiffusionBackend:),device_memory_override = replace(memory, free_mib = memory.total_mib),)- if not plan_keeps_transformer_resident(planned):+ if not torchao_survives_plan(+ planned, rung, reclaimable_host_mib = reclaimable_host_mib+ ):logger.info(- "diffusion.denoiser_prequant: an artifact-sized plan for %s streams the "- "denoiser on this card, and offload moves the denoiser via Module.to(), "- "so the released shards are kept",+ "diffusion.denoiser_prequant: an artifact-sized plan for %s places the "+ "denoiser with '%s' offload, which its torchao weights do not survive "+ "here, so the released shards are kept",rung,+ planned.offload_policy,)declined = Truecontinue+ if not plan_keeps_transformer_resident(planned):+ offloaded_rung = offloaded_rung or rung+ continuereturn rung+ if offloaded_rung is not None:+ return offloaded_rungreturn PIPELINE_SEED_DECLINED if declined else Noneexcept Exception as exc: # noqa: BLE001 -- an unanswerable probe keeps the released shardslogger.warning("diffusion.denoiser_prequant_plan_failed: %s", exc)@@ -5153,17 +5233,22 @@ class DiffusionBackend:fetch_base = fetch_base,text_encoder_quant = text_encoder_quant,)- if seeded_plan is None or not plan_keeps_transformer_resident(seeded_plan):- # Offload hooks use Module.to(), which torchao tensors reject, and live free- # memory can undercut the CAPACITY the plan settled this against.+ seeded_placement = (+ None+ if seeded_plan is None+ else torchao_offload_plan(seeded_plan, pipeline_seed_scheme)+ )+ if seeded_placement is None:+ # Live free memory can undercut the CAPACITY the plan settled this against.logger.info(- "diffusion.denoiser_prequant: an artifact-sized plan for %s offloads on "- "this card, so the released denoiser is loaded instead",+ "diffusion.denoiser_prequant: an artifact-sized plan for %s places the "+ "denoiser where its torchao weights do not survive on this card, so the "+ "released denoiser is loaded instead",pipeline_seed_scheme,)pipeline_seed_scheme = Noneelse:- plan = seeded_plan+ plan = seeded_placement# On unified memory the plan above is final (the quant re-plans below are CUDA-only) and its 'none'# policy is a placement, not a fit. Refuse here, after the eviction above freed the previous pipeline# and before any weight is materialised. A pipeline's weight term is cached SHARD bytes, which is a@@ -6298,14 +6383,28 @@ class DiffusionBackend:),text_encoder_quant = text_encoder_quant,)- if plan_keeps_transformer_resident(replanned):+ # Native schemes (incl. explicit int8 offloaded) keep the resident-only rule.+ placed = (+ (+ replanned+ if plan_keeps_transformer_resident(replanned)+ else None+ )+ if native_scheme is not None+ or native_offload_scheme is not None+ else torchao_offload_plan(replanned, preview_scheme)+ )+ if placed is not None:+ replanned = placedlogger.info(- "diffusion.transformer_quant: %s fits resident (%d MiB "- "steady, encoders streamed=%s); replacing the bf16 plan's "- "'%s' offload",+ "diffusion.transformer_quant: %s fits the '%s' placement "+ "(%d MiB steady, denoiser streamed=%s, encoders streamed=%s); "+ "replacing the bf16 plan's '%s' offload",preview_scheme,+ replanned.offload_policy,estimate.steady_transformer_mib,- replanned.offload_policy != OFFLOAD_NONE,+ not plan_keeps_transformer_resident(replanned),+ bool(getattr(replanned, "stream_text_encoders", False)),plan.offload_policy,)plan = replanned@@ -6322,34 +6421,16 @@ class DiffusionBackend:native_scheme,plan.offload_policy,)- # Group offload is WRONG for torchao: its stream cache aliases weights and swap_tensors fails when compiled.- quant_budget = int(plan.estimates.get("safe_device_budget_mib") or 0)- quant_overhead = int(- plan.estimates.get("runtime_headroom_mib") or 0- ) + int(plan.estimates.get("base_overhead_mib") or 0)- if (- not keeps_resident- and native_scheme is None- and (- plan.offload_policy != OFFLOAD_MODEL- or estimate is None- or estimate.steady_transformer_mib + quant_overhead- > quant_budget- or (largest_streamable_companion_mib(pipe) or 0) > quant_budget+ inplace_decline = None+ if not keeps_resident and native_scheme is None:+ plan, inplace_decline = _inplace_torchao_placement(+ plan,+ preview_scheme,+ estimate,+ lambda: largest_streamable_companion_mib(pipe),)- ):- if plan.offload_policy == OFFLOAD_MODEL:- transformer_quant_decline = (- "whole-module offload onloads each component whole, and the "- "quantised transformer or a text encoder is not known to fit "- "the device budget"- )- else:- transformer_quant_decline = (- f"'{plan.offload_policy}' offload streams the transformer through "- "hooks torchao weights do not survive. Pin low_vram or a resident "- "memory mode to combine the two"- )+ if inplace_decline is not None:+ transformer_quant_decline = inplace_declinelogger.info("diffusion.transformer_quant: skipped (%s)",transformer_quant_decline,@@ -6636,6 +6717,7 @@ class DiffusionBackend:cache_active = cache_graph_break or cache_may_toggle,cache_engaged = cache_graph_break,offload_active = plan.offload_policy != OFFLOAD_NONE,+ denoiser_offloaded = not plan_keeps_transformer_resident(plan),logger = logger,)if vae_fp16:@@ -6739,6 +6821,11 @@ class DiffusionBackend:placement_device = target.torch_device,logger = logger,)+ if speed_applied.get("cuda_graph") and _denoiser_hooked(pipe):+ cuda_graph.uninstall_all(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())+ pipe._unsloth_cuda_graphs = ()+ pipe._unsloth_cuda_graph_reason = "offload active"+ speed_applied["cuda_graph"] = False# Per-control provenance for status. cpu_offload=False is the unset default, so only True is# explicit.@@ -8336,6 +8423,8 @@ class DiffusionBackend:cache_active = cache_breaks_graph(state.transformer_cache) or state.cache_auto,cache_engaged = cache_breaks_graph(state.transformer_cache),offload_active = state.offload_policy != OFFLOAD_NONE,+ denoiser_offloaded = state.offload_policy != OFFLOAD_NONE+ and _denoiser_hooked(state.pipe),logger = logger,)if getattr(getattr(state.pipe, "vae", None), "_unsloth_fp16_decode", False):@@ -8936,12 +9025,11 @@ class DiffusionBackend:gen.eta_seconds = Nonetry:- # torchao aten.to fails torch's aliasing check under inference_mode; model offload moves weights.+ # torchao aten.to fails torch's aliasing check under inference_mode once offloaded.with ((torch.no_grad()- if state.transformer_quant- and state.offload_policy == OFFLOAD_MODEL+ if _torchao_render_needs_no_grad(state)else torch.inference_mode()),protect_ctx,diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex db864631521..43bed7f4738 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1002,6 +1002,234 @@ def plan_keeps_transformer_resident(plan: Any) -> bool:return policy == OFFLOAD_GROUP and not bool(getattr(plan, "stream_transformer", True))+# Oldest torchao measured bit-exact under streamed group offload; 0.17 int8 (v1, no copy stream) ran 14x slower.+_TORCHAO_GROUP_OFFLOAD_MIN = {"int8": (0, 18), "fp8": (0, 17)}+_TORCHAO_STREAM_SAFE_CLASSES = frozenset(("Int8Tensor", "Float8Tensor"))+# Before 0.38 diffusers moved only the torchao wrapper, leaving quantised data on the host.+_DIFFUSERS_TORCHAO_GROUP_OFFLOAD_MIN = (0, 38)+_UNSET: Any = object()+++def _installed_version(package: str) -> Optional[tuple[int, int]]:+ try:+ import re+ from importlib.metadata import version++ major, minor = re.match(r"(\d+)\.(\d+)", version(package)).groups()+ return int(major), int(minor)+ except Exception: # noqa: BLE001 - absent / unreadable metadata+ return None++[email protected]_cache(maxsize = 1)+def _installed_torchao_version() -> Optional[tuple[int, int]]:+ return _installed_version("torchao")++[email protected]_cache(maxsize = 1)+def _installed_diffusers_version() -> Optional[tuple[int, int]]:+ return _installed_version("diffusers")+++def _model_offload_fits_quantised(plan: Any) -> bool:+ """Quantised denoiser and encoders (SUM stands in for the largest) must each fit the budget whole."""+ try:+ est = plan.estimates+ budget = est.get("safe_device_budget_mib")+ model = est.get("model_dense_mib")+ companions = est.get("companion_dense_mib")+ if budget is None or model is None or companions is None:+ return False+ denoiser = max(0, int(model) - int(companions))+ overhead = int(est.get("runtime_headroom_mib") or 0) + int(+ est.get("base_overhead_mib") or 0+ )+ encoders = est.get("text_encoder_dense_mib")+ if encoders is None:+ encoders = companions+ return denoiser + overhead <= int(budget) and int(encoders) <= int(budget)+ except Exception: # noqa: BLE001 - malformed plan: keep the resident rule+ return False+++def torchao_offload_plan(+ plan: Any,+ scheme: Optional[str],+ *,+ torchao_version: Any = _UNSET,+ reclaimable_host_mib: int = 0,+) -> Optional[Any]:+ """Placement a torchao ``scheme`` denoiser survives under ``plan``, or None (sequential offload: never measured).+ ``reclaimable_host_mib``: host RAM the outgoing pipeline frees before this one pins."""+ if plan_keeps_transformer_resident(plan):+ return plan+ policy = getattr(plan, "offload_policy", OFFLOAD_NONE)+ if policy == OFFLOAD_MODEL and _model_offload_fits_quantised(plan):+ return plan+ if policy not in (OFFLOAD_MODEL, OFFLOAD_GROUP, OFFLOAD_STREAMING):+ return None+ if not torchao_scheme_streams(scheme, torchao_version = torchao_version):+ return None+ if not _torchao_stream_pinnable(plan, reclaimable_host_mib):+ return None+ if policy != OFFLOAD_MODEL:+ return plan+ return torchao_streaming_plan(plan)+++def _torchao_stream_pinnable(plan: Any, reclaimable_host_mib: int = 0) -> bool:+ """Lazy pinning refuses torchao and the stream-free fallback ran ~35x slower, so require an up-front pin."""+ forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()+ if forced in ("0", "off", "false", "no"):+ return False+ if forced in ("1", "on", "true", "yes"):+ return True+ if _pinned_memory_capped():+ return False+ try:+ est = plan.estimates+ denoiser = int(est["model_dense_mib"]) - int(est["companion_dense_mib"])+ except Exception: # noqa: BLE001 - an unsized plan cannot prove the pin fits+ return False+ budget = _pin_budget_mib()+ return budget is not None and 0 <= denoiser <= budget + max(0, int(reclaimable_host_mib))+++def pipeline_host_mib(pipe: Any) -> int:+ """Host MiB held by ``pipe``'s components, which an unload hands back."""+ if pipe is None:+ return 0+ total = 0+ try:+ components = getattr(pipe, "components", None) or {}+ for module in components.values():+ if hasattr(module, "parameters"):+ total += _module_host_mib_on_cpu(module)+ except Exception: # noqa: BLE001 - unsizeable: credit nothing+ return 0+ return total+++def _module_host_mib_on_cpu(module: Any) -> int:+ try:+ seen: set[int] = set()+ nbytes = 0+ for tensor in (*module.parameters(), *module.buffers()):+ if id(tensor) in seen or getattr(tensor, "device", None) is None:+ continue+ seen.add(id(tensor))+ if tensor.device.type == "cpu":+ nbytes += sum(_storage_nbytes(tensor))+ return nbytes >> 20+ except Exception: # noqa: BLE001+ return 0+++def torchao_survives_plan(+ plan: Any,+ scheme: Optional[str],+ *,+ torchao_version: Any = _UNSET,+ reclaimable_host_mib: int = 0,+) -> bool:+ return (+ torchao_offload_plan(+ plan,+ scheme,+ torchao_version = torchao_version,+ reclaimable_host_mib = reclaimable_host_mib,+ )+ is not None+ )+++def torchao_scheme_streams(scheme: Optional[str], *, torchao_version: Any = _UNSET) -> bool:+ floor = _TORCHAO_GROUP_OFFLOAD_MIN.get(str(scheme))+ if floor is None:+ return False+ diffusers_version = _installed_diffusers_version()+ if diffusers_version is None or diffusers_version < _DIFFUSERS_TORCHAO_GROUP_OFFLOAD_MIN:+ return False+ version = _installed_torchao_version() if torchao_version is _UNSET else torchao_version+ return version is not None and tuple(version)[:2] >= floor+++def torchao_streaming_plan(plan: Any) -> Any:+ return replace(+ plan,+ offload_policy = OFFLOAD_STREAMING,+ reasons = tuple(plan.reasons)+ + (+ "the quantised transformer or a text encoder does not fit the device budget whole; streaming "+ "transformer blocks and text-encoder layers",+ ),+ )+++def _torchao_weight_classes(module: Any) -> set[str]:+ classes: set[str] = set()+ try:+ for param in module.parameters():+ for tensor in (param, getattr(param, "data", None)):+ if tensor is not None and type(tensor).__module__.startswith("torchao"):+ classes.add(type(tensor).__name__)+ except Exception: # noqa: BLE001 - unreadable: report nothing, the caller keeps its kwargs+ return set()+ return classes+++def _torchao_group_offload_kwargs(+ module: Any,+ kwargs: dict[str, Any],+ pinned_mib: Optional[list] = None,+) -> dict[str, Any]:+ """Group-offload kwargs a torchao ``module`` survives. Frozen first (swap_tensors on a requires_grad torchao weight+ hits an unimplemented ``aten.view``); the stream needs an up-front pin within ``pinned_mib``, a shared running total."""+ classes = _torchao_weight_classes(module)+ if not classes:+ return kwargs+ _freeze_torchao_weights(module)+ install_group_offload_torchao_swap_retry()+ if not kwargs.get("use_stream"):+ return kwargs+ if classes <= _TORCHAO_STREAM_SAFE_CLASSES:+ if not kwargs.get("low_cpu_mem_usage"):+ return kwargs+ # Same override the planner's _torchao_stream_pinnable honoured, so the two cannot disagree.+ forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()+ if forced in ("1", "on", "true", "yes"):+ return {**kwargs, "low_cpu_mem_usage": False}+ budget = (+ None+ if forced in ("0", "off", "false", "no") or _pinned_memory_capped()+ else _pin_budget_mib()+ )+ need = _module_host_mib(module)+ already = pinned_mib[0] if pinned_mib else 0+ if budget is not None and already + need <= budget:+ if pinned_mib is not None:+ pinned_mib[0] = already + need+ return {**kwargs, "low_cpu_mem_usage": False}+ safe = {+ k: v+ for k, v in kwargs.items()+ if k not in ("non_blocking", "record_stream", "low_cpu_mem_usage")+ }+ safe["use_stream"] = False+ return safe+++def _freeze_torchao_weights(module: Any) -> None:+ try:+ for param in module.parameters():+ if type(param).__module__.startswith("torchao") and getattr(+ param, "requires_grad", False+ ):+ param.requires_grad_(False)+ except Exception: # noqa: BLE001 - an inference-only flag: leave the module as it is+ pass++def _holds_torchao_weights(module: Any) -> bool:"""Whether any parameter of ``module`` is a torchao tensor subclass (GGUF and native int8 are not)."""try:@@ -1514,6 +1742,12 @@ def apply_memory_plan("the text encoder could not be streamed beside the resident quantised ""transformer, and whole-module offload cannot move torchao weights")+ if _pipe_denoisers_hold_torchao(pipe) and not _model_offload_fits_quantised(plan):+ # Whole-module offload would onload the quantised transformer whole, which is what streaming avoided.+ raise RuntimeError(+ "group offloading could not be set up for the quantised transformer, and it does not "+ "fit the GPU whole for whole-module offload"+ )_fallback_to_model_offload()policy = OFFLOAD_MODELelif policy == OFFLOAD_STREAMING:@@ -1599,14 +1833,28 @@ def _module_host_mib(module: Any) -> int:if id(tensor) in seen:continueseen.add(id(tensor))- nbytes = int(tensor.numel()) * int(tensor.element_size())- if nbytes > 0:- total += 1 << (nbytes - 1).bit_length()+ for nbytes in _storage_nbytes(tensor):+ if nbytes > 0:+ total += 1 << (nbytes - 1).bit_length()return total // (1024 * 1024)except Exception: # noqa: BLE001 - an unsizeable module is priced as nothing to pinreturn 0+def _storage_nbytes(tensor: Any, depth: int = 0) -> list[int]:+ """Bytes per allocation; torchao subclasses report their logical bf16 size, so size the packed inner tensors."""+ flatten = getattr(type(tensor), "__tensor_flatten__", None)+ if flatten is not None and depth < 4:+ try:+ names, _ctx = tensor.__tensor_flatten__()+ inner = [getattr(tensor, name, None) for name in names]+ if inner and all(t is not None for t in inner):+ return [n for t in inner for n in _storage_nbytes(t, depth + 1)]+ except Exception: # noqa: BLE001 - fall back to the logical size, which only over-counts+ pass+ return [int(tensor.numel()) * int(tensor.element_size())]++def _pin_budget_mib() -> Optional[int]:"""Pinnable host MiB leaving ``max(4 GiB, 15%)`` free, or None if unreadable."""total, _available = _system_memory_mib()@@ -1685,6 +1933,33 @@ def _streamed_pin_plan(return pin_transformer, pin_encoders+def install_group_offload_torchao_swap_retry() -> bool:+ """``swap_tensors`` refuses a weakref'd tensor; retry once after gc so uncollected compile garbage does not fail."""+ try:+ from diffusers.hooks import group_offloading as go+ except Exception: # noqa: BLE001 - no group offload in this diffusers+ return False+ original = getattr(go, "_swap_torchao_tensor", None)+ if original is None or getattr(original, "_unsloth_swap_retry", False):+ return False++ @functools.wraps(original)+ def _swap_torchao_tensor(param: Any, source: Any) -> None:+ try:+ original(param, source)+ except RuntimeError as exc:+ if "weakref" not in str(exc):+ raise+ import gc++ gc.collect()+ original(param, source)++ _swap_torchao_tensor._unsloth_swap_retry = True+ go._swap_torchao_tensor = _swap_torchao_tensor+ return True++BACKGROUND_PIN_ENV = "UNSLOTH_DIFFUSION_BACKGROUND_PIN"_BG_PIN_ATTR = "_unsloth_background_pin"_PENDING_PINS_ATTR = "_unsloth_background_pins"@@ -2074,12 +2349,25 @@ def _apply_group_offload(continueif isinstance(comp, torch.nn.Module):comp.to(onload)+ # Encoders the plan pins count against the same budget as any torchao denoiser pinned below.+ pinned_mib = [+ sum(_module_host_mib(m) for m in streamed_encoders.values())+ if stream_text_encoders and pin_streamed[1]+ else 0+ ]for module in streamed.values():- if defer and not gkwargs.get("low_cpu_mem_usage", False):+ # torchao weights need their up-front pin (lazy pinning refuses them), so they never defer.+ if (+ defer+ and not gkwargs.get("low_cpu_mem_usage", False)+ and not _torchao_weight_classes(module)+ ):apply_group_offloading(module, **{**gkwargs, "low_cpu_mem_usage": True})_defer_pinning(pipe, module, onload, logger)else:- apply_group_offloading(module, **gkwargs)+ apply_group_offloading(+ module, **_torchao_group_offload_kwargs(module, gkwargs, pinned_mib)+ )installed += 1# The encoders come AFTER the DiTs and are applied one by one, each failure absorbed. A text encoder is a far# less well-trodden target for block-level group offloading than a DiT (a family whose encoder exposes no@@ -2121,7 +2409,9 @@ def _apply_group_offload(dkwargs = dict(gkwargs)if "low_cpu_mem_usage" in _params and use_stream:dkwargs["low_cpu_mem_usage"] = True- apply_group_offloading(dit, **dkwargs)+ apply_group_offloading(+ dit, **_torchao_group_offload_kwargs(dit, dkwargs, pinned_mib)+ )installed += 1transformer_demoted = Trueif logger is not None:@@ -2636,6 +2926,7 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:if isinstance(component, torch.nn.Module):component.to(onload)+ pinned_mib = [0]for module, offload_type in streamed.values():kwargs: dict[str, Any] = {"onload_device": onload,@@ -2652,7 +2943,9 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:kwargs["record_stream"] = Falseif use_stream and "low_cpu_mem_usage" in params:kwargs["low_cpu_mem_usage"] = True- apply_group_offloading(module, **kwargs)+ apply_group_offloading(+ module, **_torchao_group_offload_kwargs(module, kwargs, pinned_mib)+ )installed += 1if offload_type == "leaf_level":_pin_vision_embedding_device(module)diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 0836c85ea1f..3b61dda7258 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -421,6 +421,7 @@ def apply_speed_optims(offload_active: bool = False,cuda_graph_default: bool = True,cache_engaged: Optional[bool] = None,+ denoiser_offloaded: Optional[bool] = None,logger: Any = None,) -> dict[str, bool]:"""Apply the opt-in speed optims for ``speed_mode`` to a built pipeline, BEFORE placement /@@ -428,6 +429,7 @@ def apply_speed_optims(``offload_active`` (offload policy != none) installs ``@torch.compiler.disable``d onload hooks,so the compile must drop ``fullgraph`` (like an active step cache) or it crashes at step 1.+ ``denoiser_offloaded`` (None = ``offload_active``) limits the CUDA-graph refusal to a moved denoiser.``cuda_graph_default`` is what the CUDA-graph arm assumes for a family that declares nothing:True on the image backend, False on video, where ``supports_cuda_graph`` opts in.@@ -560,7 +562,9 @@ def apply_speed_optims(target,family = family,pipe = pipe,- offload_active = offload_active,+ offload_active = offload_active+ if denoiser_offloaded is None+ else bool(denoiser_offloaded),cache_active = cache_active if cache_engaged is None else bool(cache_engaged),speed_mode = mode,family_default = cuda_graph_default,diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex 4f5c40477b2..9f3b88dc171 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -54,6 +54,16 @@ from core.inference.diffusion_families import ()[email protected](autouse = True)+def _unmeasured_torchao(monkeypatch):+ """Pin "no measured torchao" so the installed release does not decide the offload tiers."""+ from core.inference import diffusion_memory++ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: None)+ # Studio's diffusers pin, so tests that opt into a measured torchao do not depend on the runner.+ monkeypatch.setattr(diffusion_memory, "_installed_diffusers_version", lambda: (0, 40))++# Pure family helpers@@ -12163,7 +12173,8 @@ def test_a_pipeline_pick_quantises_under_whole_module_offload(fake_runtime, tmp_@pytest.mark.parametrize(- ("offload_policy", "expected"), [("none", "inference_mode"), ("model", "no_grad")]+ ("offload_policy", "expected"),+ [("none", "inference_mode"), ("model", "no_grad"), ("group", "no_grad")],)def test_an_offloaded_quantised_transformer_renders_outside_inference_mode(fake_runtime, tmp_path, monkeypatch, offload_policy, expected@@ -12171,6 +12182,10 @@ def test_an_offloaded_quantised_transformer_renders_outside_inference_mode("""torchao tensors cannot change device under inference_mode, so offloaded quant renders use no_grad."""import torch+ from core.inference import diffusion_memory++ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)backend = DiffusionBackend()_stub_pipeline_dense_quant(backend, monkeypatch)monkeypatch.setattr(DiffusionBackend, "_plan_memory", _offload_plan(offload_policy))@@ -12290,6 +12305,63 @@ def test_a_pipeline_pick_stays_dense_when_the_quantised_transformer_exceeds_the_backend.unload()[email protected]("torchao_version", [(0, 17), (0, 18)])+def test_a_pipeline_pick_quantises_under_streamed_group_offload_on_a_measured_torchao(+ fake_runtime, tmp_path, monkeypatch, torchao_version+):+ from core.inference import diffusion_memory++ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: torchao_version)+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)+ backend = DiffusionBackend()+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _offload_plan("group"))+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)+ )+ assert len(calls) == 1+ assert status["transformer_quant"] == "fp8"+ backend.unload()+++ ("budget_mib", "runtime_headroom_mib", "companion_mib"),+ [(1, 0, None), (1_000_000, 1_000_000, None), (1_000_000, 0, 2_000_000)],+)+def test_a_quantised_transformer_too_big_to_onload_whole_streams_instead(+ fake_runtime, tmp_path, monkeypatch, budget_mib, runtime_headroom_mib, companion_mib+):+ from core.inference import diffusion as dmod+ from core.inference import diffusion_memory++ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)+ # diffusion.py binds its own reference; without this the host's real pin budget decides.+ monkeypatch.setattr(dmod, "_torchao_stream_pinnable", lambda plan, *_a: True)+ backend = DiffusionBackend()+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)+ monkeypatch.setattr(dmod, "largest_streamable_companion_mib", lambda pipe: companion_mib)+ placed: list = []++ def _apply(pipe, plan, **_kwargs):+ placed.append(plan.offload_policy)+ return plan.offload_policy, plan.vae_tiling++ monkeypatch.setattr(dmod, "apply_memory_plan", _apply)+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ _offload_plan("model", budget_mib = budget_mib, runtime_headroom_mib = runtime_headroom_mib),+ )+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)+ )+ assert len(calls) == 1+ assert status["transformer_quant"] == "fp8"+ assert placed == ["streaming"] and status["offload_policy"] == "streaming"+ backend.unload()++def test_an_offloaded_pipeline_replans_against_the_quantised_size(fake_runtime, tmp_path, monkeypatch):@@ -13404,6 +13476,52 @@ def test_the_act_kill_switch_keeps_nvidia_offload_native_but_weight_only(backend.unload()+def test_an_explicit_fp8_under_group_offload_engages_on_a_measured_torchao(+ fake_runtime, tmp_path, monkeypatch+):+ from core.inference import diffusion_memory++ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)+ backend = DiffusionBackend()+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch, engages = "fp8")+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "fp8",+ memory_mode = "balanced",+ _base_local_dir = str(tmp_path),+ )+ assert len(calls) == 1 and "offload" not in calls[0] and reasons == []+ assert status["transformer_quant"] == "fp8"+ backend.unload()+++ "memory",+ [{"memory_mode": "balanced"}, {"memory_mode": "low_vram"}, {"cpu_offload": True}],+)+def test_an_explicit_int8_under_offload_stays_native_on_a_measured_torchao(+ fake_runtime, tmp_path, monkeypatch, memory+):+ from core.inference import diffusion_memory++ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)+ backend = DiffusionBackend()+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "int8",+ _base_local_dir = str(tmp_path),+ **memory,+ )+ assert len(calls) == 1 and calls[0]["offload"] is True+ assert status["transformer_quant"] == "int8" and reasons == ["int8"]+ backend.unload()++@pytest.mark.parametrize("fallback", ["0", "1"])def test_an_explicit_fp8_under_offload_on_nvidia_is_still_declined(fake_runtime, tmp_path, monkeypatch, fallbackdiff --git a/studio/backend/tests/test_diffusion_offload_precision.py b/studio/backend/tests/test_diffusion_offload_precision.pynew file mode 100644index 00000000000..3abd8e186ca--- /dev/null+++ b/studio/backend/tests/test_diffusion_offload_precision.py@@ -0,0 +1,799 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A torchao int8 / fp8 denoiser stays quantised when the image plan offloads it."""++from __future__ import annotations++import types+from dataclasses import replace++import pytest++from core.inference import diffusion as dmod+from core.inference import diffusion_memory as mem+from core.inference.diffusion_denoiser_prequant import PIPELINE_SEED_DECLINED+from core.inference.diffusion_memory import (+ OFFLOAD_GROUP,+ OFFLOAD_MODEL,+ OFFLOAD_NONE,+ OFFLOAD_SEQUENTIAL,+ OFFLOAD_STREAMING,+ DeviceMemory,+ plan_diffusion_memory,+ torchao_offload_plan,+ torchao_survives_plan,+)++T017, T018, T016 = (0, 17), (0, 18), (0, 16)++[email protected](autouse = True)+def _roomy_host(monkeypatch):+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 10**7)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ # Studio's diffusers pin; a runner without diffusers would otherwise refuse every streamed tier.+ monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: (0, 40))+++def _plan(+ policy,+ *,+ stream_transformer = True,+ fits_model_offload = True,+):+ device = DeviceMemory("cuda", "cuda:0", "discrete_vram", 15_000, 16_384)+ plan = plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = device,+ model_dense_mib = 12_000,+ runtime_headroom_mib = 2_000,+ companion_dense_mib = 4_000,+ text_encoder_dense_mib = 3_800,+ )+ estimates = dict(plan.estimates)+ if not fits_model_offload:+ estimates["model_dense_mib"] = 40_000+ return replace(+ plan,+ offload_policy = policy,+ stream_transformer = stream_transformer,+ estimates = estimates,+ )+++PLACEMENTS = {+ "resident": _plan(OFFLOAD_NONE),+ "group_dit_resident": _plan(OFFLOAD_GROUP, stream_transformer = False),+ "group_dit_streamed": _plan(OFFLOAD_GROUP),+ "model_fits": _plan(OFFLOAD_MODEL),+ "model_too_big": _plan(OFFLOAD_MODEL, fits_model_offload = False),+ "streaming": _plan(OFFLOAD_STREAMING),+ "sequential": _plan(OFFLOAD_SEQUENTIAL),+}++# placement -> scheme -> torchao version -> (survives, policy it runs on)+SURVIVAL = {+ "resident": {+ s: {v: (True, OFFLOAD_NONE) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")+ },+ "group_dit_resident": {+ s: {v: (True, OFFLOAD_GROUP) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")+ },+ "group_dit_streamed": {+ "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_GROUP)},+ "fp8": {T016: (False, None), T017: (True, OFFLOAD_GROUP), T018: (True, OFFLOAD_GROUP)},+ "nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},+ },+ "model_fits": {+ s: {v: (True, OFFLOAD_MODEL) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")+ },+ "model_too_big": {+ "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_STREAMING)},+ "fp8": {+ T016: (False, None),+ T017: (True, OFFLOAD_STREAMING),+ T018: (True, OFFLOAD_STREAMING),+ },+ "nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},+ },+ "streaming": {+ "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_STREAMING)},+ "fp8": {+ T016: (False, None),+ T017: (True, OFFLOAD_STREAMING),+ T018: (True, OFFLOAD_STREAMING),+ },+ "nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},+ },+ "sequential": {+ s: {v: (False, None) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")+ },+}+CASES = [+ (placement, scheme, version, *expected)+ for placement, by_scheme in SURVIVAL.items()+ for scheme, by_version in by_scheme.items()+ for version, expected in by_version.items()+]++[email protected]("placement, scheme, version, survives, policy", CASES)+def test_survival_table(placement, scheme, version, survives, policy):+ plan = PLACEMENTS[placement]+ assert torchao_survives_plan(plan, scheme, torchao_version = version) is survives+ placed = torchao_offload_plan(plan, scheme, torchao_version = version)+ assert (placed.offload_policy if placed is not None else None) == policy+++ "diffusers_version, streams",+ [((0, 36), False), ((0, 37), False), ((0, 38), True), (None, False)],+)+def test_diffusers_before_the_torchao_swap_keeps_the_resident_rule(+ monkeypatch, diffusers_version, streams+):+ monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: diffusers_version)+ assert (+ torchao_survives_plan(PLACEMENTS["group_dit_streamed"], "int8", torchao_version = T018)+ is streams+ )+ assert torchao_survives_plan(PLACEMENTS["group_dit_resident"], "int8", torchao_version = T018)+++def test_swap_retry_collects_once_then_gives_up(monkeypatch):+ import sys++ calls: list = []+ failures = {"n": 1}++ def _swap(param, source):+ calls.append(param)+ if failures["n"] > 0:+ failures["n"] -= 1+ raise RuntimeError("Cannot swap t1 because it has weakref associated with it")++ go = types.SimpleNamespace(_swap_torchao_tensor = _swap)+ hooks = types.ModuleType("diffusers.hooks")+ hooks.group_offloading = go+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)+ monkeypatch.setitem(sys.modules, "diffusers.hooks.group_offloading", go)+ assert mem.install_group_offload_torchao_swap_retry()+ assert not mem.install_group_offload_torchao_swap_retry()+ go._swap_torchao_tensor("p", "s")+ assert calls == ["p", "p"]+ failures["n"] = 2+ with pytest.raises(RuntimeError, match = "weakref"):+ go._swap_torchao_tensor("p", "s")+++ "capped, budget, env, streams",+ [+ (False, 10**7, None, True),+ (True, 10**7, None, False),+ (False, 7_999, None, False),+ (False, None, None, False),+ (False, 10**7, "0", False),+ (True, 0, "1", True),+ ],+)+def test_a_streamed_torchao_denoiser_needs_its_pin(monkeypatch, capped, budget, env, streams):+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: capped)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)+ if env is not None:+ monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, env)+ for name in ("group_dit_streamed", "streaming", "model_too_big"):+ assert (+ torchao_survives_plan(PLACEMENTS[name], "int8", torchao_version = T018) is streams+ ), name+ assert torchao_survives_plan(PLACEMENTS["group_dit_resident"], "int8", torchao_version = T018)+ assert torchao_survives_plan(PLACEMENTS["model_fits"], "int8", torchao_version = T018)+++def test_no_torchao_keeps_the_resident_rule():+ assert not torchao_survives_plan(PLACEMENTS["group_dit_streamed"], "int8", torchao_version = None)+ assert torchao_survives_plan(PLACEMENTS["group_dit_resident"], "int8", torchao_version = None)+++def test_never_moves_still_means_resident_only():+ keeps = mem.plan_keeps_transformer_resident+ assert keeps(PLACEMENTS["resident"]) and keeps(PLACEMENTS["group_dit_resident"])+ for name in ("group_dit_streamed", "model_fits", "streaming"):+ assert not keeps(PLACEMENTS[name]), name+++class _Weight:+ def __init__(+ self,+ cls_name: str,+ module: str = "torchao.quantization.fake",+ ):+ self.data = None+ self.__class__ = type(cls_name, (_Weight,), {"__module__": module})+++class _Module:+ def __init__(+ self,+ *class_names: str,+ plain: bool = False,+ ):+ self._params = [_Weight(name) for name in class_names]+ if plain:+ self._params.append(_Weight("Parameter", module = "torch.nn.parameter"))++ def parameters(self):+ return iter(self._params)+++STREAM_KW = {+ "onload_device": "cuda",+ "use_stream": True,+ "non_blocking": True,+ "record_stream": True,+}+NO_STREAM = {"onload_device": "cuda", "use_stream": False}+++ "classes, low_cpu_mem_usage, pin_budget, expected",+ [+ (("Int8Tensor",), False, 10**7, "as is"),+ (("Float8Tensor",), False, 10**7, "as is"),+ (("Int8Tensor", "Float8Tensor"), False, 10**7, "as is"),+ (("Int8Tensor",), True, 10**7, "pinned"),+ (("Float8Tensor",), True, 10**7, "pinned"),+ (("Int8Tensor",), True, None, "no stream"),+ # torchao 0.17 default int8: no aten.is_pinned, so no copy stream+ (("LinearActivationQuantizedTensor",), False, 10**7, "no stream"),+ ],+)+def test_group_offload_kwargs_per_weight_class(+ monkeypatch, classes, low_cpu_mem_usage, pin_budget, expected+):+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: pin_budget)+ kwargs = {**STREAM_KW, "low_cpu_mem_usage": low_cpu_mem_usage}+ out = mem._torchao_group_offload_kwargs(_Module(*classes, plain = True), kwargs)+ if expected == "as is":+ assert out is kwargs+ elif expected == "pinned":+ assert out == {**STREAM_KW, "low_cpu_mem_usage": False}+ else:+ assert out == NO_STREAM+++def test_group_offload_kwargs_leave_dense_and_stream_free_modules_alone():+ dense = {**STREAM_KW, "low_cpu_mem_usage": True}+ assert mem._torchao_group_offload_kwargs(_Module(plain = True), dense) is dense+ assert mem._torchao_group_offload_kwargs(_Module("Int8Tensor"), NO_STREAM) is NO_STREAM+++ "policy, transformer_quant, text_encoder_quant, no_grad",+ [+ (OFFLOAD_NONE, "int8", None, False),+ (OFFLOAD_NONE, "int8", "fp8", False),+ (OFFLOAD_MODEL, "int8", None, True),+ (OFFLOAD_GROUP, "int8", None, True),+ (OFFLOAD_GROUP, "fp8", None, True),+ (OFFLOAD_STREAMING, "int8", None, True),+ (OFFLOAD_GROUP, None, "fp8", True),+ (OFFLOAD_GROUP, None, None, False),+ (OFFLOAD_MODEL, None, None, False),+ ],+)+def test_render_grad_mode(policy, transformer_quant, text_encoder_quant, no_grad):+ state = types.SimpleNamespace(+ offload_policy = policy,+ transformer_quant = transformer_quant,+ text_encoder_quant = text_encoder_quant,+ )+ assert dmod._torchao_render_needs_no_grad(state) is no_grad+++def test_render_uses_the_widened_switch():+ import inspect++ source = inspect.getsource(dmod.DiffusionBackend)+ assert "if _torchao_render_needs_no_grad(state)" in source+ assert "and state.offload_policy == OFFLOAD_MODEL" not in source+++def _estimate(steady):+ return types.SimpleNamespace(steady_transformer_mib = steady)+++ "placement, estimate_mib, largest_te_mib, version, policy, declined",+ [+ ("resident", 6_000, 4_000, T018, OFFLOAD_NONE, False),+ ("group_dit_streamed", 6_000, 4_000, T018, OFFLOAD_GROUP, False),+ ("group_dit_streamed", 6_000, 4_000, T016, OFFLOAD_GROUP, True),+ ("model_fits", 6_000, 4_000, T018, OFFLOAD_MODEL, False),+ # quantised denoiser or encoder too big to onload whole: stream instead of falling back to bf16+ ("model_fits", 60_000, 4_000, T018, OFFLOAD_STREAMING, False),+ ("model_fits", 6_000, 60_000, T018, OFFLOAD_STREAMING, False),+ ("model_fits", 60_000, 4_000, T016, OFFLOAD_MODEL, True),+ ("sequential", 6_000, 4_000, T018, OFFLOAD_SEQUENTIAL, True),+ ],+)+def test_inplace_quant_placement(+ monkeypatch, placement, estimate_mib, largest_te_mib, version, policy, declined+):+ monkeypatch.setattr(mem, "_installed_torchao_version", lambda: version)+ plan, why = dmod._inplace_torchao_placement(+ PLACEMENTS[placement], "int8", _estimate(estimate_mib), lambda: largest_te_mib+ )+ assert plan.offload_policy == policy+ assert (why is not None) is declined++[email protected]("estimate_mib, largest_te_mib", [(60_000, 4_000), (6_000, 60_000)])+def test_inplace_quant_declines_streaming_it_cannot_pin(monkeypatch, estimate_mib, largest_te_mib):+ """Unpinnable (Windows, capped host RAM) streaming runs without a copy stream, ~35x slower: decline it."""+ monkeypatch.setattr(mem, "_installed_torchao_version", lambda: T018)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 0)+ plan, why = dmod._inplace_torchao_placement(+ PLACEMENTS["model_fits"], "int8", _estimate(estimate_mib), lambda: largest_te_mib+ )+ assert plan.offload_policy == OFFLOAD_MODEL+ assert why is not None+++def test_inplace_quant_no_longer_calls_group_offload_wrong():+ import inspect+ source = inspect.getsource(dmod)+ assert "Group offload is WRONG for torchao" not in source+++GIB = 1024+++def planner(monkeypatch):+ import torch++ from core.inference import diffusion_auto_policy as ap+ from core.inference import diffusion_transformer_quant as tq++ monkeypatch.setattr(mem, "_installed_torchao_version", lambda: T018)+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *_a, **_k: (12, 0))+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)++ def _supported(+ scheme,+ device = None,+ unproven_ok = False,+ ):+ return scheme in ("int8", "fp8")++ monkeypatch.setattr(tq, "_scheme_supported", _supported)+ monkeypatch.setattr(tq, "_is_consumer_gpu", lambda *_a, **_k: True)+ monkeypatch.setattr(ap, "_hf_cache_free_mib", lambda: 10**7)+ card: dict = {}+ for module in (dmod, mem):+ for fn in (+ "snapshot_device_memory",+ "settled_snapshot_device_memory",+ "reclaimable_snapshot_device_memory",+ ):+ if hasattr(module, fn):+ monkeypatch.setattr(module, fn, lambda *_a, **_k: card["memory"])+ target = types.SimpleNamespace(+ device = "cuda",+ dtype = torch.bfloat16,+ backend = "cuda",+ supports_model_cpu_offload = True,+ supports_default_torch_compile = True,+ ordinal = None,+ capability = (12, 0),+ )+ backend = dmod.DiffusionBackend.__new__(dmod.DiffusionBackend)+ backend._resolve_device_target = lambda *_a, **_k: target+ backend._target_for_ordinal = lambda *_a, **_k: target+ backend._cache_bytes = lambda *_a, **_k: 0+ backend._released_transformer_cached = lambda *_a, **_k: False++ def pick(family_name: str, base: str, gib: int):+ from core.inference.diffusion_families import _FAMILIES+ from core.inference.diffusion_transformer_quant import select_transformer_quant_scheme++ fam = next(f for f in _FAMILIES if f.name == family_name)+ card["memory"] = DeviceMemory("cuda", "cuda", "discrete_vram", gib * GIB - 600, gib * GIB)+ seed = backend._pipeline_planned_denoiser_scheme(+ fam,+ base = base,+ kind = "pipeline",+ transformer_quant = None,+ speed_mode = None,+ repo_id = base,+ fetch_base = base,+ )+ if seed not in (None, PIPELINE_SEED_DECLINED):+ seeded = backend._seeded_pipeline_plan(+ seed,+ target,+ base,+ fam,+ None,+ False,+ repo_id = base,+ base_local_dir = None,+ fetch_base = base,+ )+ placed = torchao_offload_plan(seeded, seed)+ assert placed is not None+ return f"hosted {seed}", placed.offload_policy+ plan = backend._bf16_table_plan(+ target, fam, base, None, False, kind = "pipeline", repo_id = base, fetch_base = base+ )+ if dmod._auto_quant_eager_reason(fam, plan, None, "pipeline") is not None:+ return "bf16", plan.offload_policy+ scheme = select_transformer_quant_scheme(target, "auto", family = fam.name, base_repo = base)+ estimate = ap.estimate_dense_quant(fam, scheme, base_repo = base)+ if plan.offload_policy != OFFLOAD_NONE:+ replanned = backend._plan_memory(+ target,+ None,+ base,+ fam,+ None,+ False,+ kind = "pipeline",+ repo_id = base,+ fetch_base = base,+ transformer_resident_override_mib = estimate.steady_transformer_mib,+ **backend._candidate_companion_overrides(estimate, fam, base, target, None),+ )+ placed = torchao_offload_plan(replanned, scheme)+ if placed is not None:+ plan = placed+ largest_te = int(ap.family_bf16_components_gb(fam, base)[1] * 1000**3 / 1024**2)+ plan, why = dmod._inplace_torchao_placement(plan, scheme, estimate, lambda: largest_te)+ return ("bf16" if why else f"on-the-fly {scheme}"), plan.offload_policy++ return pick+++FAMILY_TABLE = {+ ("krea-2", "krea/Krea-2-Turbo"): {+ 8: ("on-the-fly int8", OFFLOAD_STREAMING),+ 12: ("on-the-fly int8", OFFLOAD_GROUP),+ 16: ("on-the-fly int8", OFFLOAD_GROUP),+ 24: ("on-the-fly int8", OFFLOAD_GROUP),+ 32: ("on-the-fly int8", OFFLOAD_GROUP),+ },+ ("lumina-2", "Alpha-VLLM/Lumina-Image-2.0"): {+ 8: ("hosted int8", OFFLOAD_STREAMING),+ 12: ("hosted int8", OFFLOAD_STREAMING),+ 16: ("hosted int8", OFFLOAD_GROUP),+ 24: ("hosted int8", OFFLOAD_NONE),+ # measured: bf16 while it fits resident+ 32: ("bf16", OFFLOAD_NONE),+ },+ ("hunyuanimage-2.1", "hunyuanvideo-community/HunyuanImage-2.1-Diffusers"): {+ 8: ("hosted int8", OFFLOAD_STREAMING),+ 12: ("hosted int8", OFFLOAD_STREAMING),+ 16: ("hosted int8", OFFLOAD_GROUP),+ 24: ("hosted int8", OFFLOAD_GROUP),+ 32: ("hosted int8", OFFLOAD_GROUP),+ },+ ("hidream-i1", "HiDream-ai/HiDream-I1-Full"): {+ 8: ("hosted int8", OFFLOAD_STREAMING),+ 12: ("hosted int8", OFFLOAD_STREAMING),+ 16: ("hosted int8", OFFLOAD_GROUP),+ 24: ("hosted int8", OFFLOAD_GROUP),+ 32: ("hosted int8", OFFLOAD_GROUP),+ },+ ("ideogram-4", "ideogram-ai/ideogram-4-fp8"): {+ 8: ("on-the-fly int8", OFFLOAD_STREAMING),+ 12: ("on-the-fly int8", OFFLOAD_STREAMING),+ 16: ("on-the-fly int8", OFFLOAD_GROUP),+ 24: ("on-the-fly int8", OFFLOAD_GROUP),+ 32: ("on-the-fly int8", OFFLOAD_GROUP),+ },+}+++ "family, base, gib, expected",+ [+ (family, base, gib, expected)+ for (family, base), by_gib in FAMILY_TABLE.items()+ for gib, expected in by_gib.items()+ ],+)+def test_auto_keeps_int8_on_the_offload_tier(planner, family, base, gib, expected):+ assert planner(family, base, gib) == expected++[email protected]("version, expected", [(T016, "bf16"), (T017, "hosted fp8")])+def test_older_torchao_streamed_seeds(planner, monkeypatch, version, expected):+ monkeypatch.setattr(mem, "_installed_torchao_version", lambda: version)+ precision, _policy = planner(+ "hunyuanimage-2.1", "hunyuanvideo-community/HunyuanImage-2.1-Diffusers", 16+ )+ assert precision == expected+++def _gpu_stack():+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA")+ pytest.importorskip("torchao")+ hooks = pytest.importorskip("diffusers.hooks")+ return torch, hooks++[email protected]("scheme", ["int8", "fp8"])[email protected]("low_cpu_mem_usage", [False, True])[email protected]("requires_grad", [False, True])+def test_quantised_blocks_survive_group_offload_under_no_grad(+ scheme, low_cpu_mem_usage, requires_grad+):+ import copy+ import inspect++ torch, hooks = _gpu_stack()+ from torchao.quantization import (+ Float8DynamicActivationFloat8WeightConfig,+ Int8DynamicActivationInt8WeightConfig,+ PerRow,+ quantize_,+ )++ if scheme == "fp8" and torch.cuda.get_device_capability() < (8, 9):+ pytest.skip("fp8 needs sm_89+")+ torch.manual_seed(0)+ blocks = torch.nn.Sequential(+ *[+ torch.nn.Sequential(+ torch.nn.Linear(256, 512), torch.nn.GELU(), torch.nn.Linear(512, 256)+ )+ for _ in range(3)+ ]+ ).to(torch.bfloat16)+ config = (+ Int8DynamicActivationInt8WeightConfig+ if scheme == "int8"+ else lambda: Float8DynamicActivationFloat8WeightConfig(granularity = PerRow())+ )+ offloaded = copy.deepcopy(blocks)+ quantize_(offloaded, config())+ # Same CPU-quantised weights: fp8 quantised on CPU can round an ulp away from CUDA (torch 2.12).+ resident = copy.deepcopy(offloaded).cuda()+ for param in offloaded.parameters():+ param.requires_grad_(requires_grad)+ params = inspect.signature(hooks.apply_group_offloading).parameters+ kwargs = {+ "onload_device": torch.device("cuda"),+ "offload_device": torch.device("cpu"),+ "offload_type": "block_level",+ "num_blocks_per_group": 1,+ "use_stream": True,+ }+ for name, value in (+ ("non_blocking", True),+ ("record_stream", True),+ ("low_cpu_mem_usage", low_cpu_mem_usage),+ ):+ if name in params:+ kwargs[name] = value+ hooks.apply_group_offloading(offloaded, **mem._torchao_group_offload_kwargs(offloaded, kwargs))+ x = torch.randn(2, 16, 256, dtype = torch.bfloat16, device = "cuda")+ with torch.no_grad():+ want = resident(x)+ for _ in range(3):+ assert torch.equal(offloaded(x), want)+ with pytest.raises(Exception), torch.inference_mode():+ offloaded(x)+++ "offload_active, denoiser_offloaded, expected",+ [(False, None, False), (True, None, True), (True, False, False), (True, True, True)],+)+def test_graph_gate_follows_the_denoiser(monkeypatch, offload_active, denoiser_offloaded, expected):+ from core.inference import diffusion_cuda_graph as dcg+ from core.inference import diffusion_speed++ seen: list = []++ def _eligible(target, **kwargs):+ seen.append(kwargs["offload_active"])+ return False, "stub"++ monkeypatch.setattr(dcg, "graph_eligible", _eligible)+ pipe = types.SimpleNamespace(components = {})+ target = types.SimpleNamespace(device = "cpu", backend = "cpu", dtype = None)+ kwargs = {} if denoiser_offloaded is None else {"denoiser_offloaded": denoiser_offloaded}+ diffusion_speed.apply_speed_optims(+ pipe,+ target,+ is_gguf = False,+ family = types.SimpleNamespace(name = "z-image"),+ speed_mode = "default",+ offload_active = offload_active,+ **kwargs,+ )+ assert seen == [expected]+++class _Hooked:+ def __init__(+ self,+ keys = (),+ hf_hook = None,+ ):+ self._diffusers_hook = types.SimpleNamespace(hooks = {k: object() for k in keys})+ self._hf_hook = hf_hook+++ "transformer, hooked",+ [+ (_Hooked(), False),+ (_Hooked(("group_offloading", "lazy_prefetch_group_offloading")), True),+ (_Hooked(hf_hook = object()), True),+ (_Hooked(("first_block_cache",)), False),+ ],+)+def test_denoiser_hooked(transformer, hooked):+ assert dmod._denoiser_hooked(types.SimpleNamespace(transformer = transformer)) is hooked+++def _int8_linear(rows = 1024, cols = 1024):+ torch = pytest.importorskip("torch")+ quant = pytest.importorskip("torchao.quantization")+ linear = torch.nn.Linear(cols, rows, bias = False).to(torch.bfloat16)+ try:+ quant.quantize_(linear, quant.Int8DynamicActivationInt8WeightConfig(version = 2))+ except Exception as exc: # noqa: BLE001 - torchao without the v2 int8 tensor+ pytest.skip(f"int8 v2 quantisation unavailable: {exc}")+ if type(linear.weight).__name__ != "Int8Tensor":+ pytest.skip("this torchao does not produce Int8Tensor")+ return linear+++def test_host_size_counts_packed_torchao_storage_not_the_logical_bf16_size():+ linear = _int8_linear(2048, 2048)+ # 2048 x 2048 int8 = 4 MiB of qdata plus per-row scales; the logical bf16 size would be 8 MiB.+ assert mem._module_host_mib(linear) < 8+ assert mem._module_host_mib(linear) >= 4+++def test_stream_kept_when_the_packed_weights_fit_the_pin_budget(monkeypatch):+ linear = _int8_linear(2048, 2048)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)+ monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: None)+ # Between the packed (~4 MiB) and logical bf16 (8 MiB) sizes.+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 6)+ kwargs = mem._torchao_group_offload_kwargs(+ linear, {"use_stream": True, "low_cpu_mem_usage": True}+ )+ assert kwargs["use_stream"] is True+ assert kwargs["low_cpu_mem_usage"] is False+++def test_storage_size_recurses_through_nested_wrappers():+ class Inner:+ def __init__(self, n):+ self._n = n++ def numel(self):+ return self._n++ def element_size(self):+ return 1++ class Wrapper:+ def __init__(self, *inner):+ self._inner = inner+ for i, t in enumerate(inner):+ setattr(self, f"t{i}", t)++ def __tensor_flatten__(self):+ return [f"t{i}" for i in range(len(self._inner))], None++ def numel(self):+ return 10**9++ def element_size(self):+ return 2++ nested = Wrapper(Wrapper(Inner(100), Inner(4)), Inner(7))+ assert mem._storage_nbytes(nested) == [100, 4, 7]+++ "env, capped, budget, keeps_stream",+ [+ ("1", True, None, True),+ ("1", False, 0, True),+ ("0", False, 10**7, False),+ ("", False, 10**7, True),+ ],+)+def test_runtime_pinning_honours_the_same_override_as_the_planner(+ monkeypatch, env, capped, budget, keeps_stream+):+ linear = _int8_linear(256, 256)+ monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, env)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: capped)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)+ monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: None)+ kwargs = mem._torchao_group_offload_kwargs(+ linear, {"use_stream": True, "low_cpu_mem_usage": True}+ )+ assert kwargs["use_stream"] is keeps_stream+++def test_several_torchao_denoisers_share_one_pin_budget(monkeypatch):+ """Ideogram 4 streams two torchao transformers: each fits the budget alone, together they do not."""+ first, second = _int8_linear(2048, 2048), _int8_linear(2048, 2048)+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)+ monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: None)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 6)+ base = {"use_stream": True, "low_cpu_mem_usage": True}+ pinned = [0]+ kept = [mem._torchao_group_offload_kwargs(m, dict(base), pinned) for m in (first, second)]+ assert kept[0]["low_cpu_mem_usage"] is False+ assert kept[1].get("low_cpu_mem_usage") is not False+ assert pinned[0] <= 6+++def test_every_group_offload_site_shares_the_pin_total():+ import inspect++ import re++ source = inspect.getsource(mem)+ calls = re.findall(r"(?<!def )_torchao_group_offload_kwargs\(([^)]*)\)", source)+ assert calls and all("pinned_mib" in args for args in calls)++[email protected]("fits_whole, raises", [(False, True), (True, False)])+def test_failed_group_setup_never_onloads_an_oversized_quantised_transformer(+ monkeypatch, fits_whole, raises+):+ """Streaming was the only placement the quantised transformer fits; model offload would OOM on first onload."""+ calls = []+ pipe = types.SimpleNamespace(+ enable_model_cpu_offload = lambda device = None: calls.append("model_offload")+ )+ monkeypatch.setattr(mem, "_apply_group_offload", lambda *a, **k: False)+ monkeypatch.setattr(mem, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ monkeypatch.setattr(mem, "_model_offload_fits_quantised", lambda plan: fits_whole)+ monkeypatch.setattr(mem, "keep_cpu_weights_on_offload", lambda *a, **k: 0)+ monkeypatch.setattr(mem, "_enable_vae_saver", lambda *a, **k: False)+ plan = _plan(OFFLOAD_GROUP)+ if raises:+ with pytest.raises(RuntimeError, match = "does not fit the GPU whole"):+ mem.apply_memory_plan(pipe, plan, device = "cuda")+ assert "model_offload" not in calls+ else:+ effective, _ = mem.apply_memory_plan(pipe, plan, device = "cuda")+ assert effective == OFFLOAD_MODEL+++def test_pipeline_host_mib_counts_only_host_resident_weights():+ torch = pytest.importorskip("torch")+ cpu = torch.nn.Linear(1024, 1024, bias = False) # 4 MiB fp32+ pipe = types.SimpleNamespace(components = {"transformer": cpu, "scheduler": object()})+ assert mem.pipeline_host_mib(pipe) == 4+ assert mem.pipeline_host_mib(None) == 0+ if torch.cuda.is_available():+ pipe.components["transformer"] = cpu.to("cuda")+ assert mem.pipeline_host_mib(pipe) == 0+++def test_reclaimable_host_ram_widens_the_pin_budget(monkeypatch):+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 4_000)+ plan = types.SimpleNamespace(+ estimates = {"model_dense_mib": 20_000, "companion_dense_mib": 8_000}+ )+ assert not mem._torchao_stream_pinnable(plan)+ assert mem._torchao_stream_pinnable(plan, 8_000)diff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.pyindex 11898f4dedc..7039e012e0b 100644--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py@@ -56,6 +56,16 @@ DENOISER_BYTES = 22_000 * MIBALL_BYTES = sum(size for _name, size in Z_IMAGE_FILES)[email protected](autouse = True)+def _unmeasured_torchao(monkeypatch):+ """Pin "no measured torchao" so the installed release does not decide the offload tiers."""+ from core.inference import diffusion_memory++ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: None)+ # Studio's diffusers pin, so tests that opt into a measured torchao do not depend on the runner.+ monkeypatch.setattr(diffusion_memory, "_installed_diffusers_version", lambda: (0, 40))++@pytest.fixture(autouse = True)def _safetensors_readable_regardless_of_the_installed_torchao(monkeypatch):"""Pin the torchao floor so planning tests ignore the installed release."""@@ -1029,6 +1039,50 @@ def test_an_artifact_plan_that_streams_the_transformer_still_declines(monkeypatcassert _settle(backend) == PIPELINE_SEED_DECLINED[email protected]("policy", ["group", "streaming"])+def test_an_artifact_plan_that_streams_the_transformer_seeds_on_a_measured_torchao(+ monkeypatch, policy+):+ from core.inference import diffusion_memory++ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)+ backend = _settle_backend(monkeypatch)+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ lambda *_a, **_k: types.SimpleNamespace(offload_policy = policy, stream_transformer = True),+ )+ assert _settle(backend) == "fp8"+++def test_a_resident_rung_still_beats_a_streamed_one(monkeypatch):+ from core.inference import diffusion_memory++ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)+ backend = _settle_backend_walking(+ monkeypatch, artifacts = ("int8", "fp8"), candidates = ("int8", "fp8")+ )+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ lambda *_a, **k: types.SimpleNamespace(+ offload_policy = "group" if k["transformer_resident_override_mib"] >= 31_000 else "none",+ stream_transformer = True,+ ),+ )+ assert _settle(backend) == "fp8"+ # no resident rung: the first streamed one is seeded+ backend = _settle_backend_walking(monkeypatch, artifacts = ("int8",), candidates = ("int8", "fp8"))+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ lambda *_a, **_k: types.SimpleNamespace(offload_policy = "group", stream_transformer = True),+ )+ assert _settle(backend) == "int8"++def test_a_seed_whose_load_plan_streams_only_the_encoders_is_kept(fake_runtime, monkeypatch):backend, spy = _load_backend(monkeypatch, offload = "group")real_plan = DiffusionBackend._plan_memory@@ -1711,3 +1765,29 @@ def test_the_kept_bf16_weights_are_placed_by_the_plan_that_proved_the_fit(assert spy.quantised == []assert placed == ["none"]++[email protected]("outgoing_host_mib, expected", [(0, False), (10_000, True)])+def test_the_streamed_seed_credits_host_ram_the_outgoing_pipeline_frees(+ monkeypatch, outgoing_host_mib, expected+):+ # The plan runs before the previous pipeline unloads, so its host weights must not veto the pin.+ from core.inference import diffusion_memory++ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))+ monkeypatch.setattr(diffusion_memory, "_pinned_memory_capped", lambda: False)+ monkeypatch.delenv(diffusion_memory.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(diffusion_memory, "_pin_budget_mib", lambda: 4_000)+ monkeypatch.setattr(dmod, "pipeline_host_mib", lambda pipe: outgoing_host_mib)+ backend = _settle_backend(monkeypatch)+ backend._state = types.SimpleNamespace(pipe = object())+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ lambda *_a, **_k: types.SimpleNamespace(+ offload_policy = "group",+ stream_transformer = True,+ estimates = {"model_dense_mib": 20_000, "companion_dense_mib": 8_000},+ ),+ )+ assert (_settle(backend) == "fp8") is expecteddiff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.tsindex 22edde8fe0f..a20daf2c5e0 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts@@ -1205,16 +1205,16 @@ for (const catalog of [IMAGE_CATALOG, VIDEO_CATALOG, AUDIO_CATALOG]) {}}-// Z-Image-Turbo needs 42.9 GB of card dense and 34.9 GB pre-quantised under the 70% rule.+// Z-Image-Turbo: 27.3 GB card dense (bf16, not the fp32 download), 19.3 GB pre-quantised, under the 70% rule.const zTurboId = "unsloth/Z-Image-Turbo";assert.equal(- curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, { gpuGb: 40, systemRamGb: 128 }),+ curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, { gpuGb: 24, systemRamGb: 128 }),false,);for (const schemes of [["fp8"], ["int8"]]) {assert.equal(curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, {- gpuGb: 40,+ gpuGb: 24,systemRamGb: 128,denseQuantSchemes: schemes,}),@@ -1224,12 +1224,16 @@ for (const schemes of [["fp8"], ["int8"]]) {}assert.equal(curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, {- gpuGb: 40,+ gpuGb: 24,systemRamGb: 128,denseQuantSchemes: [],}),false,);+assert.equal(+ curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, { gpuGb: 32, systemRamGb: 128 }),+ true,+);assert.equal(curatedArtifactFitsDevice("Qwen/Qwen-Image", IMAGE_CATALOG, {gpuGb: 64,@@ -1321,7 +1325,7 @@ const zTurboGroup = groupForRepoId(zTurboId, IMAGE_CATALOG);assert.ok(zTurboGroup);assert.equal(pickDefaultArtifact(zTurboGroup, {- gpuGb: 40,+ gpuGb: 24,systemRamGb: 128,isDownloaded: notDownloaded,}).format,@@ -1342,6 +1346,15 @@ assert.equal(systemRamGb: 128,denseQuantSchemes: ["fp8"],isDownloaded: notDownloaded,+ }).repoId,+ zTurboId,+);+assert.equal(+ pickDefaultArtifact(zTurboGroup, {+ gpuGb: 16,+ systemRamGb: 128,+ denseQuantSchemes: ["fp8"],+ isDownloaded: notDownloaded,}).format,"bnb-4bit",);diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.tsindex f71b306e62d..395aebbb812 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts@@ -178,7 +178,8 @@ export const IMAGE_CATALOG: CatalogGroup[] = [description: "Text-to-image",scope: "image",artifacts: [- bf16Mirror("Tongyi-MAI/Z-Image-Turbo", 30, {+ // Resident bf16 GiB, not the Hub's fp32 DiT: 11.5 DiT + 7.5 Qwen3-4B + 0.2 VAE.+ bf16Mirror("Tongyi-MAI/Z-Image-Turbo", 19.1, {totalParams: 6154908736,prequantRepo: "unsloth/Z-Image-Turbo-FP8",prequantSizeGb: { fp8: 5.86, int8: 5.86 },diff --git a/studio/frontend/tests/dense-quant-prequant-routing.test.ts b/studio/frontend/tests/dense-quant-prequant-routing.test.tsindex 56e3c58c4f5..178462a2670 100644--- a/studio/frontend/tests/dense-quant-prequant-routing.test.ts+++ b/studio/frontend/tests/dense-quant-prequant-routing.test.ts@@ -63,19 +63,19 @@ test("the catalog states the hosted checkpoint each official row would be fetchetest("the fit verdict is the quantised resident size on a host that runs a scheme", () => {assert.equal(- curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40)),+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24)),false,);assert.equal(- curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40, ["fp8"])),+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24, ["fp8"])),true,);assert.equal(- curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40, ["int8"])),+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24, ["int8"])),true,);assert.equal(- curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24, ["fp8"])),+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(16, ["fp8"])),false,);});@@ -118,7 +118,7 @@ test("the 2512 pick is judged by the checkpoint the backend seeds for it, not bytest("a host with no scheme, and a row with no hosted checkpoint, are unchanged", () => {assert.equal(- curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40, [])),+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24, [])),false,);for (const id of ["black-forest-labs/FLUX.1-dev", "stabilityai/sdxl-turbo"]) {@@ -134,20 +134,20 @@ test("the router sends a card that only fits the quantised form to the officialconst group = groupForRepoId(Z_TURBO, IMAGE_CATALOG);assert.ok(group);assert.equal(- pickDefaultArtifact(group, { ...onCard(40), isDownloaded: notDownloaded })+ pickDefaultArtifact(group, { ...onCard(24), isDownloaded: notDownloaded }).format,"bnb-4bit",);assert.equal(pickDefaultArtifact(group, {- ...onCard(40, ["fp8"]),+ ...onCard(24, ["fp8"]),isDownloaded: notDownloaded,}).repoId,Z_TURBO,);assert.equal(pickDefaultArtifact(group, {- ...onCard(24, ["fp8"]),+ ...onCard(16, ["fp8"]),isDownloaded: notDownloaded,}).format,"bnb-4bit",@@ -314,10 +314,10 @@ test("the over-budget text never shows the size below the budget it exceeds", ()assert.ok(budget);return curatedBudgetText(Math.round(fit.sizeGb), gpuGb, budget);};- // 24.40 GB rounds to 24, under the 24.1 budget, so it is shown rounded up.+ // 13.50 GB rounds to 13, under the 13.3 budget, so it is shown rounded up.assert.equal(- text(Z_TURBO, 34.5),- "Needs ~24.4GB for weights (budget: ~24.1GB, 70% of a 34.5GB GPU)",+ text(Z_TURBO, 19),+ "Needs ~13.5GB for weights (budget: ~13.3GB, 70% of a 19GB GPU)",);// A whole 4 GB against a 3.99 allowance: the budget is rounded down so 4.0 still reads as over.const whisper = curatedArtifactFit("unsloth/whisper-large-v3", AUDIO_CATALOG, {