Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep int8 / fp8 image transformers quantised when the load has to offload (#12287)

Daniel Hane477badf8125
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 2c1cbfa45d3..3faf0475f48 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -127,6 +127,12 @@ from .diffusion_memory import (
refine_memory_plan_for_components,
settled_snapshot_device_memory,
snapshot_device_memory,
+ _torchao_stream_pinnable,
+ pipeline_host_mib,
+ torchao_offload_plan,
+ torchao_scheme_streams,
+ torchao_streaming_plan,
+ torchao_survives_plan,
unified_memory_shortfall_message,
vae_tile_side,
vae_can_slice,
@@ -1626,6 +1632,66 @@ def _activation_guard_batch(chunks: Sequence[Sequence[Any]]) -> int:
return max((len(chunk) for chunk in chunks), default = 1)
+def _inplace_torchao_placement(
+ plan: Any,
+ scheme: Optional[str],
+ estimate: Any,
+ largest_companion_mib: Callable[[], Optional[int]],
+) -> tuple[Any, Optional[str]]:
+ """(placement, None) for the in-place torchao quant, or (``plan``, why not). Model offload checks LOADED encoders."""
+ if plan_keeps_transformer_resident(plan):
+ return plan, None
+ if plan.offload_policy == OFFLOAD_MODEL:
+ estimates = getattr(plan, "estimates", None) or {}
+ budget = int(estimates.get("safe_device_budget_mib") or 0)
+ overhead = int(estimates.get("runtime_headroom_mib") or 0) + int(
+ estimates.get("base_overhead_mib") or 0
+ )
+ if (
+ estimate is not None
+ and estimate.steady_transformer_mib + overhead <= budget
+ and (largest_companion_mib() or 0) <= budget
+ ):
+ return plan, None
+ if torchao_scheme_streams(scheme) and _torchao_stream_pinnable(plan):
+ return torchao_streaming_plan(plan), None
+ return plan, (
+ "whole-module offload onloads each component whole, and the quantised transformer or a text "
+ "encoder is not known to fit the device budget"
+ )
+ placed = torchao_offload_plan(plan, scheme)
+ if placed is not None:
+ return placed, None
+ return plan, (
+ f"'{plan.offload_policy}' offload streams the transformer through hooks torchao weights do not survive "
+ f"({scheme} on this torchao). Pin low_vram or a resident memory mode to combine the two"
+ )
+
+
+def _denoiser_hooked(pipe: Any) -> bool:
+ """Whether an offload hook moves a denoiser of ``pipe``; a CUDA graph over it would replay stale pointers."""
+ for name in ("transformer", "transformer_2", "unconditional_transformer", "unet"):
+ module = getattr(pipe, name, None)
+ if module is None:
+ continue
+ if getattr(module, "_hf_hook", None) is not None:
+ return True
+ registry = getattr(module, "_diffusers_hook", None)
+ hooks = getattr(registry, "hooks", None) or {}
+ if any("offload" in str(key) for key in hooks):
+ return True
+ return False
+
+
+def _torchao_render_needs_no_grad(state: Any) -> bool:
+ """inference_mode rejects moving torchao weights between devices, so offloaded quant renders use no_grad."""
+ if getattr(state, "offload_policy", OFFLOAD_NONE) == OFFLOAD_NONE:
+ return False
+ return bool(
+ getattr(state, "transformer_quant", None) or getattr(state, "text_encoder_quant", None)
+ )
+
+
def _memory_request_forces_offload(memory_mode: Optional[str], cpu_offload: bool) -> bool:
"""Whether this memory request offloads the transformer no matter what the weights measure.
``balanced`` and ``low_vram`` name their policy outright in ``resolve_offload_policy``, and
@@ -3410,7 +3476,13 @@ class DiffusionBackend:
except Exception: # noqa: BLE001 -- no lower rungs is just "no retry"
pass
declined = False
+ # An offloaded quantised rung beats bf16 shards; a later resident rung beats both.
+ offloaded_rung: Optional[str] = None
memory = snapshot_device_memory(target)
+ # Like the VRAM override below: the outgoing pipeline's host weights are gone before this one pins.
+ reclaimable_host_mib = pipeline_host_mib(
+ getattr(getattr(self, "_state", None), "pipe", None)
+ )
for rung in rungs:
source = denoiser_prequant_source(
fam,
@@ -3453,16 +3525,24 @@ class DiffusionBackend:
),
device_memory_override = replace(memory, free_mib = memory.total_mib),
)
- if not plan_keeps_transformer_resident(planned):
+ if not torchao_survives_plan(
+ planned, rung, reclaimable_host_mib = reclaimable_host_mib
+ ):
logger.info(
- "diffusion.denoiser_prequant: an artifact-sized plan for %s streams the "
- "denoiser on this card, and offload moves the denoiser via Module.to(), "
- "so the released shards are kept",
+ "diffusion.denoiser_prequant: an artifact-sized plan for %s places the "
+ "denoiser with '%s' offload, which its torchao weights do not survive "
+ "here, so the released shards are kept",
rung,
+ planned.offload_policy,
)
declined = True
continue
+ if not plan_keeps_transformer_resident(planned):
+ offloaded_rung = offloaded_rung or rung
+ continue
return rung
+ if offloaded_rung is not None:
+ return offloaded_rung
return PIPELINE_SEED_DECLINED if declined else None
except Exception as exc: # noqa: BLE001 -- an unanswerable probe keeps the released shards
logger.warning("diffusion.denoiser_prequant_plan_failed: %s", exc)
@@ -5153,17 +5233,22 @@ class DiffusionBackend:
fetch_base = fetch_base,
text_encoder_quant = text_encoder_quant,
)
- if seeded_plan is None or not plan_keeps_transformer_resident(seeded_plan):
- # Offload hooks use Module.to(), which torchao tensors reject, and live free
- # memory can undercut the CAPACITY the plan settled this against.
+ seeded_placement = (
+ None
+ if seeded_plan is None
+ else torchao_offload_plan(seeded_plan, pipeline_seed_scheme)
+ )
+ if seeded_placement is None:
+ # Live free memory can undercut the CAPACITY the plan settled this against.
logger.info(
- "diffusion.denoiser_prequant: an artifact-sized plan for %s offloads on "
- "this card, so the released denoiser is loaded instead",
+ "diffusion.denoiser_prequant: an artifact-sized plan for %s places the "
+ "denoiser where its torchao weights do not survive on this card, so the "
+ "released denoiser is loaded instead",
pipeline_seed_scheme,
)
pipeline_seed_scheme = None
else:
- plan = seeded_plan
+ plan = seeded_placement
# On unified memory the plan above is final (the quant re-plans below are CUDA-only) and its 'none'
# policy is a placement, not a fit. Refuse here, after the eviction above freed the previous pipeline
# and before any weight is materialised. A pipeline's weight term is cached SHARD bytes, which is a
@@ -6298,14 +6383,28 @@ class DiffusionBackend:
),
text_encoder_quant = text_encoder_quant,
)
- if plan_keeps_transformer_resident(replanned):
+ # Native schemes (incl. explicit int8 offloaded) keep the resident-only rule.
+ placed = (
+ (
+ replanned
+ if plan_keeps_transformer_resident(replanned)
+ else None
+ )
+ if native_scheme is not None
+ or native_offload_scheme is not None
+ else torchao_offload_plan(replanned, preview_scheme)
+ )
+ if placed is not None:
+ replanned = placed
logger.info(
- "diffusion.transformer_quant: %s fits resident (%d MiB "
- "steady, encoders streamed=%s); replacing the bf16 plan's "
- "'%s' offload",
+ "diffusion.transformer_quant: %s fits the '%s' placement "
+ "(%d MiB steady, denoiser streamed=%s, encoders streamed=%s); "
+ "replacing the bf16 plan's '%s' offload",
preview_scheme,
+ replanned.offload_policy,
estimate.steady_transformer_mib,
- replanned.offload_policy != OFFLOAD_NONE,
+ not plan_keeps_transformer_resident(replanned),
+ bool(getattr(replanned, "stream_text_encoders", False)),
plan.offload_policy,
)
plan = replanned
@@ -6322,34 +6421,16 @@ class DiffusionBackend:
native_scheme,
plan.offload_policy,
)
- # Group offload is WRONG for torchao: its stream cache aliases weights and swap_tensors fails when compiled.
- quant_budget = int(plan.estimates.get("safe_device_budget_mib") or 0)
- quant_overhead = int(
- plan.estimates.get("runtime_headroom_mib") or 0
- ) + int(plan.estimates.get("base_overhead_mib") or 0)
- if (
- not keeps_resident
- and native_scheme is None
- and (
- plan.offload_policy != OFFLOAD_MODEL
- or estimate is None
- or estimate.steady_transformer_mib + quant_overhead
- > quant_budget
- or (largest_streamable_companion_mib(pipe) or 0) > quant_budget
+ inplace_decline = None
+ if not keeps_resident and native_scheme is None:
+ plan, inplace_decline = _inplace_torchao_placement(
+ plan,
+ preview_scheme,
+ estimate,
+ lambda: largest_streamable_companion_mib(pipe),
)
- ):
- if plan.offload_policy == OFFLOAD_MODEL:
- transformer_quant_decline = (
- "whole-module offload onloads each component whole, and the "
- "quantised transformer or a text encoder is not known to fit "
- "the device budget"
- )
- else:
- transformer_quant_decline = (
- f"'{plan.offload_policy}' offload streams the transformer through "
- "hooks torchao weights do not survive. Pin low_vram or a resident "
- "memory mode to combine the two"
- )
+ if inplace_decline is not None:
+ transformer_quant_decline = inplace_decline
logger.info(
"diffusion.transformer_quant: skipped (%s)",
transformer_quant_decline,
@@ -6636,6 +6717,7 @@ class DiffusionBackend:
cache_active = cache_graph_break or cache_may_toggle,
cache_engaged = cache_graph_break,
offload_active = plan.offload_policy != OFFLOAD_NONE,
+ denoiser_offloaded = not plan_keeps_transformer_resident(plan),
logger = logger,
)
if vae_fp16:
@@ -6739,6 +6821,11 @@ class DiffusionBackend:
placement_device = target.torch_device,
logger = logger,
)
+ if speed_applied.get("cuda_graph") and _denoiser_hooked(pipe):
+ cuda_graph.uninstall_all(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())
+ pipe._unsloth_cuda_graphs = ()
+ pipe._unsloth_cuda_graph_reason = "offload active"
+ speed_applied["cuda_graph"] = False
# Per-control provenance for status. cpu_offload=False is the unset default, so only True is
# explicit.
@@ -8336,6 +8423,8 @@ class DiffusionBackend:
cache_active = cache_breaks_graph(state.transformer_cache) or state.cache_auto,
cache_engaged = cache_breaks_graph(state.transformer_cache),
offload_active = state.offload_policy != OFFLOAD_NONE,
+ denoiser_offloaded = state.offload_policy != OFFLOAD_NONE
+ and _denoiser_hooked(state.pipe),
logger = logger,
)
if getattr(getattr(state.pipe, "vae", None), "_unsloth_fp16_decode", False):
@@ -8936,12 +9025,11 @@ class DiffusionBackend:
gen.eta_seconds = None
try:
- # torchao aten.to fails torch's aliasing check under inference_mode; model offload moves weights.
+ # torchao aten.to fails torch's aliasing check under inference_mode once offloaded.
with (
(
torch.no_grad()
- if state.transformer_quant
- and state.offload_policy == OFFLOAD_MODEL
+ if _torchao_render_needs_no_grad(state)
else torch.inference_mode()
),
protect_ctx,
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index db864631521..43bed7f4738 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1002,6 +1002,234 @@ def plan_keeps_transformer_resident(plan: Any) -> bool:
return policy == OFFLOAD_GROUP and not bool(getattr(plan, "stream_transformer", True))
+# Oldest torchao measured bit-exact under streamed group offload; 0.17 int8 (v1, no copy stream) ran 14x slower.
+_TORCHAO_GROUP_OFFLOAD_MIN = {"int8": (0, 18), "fp8": (0, 17)}
+_TORCHAO_STREAM_SAFE_CLASSES = frozenset(("Int8Tensor", "Float8Tensor"))
+# Before 0.38 diffusers moved only the torchao wrapper, leaving quantised data on the host.
+_DIFFUSERS_TORCHAO_GROUP_OFFLOAD_MIN = (0, 38)
+_UNSET: Any = object()
+
+
+def _installed_version(package: str) -> Optional[tuple[int, int]]:
+ try:
+ import re
+ from importlib.metadata import version
+
+ major, minor = re.match(r"(\d+)\.(\d+)", version(package)).groups()
+ return int(major), int(minor)
+ except Exception: # noqa: BLE001 - absent / unreadable metadata
+ return None
+
+
[email protected]_cache(maxsize = 1)
+def _installed_torchao_version() -> Optional[tuple[int, int]]:
+ return _installed_version("torchao")
+
+
[email protected]_cache(maxsize = 1)
+def _installed_diffusers_version() -> Optional[tuple[int, int]]:
+ return _installed_version("diffusers")
+
+
+def _model_offload_fits_quantised(plan: Any) -> bool:
+ """Quantised denoiser and encoders (SUM stands in for the largest) must each fit the budget whole."""
+ try:
+ est = plan.estimates
+ budget = est.get("safe_device_budget_mib")
+ model = est.get("model_dense_mib")
+ companions = est.get("companion_dense_mib")
+ if budget is None or model is None or companions is None:
+ return False
+ denoiser = max(0, int(model) - int(companions))
+ overhead = int(est.get("runtime_headroom_mib") or 0) + int(
+ est.get("base_overhead_mib") or 0
+ )
+ encoders = est.get("text_encoder_dense_mib")
+ if encoders is None:
+ encoders = companions
+ return denoiser + overhead <= int(budget) and int(encoders) <= int(budget)
+ except Exception: # noqa: BLE001 - malformed plan: keep the resident rule
+ return False
+
+
+def torchao_offload_plan(
+ plan: Any,
+ scheme: Optional[str],
+ *,
+ torchao_version: Any = _UNSET,
+ reclaimable_host_mib: int = 0,
+) -> Optional[Any]:
+ """Placement a torchao ``scheme`` denoiser survives under ``plan``, or None (sequential offload: never measured).
+ ``reclaimable_host_mib``: host RAM the outgoing pipeline frees before this one pins."""
+ if plan_keeps_transformer_resident(plan):
+ return plan
+ policy = getattr(plan, "offload_policy", OFFLOAD_NONE)
+ if policy == OFFLOAD_MODEL and _model_offload_fits_quantised(plan):
+ return plan
+ if policy not in (OFFLOAD_MODEL, OFFLOAD_GROUP, OFFLOAD_STREAMING):
+ return None
+ if not torchao_scheme_streams(scheme, torchao_version = torchao_version):
+ return None
+ if not _torchao_stream_pinnable(plan, reclaimable_host_mib):
+ return None
+ if policy != OFFLOAD_MODEL:
+ return plan
+ return torchao_streaming_plan(plan)
+
+
+def _torchao_stream_pinnable(plan: Any, reclaimable_host_mib: int = 0) -> bool:
+ """Lazy pinning refuses torchao and the stream-free fallback ran ~35x slower, so require an up-front pin."""
+ forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()
+ if forced in ("0", "off", "false", "no"):
+ return False
+ if forced in ("1", "on", "true", "yes"):
+ return True
+ if _pinned_memory_capped():
+ return False
+ try:
+ est = plan.estimates
+ denoiser = int(est["model_dense_mib"]) - int(est["companion_dense_mib"])
+ except Exception: # noqa: BLE001 - an unsized plan cannot prove the pin fits
+ return False
+ budget = _pin_budget_mib()
+ return budget is not None and 0 <= denoiser <= budget + max(0, int(reclaimable_host_mib))
+
+
+def pipeline_host_mib(pipe: Any) -> int:
+ """Host MiB held by ``pipe``'s components, which an unload hands back."""
+ if pipe is None:
+ return 0
+ total = 0
+ try:
+ components = getattr(pipe, "components", None) or {}
+ for module in components.values():
+ if hasattr(module, "parameters"):
+ total += _module_host_mib_on_cpu(module)
+ except Exception: # noqa: BLE001 - unsizeable: credit nothing
+ return 0
+ return total
+
+
+def _module_host_mib_on_cpu(module: Any) -> int:
+ try:
+ seen: set[int] = set()
+ nbytes = 0
+ for tensor in (*module.parameters(), *module.buffers()):
+ if id(tensor) in seen or getattr(tensor, "device", None) is None:
+ continue
+ seen.add(id(tensor))
+ if tensor.device.type == "cpu":
+ nbytes += sum(_storage_nbytes(tensor))
+ return nbytes >> 20
+ except Exception: # noqa: BLE001
+ return 0
+
+
+def torchao_survives_plan(
+ plan: Any,
+ scheme: Optional[str],
+ *,
+ torchao_version: Any = _UNSET,
+ reclaimable_host_mib: int = 0,
+) -> bool:
+ return (
+ torchao_offload_plan(
+ plan,
+ scheme,
+ torchao_version = torchao_version,
+ reclaimable_host_mib = reclaimable_host_mib,
+ )
+ is not None
+ )
+
+
+def torchao_scheme_streams(scheme: Optional[str], *, torchao_version: Any = _UNSET) -> bool:
+ floor = _TORCHAO_GROUP_OFFLOAD_MIN.get(str(scheme))
+ if floor is None:
+ return False
+ diffusers_version = _installed_diffusers_version()
+ if diffusers_version is None or diffusers_version < _DIFFUSERS_TORCHAO_GROUP_OFFLOAD_MIN:
+ return False
+ version = _installed_torchao_version() if torchao_version is _UNSET else torchao_version
+ return version is not None and tuple(version)[:2] >= floor
+
+
+def torchao_streaming_plan(plan: Any) -> Any:
+ return replace(
+ plan,
+ offload_policy = OFFLOAD_STREAMING,
+ reasons = tuple(plan.reasons)
+ + (
+ "the quantised transformer or a text encoder does not fit the device budget whole; streaming "
+ "transformer blocks and text-encoder layers",
+ ),
+ )
+
+
+def _torchao_weight_classes(module: Any) -> set[str]:
+ classes: set[str] = set()
+ try:
+ for param in module.parameters():
+ for tensor in (param, getattr(param, "data", None)):
+ if tensor is not None and type(tensor).__module__.startswith("torchao"):
+ classes.add(type(tensor).__name__)
+ except Exception: # noqa: BLE001 - unreadable: report nothing, the caller keeps its kwargs
+ return set()
+ return classes
+
+
+def _torchao_group_offload_kwargs(
+ module: Any,
+ kwargs: dict[str, Any],
+ pinned_mib: Optional[list] = None,
+) -> dict[str, Any]:
+ """Group-offload kwargs a torchao ``module`` survives. Frozen first (swap_tensors on a requires_grad torchao weight
+ hits an unimplemented ``aten.view``); the stream needs an up-front pin within ``pinned_mib``, a shared running total."""
+ classes = _torchao_weight_classes(module)
+ if not classes:
+ return kwargs
+ _freeze_torchao_weights(module)
+ install_group_offload_torchao_swap_retry()
+ if not kwargs.get("use_stream"):
+ return kwargs
+ if classes <= _TORCHAO_STREAM_SAFE_CLASSES:
+ if not kwargs.get("low_cpu_mem_usage"):
+ return kwargs
+ # Same override the planner's _torchao_stream_pinnable honoured, so the two cannot disagree.
+ forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()
+ if forced in ("1", "on", "true", "yes"):
+ return {**kwargs, "low_cpu_mem_usage": False}
+ budget = (
+ None
+ if forced in ("0", "off", "false", "no") or _pinned_memory_capped()
+ else _pin_budget_mib()
+ )
+ need = _module_host_mib(module)
+ already = pinned_mib[0] if pinned_mib else 0
+ if budget is not None and already + need <= budget:
+ if pinned_mib is not None:
+ pinned_mib[0] = already + need
+ return {**kwargs, "low_cpu_mem_usage": False}
+ safe = {
+ k: v
+ for k, v in kwargs.items()
+ if k not in ("non_blocking", "record_stream", "low_cpu_mem_usage")
+ }
+ safe["use_stream"] = False
+ return safe
+
+
+def _freeze_torchao_weights(module: Any) -> None:
+ try:
+ for param in module.parameters():
+ if type(param).__module__.startswith("torchao") and getattr(
+ param, "requires_grad", False
+ ):
+ param.requires_grad_(False)
+ except Exception: # noqa: BLE001 - an inference-only flag: leave the module as it is
+ pass
+
+
def _holds_torchao_weights(module: Any) -> bool:
"""Whether any parameter of ``module`` is a torchao tensor subclass (GGUF and native int8 are not)."""
try:
@@ -1514,6 +1742,12 @@ def apply_memory_plan(
"the text encoder could not be streamed beside the resident quantised "
"transformer, and whole-module offload cannot move torchao weights"
)
+ if _pipe_denoisers_hold_torchao(pipe) and not _model_offload_fits_quantised(plan):
+ # Whole-module offload would onload the quantised transformer whole, which is what streaming avoided.
+ raise RuntimeError(
+ "group offloading could not be set up for the quantised transformer, and it does not "
+ "fit the GPU whole for whole-module offload"
+ )
_fallback_to_model_offload()
policy = OFFLOAD_MODEL
elif policy == OFFLOAD_STREAMING:
@@ -1599,14 +1833,28 @@ def _module_host_mib(module: Any) -> int:
if id(tensor) in seen:
continue
seen.add(id(tensor))
- nbytes = int(tensor.numel()) * int(tensor.element_size())
- if nbytes > 0:
- total += 1 << (nbytes - 1).bit_length()
+ for nbytes in _storage_nbytes(tensor):
+ if nbytes > 0:
+ total += 1 << (nbytes - 1).bit_length()
return total // (1024 * 1024)
except Exception: # noqa: BLE001 - an unsizeable module is priced as nothing to pin
return 0
+def _storage_nbytes(tensor: Any, depth: int = 0) -> list[int]:
+ """Bytes per allocation; torchao subclasses report their logical bf16 size, so size the packed inner tensors."""
+ flatten = getattr(type(tensor), "__tensor_flatten__", None)
+ if flatten is not None and depth < 4:
+ try:
+ names, _ctx = tensor.__tensor_flatten__()
+ inner = [getattr(tensor, name, None) for name in names]
+ if inner and all(t is not None for t in inner):
+ return [n for t in inner for n in _storage_nbytes(t, depth + 1)]
+ except Exception: # noqa: BLE001 - fall back to the logical size, which only over-counts
+ pass
+ return [int(tensor.numel()) * int(tensor.element_size())]
+
+
def _pin_budget_mib() -> Optional[int]:
"""Pinnable host MiB leaving ``max(4 GiB, 15%)`` free, or None if unreadable."""
total, _available = _system_memory_mib()
@@ -1685,6 +1933,33 @@ def _streamed_pin_plan(
return pin_transformer, pin_encoders
+def install_group_offload_torchao_swap_retry() -> bool:
+ """``swap_tensors`` refuses a weakref'd tensor; retry once after gc so uncollected compile garbage does not fail."""
+ try:
+ from diffusers.hooks import group_offloading as go
+ except Exception: # noqa: BLE001 - no group offload in this diffusers
+ return False
+ original = getattr(go, "_swap_torchao_tensor", None)
+ if original is None or getattr(original, "_unsloth_swap_retry", False):
+ return False
+
+ @functools.wraps(original)
+ def _swap_torchao_tensor(param: Any, source: Any) -> None:
+ try:
+ original(param, source)
+ except RuntimeError as exc:
+ if "weakref" not in str(exc):
+ raise
+ import gc
+
+ gc.collect()
+ original(param, source)
+
+ _swap_torchao_tensor._unsloth_swap_retry = True
+ go._swap_torchao_tensor = _swap_torchao_tensor
+ return True
+
+
BACKGROUND_PIN_ENV = "UNSLOTH_DIFFUSION_BACKGROUND_PIN"
_BG_PIN_ATTR = "_unsloth_background_pin"
_PENDING_PINS_ATTR = "_unsloth_background_pins"
@@ -2074,12 +2349,25 @@ def _apply_group_offload(
continue
if isinstance(comp, torch.nn.Module):
comp.to(onload)
+ # Encoders the plan pins count against the same budget as any torchao denoiser pinned below.
+ pinned_mib = [
+ sum(_module_host_mib(m) for m in streamed_encoders.values())
+ if stream_text_encoders and pin_streamed[1]
+ else 0
+ ]
for module in streamed.values():
- if defer and not gkwargs.get("low_cpu_mem_usage", False):
+ # torchao weights need their up-front pin (lazy pinning refuses them), so they never defer.
+ if (
+ defer
+ and not gkwargs.get("low_cpu_mem_usage", False)
+ and not _torchao_weight_classes(module)
+ ):
apply_group_offloading(module, **{**gkwargs, "low_cpu_mem_usage": True})
_defer_pinning(pipe, module, onload, logger)
else:
- apply_group_offloading(module, **gkwargs)
+ apply_group_offloading(
+ module, **_torchao_group_offload_kwargs(module, gkwargs, pinned_mib)
+ )
installed += 1
# The encoders come AFTER the DiTs and are applied one by one, each failure absorbed. A text encoder is a far
# less well-trodden target for block-level group offloading than a DiT (a family whose encoder exposes no
@@ -2121,7 +2409,9 @@ def _apply_group_offload(
dkwargs = dict(gkwargs)
if "low_cpu_mem_usage" in _params and use_stream:
dkwargs["low_cpu_mem_usage"] = True
- apply_group_offloading(dit, **dkwargs)
+ apply_group_offloading(
+ dit, **_torchao_group_offload_kwargs(dit, dkwargs, pinned_mib)
+ )
installed += 1
transformer_demoted = True
if logger is not None:
@@ -2636,6 +2926,7 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:
if isinstance(component, torch.nn.Module):
component.to(onload)
+ pinned_mib = [0]
for module, offload_type in streamed.values():
kwargs: dict[str, Any] = {
"onload_device": onload,
@@ -2652,7 +2943,9 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:
kwargs["record_stream"] = False
if use_stream and "low_cpu_mem_usage" in params:
kwargs["low_cpu_mem_usage"] = True
- apply_group_offloading(module, **kwargs)
+ apply_group_offloading(
+ module, **_torchao_group_offload_kwargs(module, kwargs, pinned_mib)
+ )
installed += 1
if offload_type == "leaf_level":
_pin_vision_embedding_device(module)
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 0836c85ea1f..3b61dda7258 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -421,6 +421,7 @@ def apply_speed_optims(
offload_active: bool = False,
cuda_graph_default: bool = True,
cache_engaged: Optional[bool] = None,
+ denoiser_offloaded: Optional[bool] = None,
logger: Any = None,
) -> dict[str, bool]:
"""Apply the opt-in speed optims for ``speed_mode`` to a built pipeline, BEFORE placement /
@@ -428,6 +429,7 @@ def apply_speed_optims(
``offload_active`` (offload policy != none) installs ``@torch.compiler.disable``d onload hooks,
so the compile must drop ``fullgraph`` (like an active step cache) or it crashes at step 1.
+ ``denoiser_offloaded`` (None = ``offload_active``) limits the CUDA-graph refusal to a moved denoiser.
``cuda_graph_default`` is what the CUDA-graph arm assumes for a family that declares nothing:
True on the image backend, False on video, where ``supports_cuda_graph`` opts in.
@@ -560,7 +562,9 @@ def apply_speed_optims(
target,
family = family,
pipe = pipe,
- offload_active = offload_active,
+ offload_active = offload_active
+ if denoiser_offloaded is None
+ else bool(denoiser_offloaded),
cache_active = cache_active if cache_engaged is None else bool(cache_engaged),
speed_mode = mode,
family_default = cuda_graph_default,
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 4f5c40477b2..9f3b88dc171 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -54,6 +54,16 @@ from core.inference.diffusion_families import (
)
[email protected](autouse = True)
+def _unmeasured_torchao(monkeypatch):
+ """Pin "no measured torchao" so the installed release does not decide the offload tiers."""
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: None)
+ # Studio's diffusers pin, so tests that opt into a measured torchao do not depend on the runner.
+ monkeypatch.setattr(diffusion_memory, "_installed_diffusers_version", lambda: (0, 40))
+
+
# Pure family helpers
@@ -12163,7 +12173,8 @@ def test_a_pipeline_pick_quantises_under_whole_module_offload(fake_runtime, tmp_
@pytest.mark.parametrize(
- ("offload_policy", "expected"), [("none", "inference_mode"), ("model", "no_grad")]
+ ("offload_policy", "expected"),
+ [("none", "inference_mode"), ("model", "no_grad"), ("group", "no_grad")],
)
def test_an_offloaded_quantised_transformer_renders_outside_inference_mode(
fake_runtime, tmp_path, monkeypatch, offload_policy, expected
@@ -12171,6 +12182,10 @@ def test_an_offloaded_quantised_transformer_renders_outside_inference_mode(
"""torchao tensors cannot change device under inference_mode, so offloaded quant renders use no_grad."""
import torch
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))
+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)
backend = DiffusionBackend()
_stub_pipeline_dense_quant(backend, monkeypatch)
monkeypatch.setattr(DiffusionBackend, "_plan_memory", _offload_plan(offload_policy))
@@ -12290,6 +12305,63 @@ def test_a_pipeline_pick_stays_dense_when_the_quantised_transformer_exceeds_the_
backend.unload()
[email protected]("torchao_version", [(0, 17), (0, 18)])
+def test_a_pipeline_pick_quantises_under_streamed_group_offload_on_a_measured_torchao(
+ fake_runtime, tmp_path, monkeypatch, torchao_version
+):
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: torchao_version)
+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _offload_plan("group"))
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert len(calls) == 1
+ assert status["transformer_quant"] == "fp8"
+ backend.unload()
+
+
+ ("budget_mib", "runtime_headroom_mib", "companion_mib"),
+ [(1, 0, None), (1_000_000, 1_000_000, None), (1_000_000, 0, 2_000_000)],
+)
+def test_a_quantised_transformer_too_big_to_onload_whole_streams_instead(
+ fake_runtime, tmp_path, monkeypatch, budget_mib, runtime_headroom_mib, companion_mib
+):
+ from core.inference import diffusion as dmod
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))
+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)
+ # diffusion.py binds its own reference; without this the host's real pin budget decides.
+ monkeypatch.setattr(dmod, "_torchao_stream_pinnable", lambda plan, *_a: True)
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ monkeypatch.setattr(dmod, "largest_streamable_companion_mib", lambda pipe: companion_mib)
+ placed: list = []
+
+ def _apply(pipe, plan, **_kwargs):
+ placed.append(plan.offload_policy)
+ return plan.offload_policy, plan.vae_tiling
+
+ monkeypatch.setattr(dmod, "apply_memory_plan", _apply)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ _offload_plan("model", budget_mib = budget_mib, runtime_headroom_mib = runtime_headroom_mib),
+ )
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert len(calls) == 1
+ assert status["transformer_quant"] == "fp8"
+ assert placed == ["streaming"] and status["offload_policy"] == "streaming"
+ backend.unload()
+
+
def test_an_offloaded_pipeline_replans_against_the_quantised_size(
fake_runtime, tmp_path, monkeypatch
):
@@ -13404,6 +13476,52 @@ def test_the_act_kill_switch_keeps_nvidia_offload_native_but_weight_only(
backend.unload()
+def test_an_explicit_fp8_under_group_offload_engages_on_a_measured_torchao(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))
+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)
+ backend = DiffusionBackend()
+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch, engages = "fp8")
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ memory_mode = "balanced",
+ _base_local_dir = str(tmp_path),
+ )
+ assert len(calls) == 1 and "offload" not in calls[0] and reasons == []
+ assert status["transformer_quant"] == "fp8"
+ backend.unload()
+
+
+ "memory",
+ [{"memory_mode": "balanced"}, {"memory_mode": "low_vram"}, {"cpu_offload": True}],
+)
+def test_an_explicit_int8_under_offload_stays_native_on_a_measured_torchao(
+ fake_runtime, tmp_path, monkeypatch, memory
+):
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))
+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)
+ backend = DiffusionBackend()
+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "int8",
+ _base_local_dir = str(tmp_path),
+ **memory,
+ )
+ assert len(calls) == 1 and calls[0]["offload"] is True
+ assert status["transformer_quant"] == "int8" and reasons == ["int8"]
+ backend.unload()
+
+
@pytest.mark.parametrize("fallback", ["0", "1"])
def test_an_explicit_fp8_under_offload_on_nvidia_is_still_declined(
fake_runtime, tmp_path, monkeypatch, fallback
diff --git a/studio/backend/tests/test_diffusion_offload_precision.py b/studio/backend/tests/test_diffusion_offload_precision.py
new file mode 100644
index 00000000000..3abd8e186ca
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_offload_precision.py
@@ -0,0 +1,799 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A torchao int8 / fp8 denoiser stays quantised when the image plan offloads it."""
+
+from __future__ import annotations
+
+import types
+from dataclasses import replace
+
+import pytest
+
+from core.inference import diffusion as dmod
+from core.inference import diffusion_memory as mem
+from core.inference.diffusion_denoiser_prequant import PIPELINE_SEED_DECLINED
+from core.inference.diffusion_memory import (
+ OFFLOAD_GROUP,
+ OFFLOAD_MODEL,
+ OFFLOAD_NONE,
+ OFFLOAD_SEQUENTIAL,
+ OFFLOAD_STREAMING,
+ DeviceMemory,
+ plan_diffusion_memory,
+ torchao_offload_plan,
+ torchao_survives_plan,
+)
+
+T017, T018, T016 = (0, 17), (0, 18), (0, 16)
+
+
[email protected](autouse = True)
+def _roomy_host(monkeypatch):
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 10**7)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ # Studio's diffusers pin; a runner without diffusers would otherwise refuse every streamed tier.
+ monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: (0, 40))
+
+
+def _plan(
+ policy,
+ *,
+ stream_transformer = True,
+ fits_model_offload = True,
+):
+ device = DeviceMemory("cuda", "cuda:0", "discrete_vram", 15_000, 16_384)
+ plan = plan_diffusion_memory(
+ target = types.SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = device,
+ model_dense_mib = 12_000,
+ runtime_headroom_mib = 2_000,
+ companion_dense_mib = 4_000,
+ text_encoder_dense_mib = 3_800,
+ )
+ estimates = dict(plan.estimates)
+ if not fits_model_offload:
+ estimates["model_dense_mib"] = 40_000
+ return replace(
+ plan,
+ offload_policy = policy,
+ stream_transformer = stream_transformer,
+ estimates = estimates,
+ )
+
+
+PLACEMENTS = {
+ "resident": _plan(OFFLOAD_NONE),
+ "group_dit_resident": _plan(OFFLOAD_GROUP, stream_transformer = False),
+ "group_dit_streamed": _plan(OFFLOAD_GROUP),
+ "model_fits": _plan(OFFLOAD_MODEL),
+ "model_too_big": _plan(OFFLOAD_MODEL, fits_model_offload = False),
+ "streaming": _plan(OFFLOAD_STREAMING),
+ "sequential": _plan(OFFLOAD_SEQUENTIAL),
+}
+
+# placement -> scheme -> torchao version -> (survives, policy it runs on)
+SURVIVAL = {
+ "resident": {
+ s: {v: (True, OFFLOAD_NONE) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")
+ },
+ "group_dit_resident": {
+ s: {v: (True, OFFLOAD_GROUP) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")
+ },
+ "group_dit_streamed": {
+ "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_GROUP)},
+ "fp8": {T016: (False, None), T017: (True, OFFLOAD_GROUP), T018: (True, OFFLOAD_GROUP)},
+ "nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},
+ },
+ "model_fits": {
+ s: {v: (True, OFFLOAD_MODEL) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")
+ },
+ "model_too_big": {
+ "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_STREAMING)},
+ "fp8": {
+ T016: (False, None),
+ T017: (True, OFFLOAD_STREAMING),
+ T018: (True, OFFLOAD_STREAMING),
+ },
+ "nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},
+ },
+ "streaming": {
+ "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_STREAMING)},
+ "fp8": {
+ T016: (False, None),
+ T017: (True, OFFLOAD_STREAMING),
+ T018: (True, OFFLOAD_STREAMING),
+ },
+ "nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},
+ },
+ "sequential": {
+ s: {v: (False, None) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")
+ },
+}
+CASES = [
+ (placement, scheme, version, *expected)
+ for placement, by_scheme in SURVIVAL.items()
+ for scheme, by_version in by_scheme.items()
+ for version, expected in by_version.items()
+]
+
+
[email protected]("placement, scheme, version, survives, policy", CASES)
+def test_survival_table(placement, scheme, version, survives, policy):
+ plan = PLACEMENTS[placement]
+ assert torchao_survives_plan(plan, scheme, torchao_version = version) is survives
+ placed = torchao_offload_plan(plan, scheme, torchao_version = version)
+ assert (placed.offload_policy if placed is not None else None) == policy
+
+
+ "diffusers_version, streams",
+ [((0, 36), False), ((0, 37), False), ((0, 38), True), (None, False)],
+)
+def test_diffusers_before_the_torchao_swap_keeps_the_resident_rule(
+ monkeypatch, diffusers_version, streams
+):
+ monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: diffusers_version)
+ assert (
+ torchao_survives_plan(PLACEMENTS["group_dit_streamed"], "int8", torchao_version = T018)
+ is streams
+ )
+ assert torchao_survives_plan(PLACEMENTS["group_dit_resident"], "int8", torchao_version = T018)
+
+
+def test_swap_retry_collects_once_then_gives_up(monkeypatch):
+ import sys
+
+ calls: list = []
+ failures = {"n": 1}
+
+ def _swap(param, source):
+ calls.append(param)
+ if failures["n"] > 0:
+ failures["n"] -= 1
+ raise RuntimeError("Cannot swap t1 because it has weakref associated with it")
+
+ go = types.SimpleNamespace(_swap_torchao_tensor = _swap)
+ hooks = types.ModuleType("diffusers.hooks")
+ hooks.group_offloading = go
+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)
+ monkeypatch.setitem(sys.modules, "diffusers.hooks.group_offloading", go)
+ assert mem.install_group_offload_torchao_swap_retry()
+ assert not mem.install_group_offload_torchao_swap_retry()
+ go._swap_torchao_tensor("p", "s")
+ assert calls == ["p", "p"]
+ failures["n"] = 2
+ with pytest.raises(RuntimeError, match = "weakref"):
+ go._swap_torchao_tensor("p", "s")
+
+
+ "capped, budget, env, streams",
+ [
+ (False, 10**7, None, True),
+ (True, 10**7, None, False),
+ (False, 7_999, None, False),
+ (False, None, None, False),
+ (False, 10**7, "0", False),
+ (True, 0, "1", True),
+ ],
+)
+def test_a_streamed_torchao_denoiser_needs_its_pin(monkeypatch, capped, budget, env, streams):
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: capped)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)
+ if env is not None:
+ monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, env)
+ for name in ("group_dit_streamed", "streaming", "model_too_big"):
+ assert (
+ torchao_survives_plan(PLACEMENTS[name], "int8", torchao_version = T018) is streams
+ ), name
+ assert torchao_survives_plan(PLACEMENTS["group_dit_resident"], "int8", torchao_version = T018)
+ assert torchao_survives_plan(PLACEMENTS["model_fits"], "int8", torchao_version = T018)
+
+
+def test_no_torchao_keeps_the_resident_rule():
+ assert not torchao_survives_plan(PLACEMENTS["group_dit_streamed"], "int8", torchao_version = None)
+ assert torchao_survives_plan(PLACEMENTS["group_dit_resident"], "int8", torchao_version = None)
+
+
+def test_never_moves_still_means_resident_only():
+ keeps = mem.plan_keeps_transformer_resident
+ assert keeps(PLACEMENTS["resident"]) and keeps(PLACEMENTS["group_dit_resident"])
+ for name in ("group_dit_streamed", "model_fits", "streaming"):
+ assert not keeps(PLACEMENTS[name]), name
+
+
+class _Weight:
+ def __init__(
+ self,
+ cls_name: str,
+ module: str = "torchao.quantization.fake",
+ ):
+ self.data = None
+ self.__class__ = type(cls_name, (_Weight,), {"__module__": module})
+
+
+class _Module:
+ def __init__(
+ self,
+ *class_names: str,
+ plain: bool = False,
+ ):
+ self._params = [_Weight(name) for name in class_names]
+ if plain:
+ self._params.append(_Weight("Parameter", module = "torch.nn.parameter"))
+
+ def parameters(self):
+ return iter(self._params)
+
+
+STREAM_KW = {
+ "onload_device": "cuda",
+ "use_stream": True,
+ "non_blocking": True,
+ "record_stream": True,
+}
+NO_STREAM = {"onload_device": "cuda", "use_stream": False}
+
+
+ "classes, low_cpu_mem_usage, pin_budget, expected",
+ [
+ (("Int8Tensor",), False, 10**7, "as is"),
+ (("Float8Tensor",), False, 10**7, "as is"),
+ (("Int8Tensor", "Float8Tensor"), False, 10**7, "as is"),
+ (("Int8Tensor",), True, 10**7, "pinned"),
+ (("Float8Tensor",), True, 10**7, "pinned"),
+ (("Int8Tensor",), True, None, "no stream"),
+ # torchao 0.17 default int8: no aten.is_pinned, so no copy stream
+ (("LinearActivationQuantizedTensor",), False, 10**7, "no stream"),
+ ],
+)
+def test_group_offload_kwargs_per_weight_class(
+ monkeypatch, classes, low_cpu_mem_usage, pin_budget, expected
+):
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: pin_budget)
+ kwargs = {**STREAM_KW, "low_cpu_mem_usage": low_cpu_mem_usage}
+ out = mem._torchao_group_offload_kwargs(_Module(*classes, plain = True), kwargs)
+ if expected == "as is":
+ assert out is kwargs
+ elif expected == "pinned":
+ assert out == {**STREAM_KW, "low_cpu_mem_usage": False}
+ else:
+ assert out == NO_STREAM
+
+
+def test_group_offload_kwargs_leave_dense_and_stream_free_modules_alone():
+ dense = {**STREAM_KW, "low_cpu_mem_usage": True}
+ assert mem._torchao_group_offload_kwargs(_Module(plain = True), dense) is dense
+ assert mem._torchao_group_offload_kwargs(_Module("Int8Tensor"), NO_STREAM) is NO_STREAM
+
+
+ "policy, transformer_quant, text_encoder_quant, no_grad",
+ [
+ (OFFLOAD_NONE, "int8", None, False),
+ (OFFLOAD_NONE, "int8", "fp8", False),
+ (OFFLOAD_MODEL, "int8", None, True),
+ (OFFLOAD_GROUP, "int8", None, True),
+ (OFFLOAD_GROUP, "fp8", None, True),
+ (OFFLOAD_STREAMING, "int8", None, True),
+ (OFFLOAD_GROUP, None, "fp8", True),
+ (OFFLOAD_GROUP, None, None, False),
+ (OFFLOAD_MODEL, None, None, False),
+ ],
+)
+def test_render_grad_mode(policy, transformer_quant, text_encoder_quant, no_grad):
+ state = types.SimpleNamespace(
+ offload_policy = policy,
+ transformer_quant = transformer_quant,
+ text_encoder_quant = text_encoder_quant,
+ )
+ assert dmod._torchao_render_needs_no_grad(state) is no_grad
+
+
+def test_render_uses_the_widened_switch():
+ import inspect
+
+ source = inspect.getsource(dmod.DiffusionBackend)
+ assert "if _torchao_render_needs_no_grad(state)" in source
+ assert "and state.offload_policy == OFFLOAD_MODEL" not in source
+
+
+def _estimate(steady):
+ return types.SimpleNamespace(steady_transformer_mib = steady)
+
+
+ "placement, estimate_mib, largest_te_mib, version, policy, declined",
+ [
+ ("resident", 6_000, 4_000, T018, OFFLOAD_NONE, False),
+ ("group_dit_streamed", 6_000, 4_000, T018, OFFLOAD_GROUP, False),
+ ("group_dit_streamed", 6_000, 4_000, T016, OFFLOAD_GROUP, True),
+ ("model_fits", 6_000, 4_000, T018, OFFLOAD_MODEL, False),
+ # quantised denoiser or encoder too big to onload whole: stream instead of falling back to bf16
+ ("model_fits", 60_000, 4_000, T018, OFFLOAD_STREAMING, False),
+ ("model_fits", 6_000, 60_000, T018, OFFLOAD_STREAMING, False),
+ ("model_fits", 60_000, 4_000, T016, OFFLOAD_MODEL, True),
+ ("sequential", 6_000, 4_000, T018, OFFLOAD_SEQUENTIAL, True),
+ ],
+)
+def test_inplace_quant_placement(
+ monkeypatch, placement, estimate_mib, largest_te_mib, version, policy, declined
+):
+ monkeypatch.setattr(mem, "_installed_torchao_version", lambda: version)
+ plan, why = dmod._inplace_torchao_placement(
+ PLACEMENTS[placement], "int8", _estimate(estimate_mib), lambda: largest_te_mib
+ )
+ assert plan.offload_policy == policy
+ assert (why is not None) is declined
+
+
[email protected]("estimate_mib, largest_te_mib", [(60_000, 4_000), (6_000, 60_000)])
+def test_inplace_quant_declines_streaming_it_cannot_pin(monkeypatch, estimate_mib, largest_te_mib):
+ """Unpinnable (Windows, capped host RAM) streaming runs without a copy stream, ~35x slower: decline it."""
+ monkeypatch.setattr(mem, "_installed_torchao_version", lambda: T018)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 0)
+ plan, why = dmod._inplace_torchao_placement(
+ PLACEMENTS["model_fits"], "int8", _estimate(estimate_mib), lambda: largest_te_mib
+ )
+ assert plan.offload_policy == OFFLOAD_MODEL
+ assert why is not None
+
+
+def test_inplace_quant_no_longer_calls_group_offload_wrong():
+ import inspect
+ source = inspect.getsource(dmod)
+ assert "Group offload is WRONG for torchao" not in source
+
+
+GIB = 1024
+
+
+def planner(monkeypatch):
+ import torch
+
+ from core.inference import diffusion_auto_policy as ap
+ from core.inference import diffusion_transformer_quant as tq
+
+ monkeypatch.setattr(mem, "_installed_torchao_version", lambda: T018)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *_a, **_k: (12, 0))
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+
+ def _supported(
+ scheme,
+ device = None,
+ unproven_ok = False,
+ ):
+ return scheme in ("int8", "fp8")
+
+ monkeypatch.setattr(tq, "_scheme_supported", _supported)
+ monkeypatch.setattr(tq, "_is_consumer_gpu", lambda *_a, **_k: True)
+ monkeypatch.setattr(ap, "_hf_cache_free_mib", lambda: 10**7)
+ card: dict = {}
+ for module in (dmod, mem):
+ for fn in (
+ "snapshot_device_memory",
+ "settled_snapshot_device_memory",
+ "reclaimable_snapshot_device_memory",
+ ):
+ if hasattr(module, fn):
+ monkeypatch.setattr(module, fn, lambda *_a, **_k: card["memory"])
+ target = types.SimpleNamespace(
+ device = "cuda",
+ dtype = torch.bfloat16,
+ backend = "cuda",
+ supports_model_cpu_offload = True,
+ supports_default_torch_compile = True,
+ ordinal = None,
+ capability = (12, 0),
+ )
+ backend = dmod.DiffusionBackend.__new__(dmod.DiffusionBackend)
+ backend._resolve_device_target = lambda *_a, **_k: target
+ backend._target_for_ordinal = lambda *_a, **_k: target
+ backend._cache_bytes = lambda *_a, **_k: 0
+ backend._released_transformer_cached = lambda *_a, **_k: False
+
+ def pick(family_name: str, base: str, gib: int):
+ from core.inference.diffusion_families import _FAMILIES
+ from core.inference.diffusion_transformer_quant import select_transformer_quant_scheme
+
+ fam = next(f for f in _FAMILIES if f.name == family_name)
+ card["memory"] = DeviceMemory("cuda", "cuda", "discrete_vram", gib * GIB - 600, gib * GIB)
+ seed = backend._pipeline_planned_denoiser_scheme(
+ fam,
+ base = base,
+ kind = "pipeline",
+ transformer_quant = None,
+ speed_mode = None,
+ repo_id = base,
+ fetch_base = base,
+ )
+ if seed not in (None, PIPELINE_SEED_DECLINED):
+ seeded = backend._seeded_pipeline_plan(
+ seed,
+ target,
+ base,
+ fam,
+ None,
+ False,
+ repo_id = base,
+ base_local_dir = None,
+ fetch_base = base,
+ )
+ placed = torchao_offload_plan(seeded, seed)
+ assert placed is not None
+ return f"hosted {seed}", placed.offload_policy
+ plan = backend._bf16_table_plan(
+ target, fam, base, None, False, kind = "pipeline", repo_id = base, fetch_base = base
+ )
+ if dmod._auto_quant_eager_reason(fam, plan, None, "pipeline") is not None:
+ return "bf16", plan.offload_policy
+ scheme = select_transformer_quant_scheme(target, "auto", family = fam.name, base_repo = base)
+ estimate = ap.estimate_dense_quant(fam, scheme, base_repo = base)
+ if plan.offload_policy != OFFLOAD_NONE:
+ replanned = backend._plan_memory(
+ target,
+ None,
+ base,
+ fam,
+ None,
+ False,
+ kind = "pipeline",
+ repo_id = base,
+ fetch_base = base,
+ transformer_resident_override_mib = estimate.steady_transformer_mib,
+ **backend._candidate_companion_overrides(estimate, fam, base, target, None),
+ )
+ placed = torchao_offload_plan(replanned, scheme)
+ if placed is not None:
+ plan = placed
+ largest_te = int(ap.family_bf16_components_gb(fam, base)[1] * 1000**3 / 1024**2)
+ plan, why = dmod._inplace_torchao_placement(plan, scheme, estimate, lambda: largest_te)
+ return ("bf16" if why else f"on-the-fly {scheme}"), plan.offload_policy
+
+ return pick
+
+
+FAMILY_TABLE = {
+ ("krea-2", "krea/Krea-2-Turbo"): {
+ 8: ("on-the-fly int8", OFFLOAD_STREAMING),
+ 12: ("on-the-fly int8", OFFLOAD_GROUP),
+ 16: ("on-the-fly int8", OFFLOAD_GROUP),
+ 24: ("on-the-fly int8", OFFLOAD_GROUP),
+ 32: ("on-the-fly int8", OFFLOAD_GROUP),
+ },
+ ("lumina-2", "Alpha-VLLM/Lumina-Image-2.0"): {
+ 8: ("hosted int8", OFFLOAD_STREAMING),
+ 12: ("hosted int8", OFFLOAD_STREAMING),
+ 16: ("hosted int8", OFFLOAD_GROUP),
+ 24: ("hosted int8", OFFLOAD_NONE),
+ # measured: bf16 while it fits resident
+ 32: ("bf16", OFFLOAD_NONE),
+ },
+ ("hunyuanimage-2.1", "hunyuanvideo-community/HunyuanImage-2.1-Diffusers"): {
+ 8: ("hosted int8", OFFLOAD_STREAMING),
+ 12: ("hosted int8", OFFLOAD_STREAMING),
+ 16: ("hosted int8", OFFLOAD_GROUP),
+ 24: ("hosted int8", OFFLOAD_GROUP),
+ 32: ("hosted int8", OFFLOAD_GROUP),
+ },
+ ("hidream-i1", "HiDream-ai/HiDream-I1-Full"): {
+ 8: ("hosted int8", OFFLOAD_STREAMING),
+ 12: ("hosted int8", OFFLOAD_STREAMING),
+ 16: ("hosted int8", OFFLOAD_GROUP),
+ 24: ("hosted int8", OFFLOAD_GROUP),
+ 32: ("hosted int8", OFFLOAD_GROUP),
+ },
+ ("ideogram-4", "ideogram-ai/ideogram-4-fp8"): {
+ 8: ("on-the-fly int8", OFFLOAD_STREAMING),
+ 12: ("on-the-fly int8", OFFLOAD_STREAMING),
+ 16: ("on-the-fly int8", OFFLOAD_GROUP),
+ 24: ("on-the-fly int8", OFFLOAD_GROUP),
+ 32: ("on-the-fly int8", OFFLOAD_GROUP),
+ },
+}
+
+
+ "family, base, gib, expected",
+ [
+ (family, base, gib, expected)
+ for (family, base), by_gib in FAMILY_TABLE.items()
+ for gib, expected in by_gib.items()
+ ],
+)
+def test_auto_keeps_int8_on_the_offload_tier(planner, family, base, gib, expected):
+ assert planner(family, base, gib) == expected
+
+
[email protected]("version, expected", [(T016, "bf16"), (T017, "hosted fp8")])
+def test_older_torchao_streamed_seeds(planner, monkeypatch, version, expected):
+ monkeypatch.setattr(mem, "_installed_torchao_version", lambda: version)
+ precision, _policy = planner(
+ "hunyuanimage-2.1", "hunyuanvideo-community/HunyuanImage-2.1-Diffusers", 16
+ )
+ assert precision == expected
+
+
+def _gpu_stack():
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+ pytest.importorskip("torchao")
+ hooks = pytest.importorskip("diffusers.hooks")
+ return torch, hooks
+
+
[email protected]("scheme", ["int8", "fp8"])
[email protected]("low_cpu_mem_usage", [False, True])
[email protected]("requires_grad", [False, True])
+def test_quantised_blocks_survive_group_offload_under_no_grad(
+ scheme, low_cpu_mem_usage, requires_grad
+):
+ import copy
+ import inspect
+
+ torch, hooks = _gpu_stack()
+ from torchao.quantization import (
+ Float8DynamicActivationFloat8WeightConfig,
+ Int8DynamicActivationInt8WeightConfig,
+ PerRow,
+ quantize_,
+ )
+
+ if scheme == "fp8" and torch.cuda.get_device_capability() < (8, 9):
+ pytest.skip("fp8 needs sm_89+")
+ torch.manual_seed(0)
+ blocks = torch.nn.Sequential(
+ *[
+ torch.nn.Sequential(
+ torch.nn.Linear(256, 512), torch.nn.GELU(), torch.nn.Linear(512, 256)
+ )
+ for _ in range(3)
+ ]
+ ).to(torch.bfloat16)
+ config = (
+ Int8DynamicActivationInt8WeightConfig
+ if scheme == "int8"
+ else lambda: Float8DynamicActivationFloat8WeightConfig(granularity = PerRow())
+ )
+ offloaded = copy.deepcopy(blocks)
+ quantize_(offloaded, config())
+ # Same CPU-quantised weights: fp8 quantised on CPU can round an ulp away from CUDA (torch 2.12).
+ resident = copy.deepcopy(offloaded).cuda()
+ for param in offloaded.parameters():
+ param.requires_grad_(requires_grad)
+ params = inspect.signature(hooks.apply_group_offloading).parameters
+ kwargs = {
+ "onload_device": torch.device("cuda"),
+ "offload_device": torch.device("cpu"),
+ "offload_type": "block_level",
+ "num_blocks_per_group": 1,
+ "use_stream": True,
+ }
+ for name, value in (
+ ("non_blocking", True),
+ ("record_stream", True),
+ ("low_cpu_mem_usage", low_cpu_mem_usage),
+ ):
+ if name in params:
+ kwargs[name] = value
+ hooks.apply_group_offloading(offloaded, **mem._torchao_group_offload_kwargs(offloaded, kwargs))
+ x = torch.randn(2, 16, 256, dtype = torch.bfloat16, device = "cuda")
+ with torch.no_grad():
+ want = resident(x)
+ for _ in range(3):
+ assert torch.equal(offloaded(x), want)
+ with pytest.raises(Exception), torch.inference_mode():
+ offloaded(x)
+
+
+ "offload_active, denoiser_offloaded, expected",
+ [(False, None, False), (True, None, True), (True, False, False), (True, True, True)],
+)
+def test_graph_gate_follows_the_denoiser(monkeypatch, offload_active, denoiser_offloaded, expected):
+ from core.inference import diffusion_cuda_graph as dcg
+ from core.inference import diffusion_speed
+
+ seen: list = []
+
+ def _eligible(target, **kwargs):
+ seen.append(kwargs["offload_active"])
+ return False, "stub"
+
+ monkeypatch.setattr(dcg, "graph_eligible", _eligible)
+ pipe = types.SimpleNamespace(components = {})
+ target = types.SimpleNamespace(device = "cpu", backend = "cpu", dtype = None)
+ kwargs = {} if denoiser_offloaded is None else {"denoiser_offloaded": denoiser_offloaded}
+ diffusion_speed.apply_speed_optims(
+ pipe,
+ target,
+ is_gguf = False,
+ family = types.SimpleNamespace(name = "z-image"),
+ speed_mode = "default",
+ offload_active = offload_active,
+ **kwargs,
+ )
+ assert seen == [expected]
+
+
+class _Hooked:
+ def __init__(
+ self,
+ keys = (),
+ hf_hook = None,
+ ):
+ self._diffusers_hook = types.SimpleNamespace(hooks = {k: object() for k in keys})
+ self._hf_hook = hf_hook
+
+
+ "transformer, hooked",
+ [
+ (_Hooked(), False),
+ (_Hooked(("group_offloading", "lazy_prefetch_group_offloading")), True),
+ (_Hooked(hf_hook = object()), True),
+ (_Hooked(("first_block_cache",)), False),
+ ],
+)
+def test_denoiser_hooked(transformer, hooked):
+ assert dmod._denoiser_hooked(types.SimpleNamespace(transformer = transformer)) is hooked
+
+
+def _int8_linear(rows = 1024, cols = 1024):
+ torch = pytest.importorskip("torch")
+ quant = pytest.importorskip("torchao.quantization")
+ linear = torch.nn.Linear(cols, rows, bias = False).to(torch.bfloat16)
+ try:
+ quant.quantize_(linear, quant.Int8DynamicActivationInt8WeightConfig(version = 2))
+ except Exception as exc: # noqa: BLE001 - torchao without the v2 int8 tensor
+ pytest.skip(f"int8 v2 quantisation unavailable: {exc}")
+ if type(linear.weight).__name__ != "Int8Tensor":
+ pytest.skip("this torchao does not produce Int8Tensor")
+ return linear
+
+
+def test_host_size_counts_packed_torchao_storage_not_the_logical_bf16_size():
+ linear = _int8_linear(2048, 2048)
+ # 2048 x 2048 int8 = 4 MiB of qdata plus per-row scales; the logical bf16 size would be 8 MiB.
+ assert mem._module_host_mib(linear) < 8
+ assert mem._module_host_mib(linear) >= 4
+
+
+def test_stream_kept_when_the_packed_weights_fit_the_pin_budget(monkeypatch):
+ linear = _int8_linear(2048, 2048)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+ monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: None)
+ # Between the packed (~4 MiB) and logical bf16 (8 MiB) sizes.
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 6)
+ kwargs = mem._torchao_group_offload_kwargs(
+ linear, {"use_stream": True, "low_cpu_mem_usage": True}
+ )
+ assert kwargs["use_stream"] is True
+ assert kwargs["low_cpu_mem_usage"] is False
+
+
+def test_storage_size_recurses_through_nested_wrappers():
+ class Inner:
+ def __init__(self, n):
+ self._n = n
+
+ def numel(self):
+ return self._n
+
+ def element_size(self):
+ return 1
+
+ class Wrapper:
+ def __init__(self, *inner):
+ self._inner = inner
+ for i, t in enumerate(inner):
+ setattr(self, f"t{i}", t)
+
+ def __tensor_flatten__(self):
+ return [f"t{i}" for i in range(len(self._inner))], None
+
+ def numel(self):
+ return 10**9
+
+ def element_size(self):
+ return 2
+
+ nested = Wrapper(Wrapper(Inner(100), Inner(4)), Inner(7))
+ assert mem._storage_nbytes(nested) == [100, 4, 7]
+
+
+ "env, capped, budget, keeps_stream",
+ [
+ ("1", True, None, True),
+ ("1", False, 0, True),
+ ("0", False, 10**7, False),
+ ("", False, 10**7, True),
+ ],
+)
+def test_runtime_pinning_honours_the_same_override_as_the_planner(
+ monkeypatch, env, capped, budget, keeps_stream
+):
+ linear = _int8_linear(256, 256)
+ monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, env)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: capped)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)
+ monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: None)
+ kwargs = mem._torchao_group_offload_kwargs(
+ linear, {"use_stream": True, "low_cpu_mem_usage": True}
+ )
+ assert kwargs["use_stream"] is keeps_stream
+
+
+def test_several_torchao_denoisers_share_one_pin_budget(monkeypatch):
+ """Ideogram 4 streams two torchao transformers: each fits the budget alone, together they do not."""
+ first, second = _int8_linear(2048, 2048), _int8_linear(2048, 2048)
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+ monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: None)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 6)
+ base = {"use_stream": True, "low_cpu_mem_usage": True}
+ pinned = [0]
+ kept = [mem._torchao_group_offload_kwargs(m, dict(base), pinned) for m in (first, second)]
+ assert kept[0]["low_cpu_mem_usage"] is False
+ assert kept[1].get("low_cpu_mem_usage") is not False
+ assert pinned[0] <= 6
+
+
+def test_every_group_offload_site_shares_the_pin_total():
+ import inspect
+
+ import re
+
+ source = inspect.getsource(mem)
+ calls = re.findall(r"(?<!def )_torchao_group_offload_kwargs\(([^)]*)\)", source)
+ assert calls and all("pinned_mib" in args for args in calls)
+
+
[email protected]("fits_whole, raises", [(False, True), (True, False)])
+def test_failed_group_setup_never_onloads_an_oversized_quantised_transformer(
+ monkeypatch, fits_whole, raises
+):
+ """Streaming was the only placement the quantised transformer fits; model offload would OOM on first onload."""
+ calls = []
+ pipe = types.SimpleNamespace(
+ enable_model_cpu_offload = lambda device = None: calls.append("model_offload")
+ )
+ monkeypatch.setattr(mem, "_apply_group_offload", lambda *a, **k: False)
+ monkeypatch.setattr(mem, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ monkeypatch.setattr(mem, "_model_offload_fits_quantised", lambda plan: fits_whole)
+ monkeypatch.setattr(mem, "keep_cpu_weights_on_offload", lambda *a, **k: 0)
+ monkeypatch.setattr(mem, "_enable_vae_saver", lambda *a, **k: False)
+ plan = _plan(OFFLOAD_GROUP)
+ if raises:
+ with pytest.raises(RuntimeError, match = "does not fit the GPU whole"):
+ mem.apply_memory_plan(pipe, plan, device = "cuda")
+ assert "model_offload" not in calls
+ else:
+ effective, _ = mem.apply_memory_plan(pipe, plan, device = "cuda")
+ assert effective == OFFLOAD_MODEL
+
+
+def test_pipeline_host_mib_counts_only_host_resident_weights():
+ torch = pytest.importorskip("torch")
+ cpu = torch.nn.Linear(1024, 1024, bias = False) # 4 MiB fp32
+ pipe = types.SimpleNamespace(components = {"transformer": cpu, "scheduler": object()})
+ assert mem.pipeline_host_mib(pipe) == 4
+ assert mem.pipeline_host_mib(None) == 0
+ if torch.cuda.is_available():
+ pipe.components["transformer"] = cpu.to("cuda")
+ assert mem.pipeline_host_mib(pipe) == 0
+
+
+def test_reclaimable_host_ram_widens_the_pin_budget(monkeypatch):
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 4_000)
+ plan = types.SimpleNamespace(
+ estimates = {"model_dense_mib": 20_000, "companion_dense_mib": 8_000}
+ )
+ assert not mem._torchao_stream_pinnable(plan)
+ assert mem._torchao_stream_pinnable(plan, 8_000)
diff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.py
index 11898f4dedc..7039e012e0b 100644
--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py
+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py
@@ -56,6 +56,16 @@ DENOISER_BYTES = 22_000 * MIB
ALL_BYTES = sum(size for _name, size in Z_IMAGE_FILES)
[email protected](autouse = True)
+def _unmeasured_torchao(monkeypatch):
+ """Pin "no measured torchao" so the installed release does not decide the offload tiers."""
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: None)
+ # Studio's diffusers pin, so tests that opt into a measured torchao do not depend on the runner.
+ monkeypatch.setattr(diffusion_memory, "_installed_diffusers_version", lambda: (0, 40))
+
+
@pytest.fixture(autouse = True)
def _safetensors_readable_regardless_of_the_installed_torchao(monkeypatch):
"""Pin the torchao floor so planning tests ignore the installed release."""
@@ -1029,6 +1039,50 @@ def test_an_artifact_plan_that_streams_the_transformer_still_declines(monkeypatc
assert _settle(backend) == PIPELINE_SEED_DECLINED
[email protected]("policy", ["group", "streaming"])
+def test_an_artifact_plan_that_streams_the_transformer_seeds_on_a_measured_torchao(
+ monkeypatch, policy
+):
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))
+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)
+ backend = _settle_backend(monkeypatch)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **_k: types.SimpleNamespace(offload_policy = policy, stream_transformer = True),
+ )
+ assert _settle(backend) == "fp8"
+
+
+def test_a_resident_rung_still_beats_a_streamed_one(monkeypatch):
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))
+ monkeypatch.setattr(diffusion_memory, "_torchao_stream_pinnable", lambda plan, *_a: True)
+ backend = _settle_backend_walking(
+ monkeypatch, artifacts = ("int8", "fp8"), candidates = ("int8", "fp8")
+ )
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **k: types.SimpleNamespace(
+ offload_policy = "group" if k["transformer_resident_override_mib"] >= 31_000 else "none",
+ stream_transformer = True,
+ ),
+ )
+ assert _settle(backend) == "fp8"
+ # no resident rung: the first streamed one is seeded
+ backend = _settle_backend_walking(monkeypatch, artifacts = ("int8",), candidates = ("int8", "fp8"))
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **_k: types.SimpleNamespace(offload_policy = "group", stream_transformer = True),
+ )
+ assert _settle(backend) == "int8"
+
+
def test_a_seed_whose_load_plan_streams_only_the_encoders_is_kept(fake_runtime, monkeypatch):
backend, spy = _load_backend(monkeypatch, offload = "group")
real_plan = DiffusionBackend._plan_memory
@@ -1711,3 +1765,29 @@ def test_the_kept_bf16_weights_are_placed_by_the_plan_that_proved_the_fit(
assert spy.quantised == []
assert placed == ["none"]
+
+
[email protected]("outgoing_host_mib, expected", [(0, False), (10_000, True)])
+def test_the_streamed_seed_credits_host_ram_the_outgoing_pipeline_frees(
+ monkeypatch, outgoing_host_mib, expected
+):
+ # The plan runs before the previous pipeline unloads, so its host weights must not veto the pin.
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_installed_torchao_version", lambda: (0, 18))
+ monkeypatch.setattr(diffusion_memory, "_pinned_memory_capped", lambda: False)
+ monkeypatch.delenv(diffusion_memory.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(diffusion_memory, "_pin_budget_mib", lambda: 4_000)
+ monkeypatch.setattr(dmod, "pipeline_host_mib", lambda pipe: outgoing_host_mib)
+ backend = _settle_backend(monkeypatch)
+ backend._state = types.SimpleNamespace(pipe = object())
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **_k: types.SimpleNamespace(
+ offload_policy = "group",
+ stream_transformer = True,
+ estimates = {"model_dense_mib": 20_000, "companion_dense_mib": 8_000},
+ ),
+ )
+ assert (_settle(backend) == "fp8") is expected
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts
index 22edde8fe0f..a20daf2c5e0 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts
@@ -1205,16 +1205,16 @@ for (const catalog of [IMAGE_CATALOG, VIDEO_CATALOG, AUDIO_CATALOG]) {
}
}
-// Z-Image-Turbo needs 42.9 GB of card dense and 34.9 GB pre-quantised under the 70% rule.
+// Z-Image-Turbo: 27.3 GB card dense (bf16, not the fp32 download), 19.3 GB pre-quantised, under the 70% rule.
const zTurboId = "unsloth/Z-Image-Turbo";
assert.equal(
- curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, { gpuGb: 40, systemRamGb: 128 }),
+ curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, { gpuGb: 24, systemRamGb: 128 }),
false,
);
for (const schemes of [["fp8"], ["int8"]]) {
assert.equal(
curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, {
- gpuGb: 40,
+ gpuGb: 24,
systemRamGb: 128,
denseQuantSchemes: schemes,
}),
@@ -1224,12 +1224,16 @@ for (const schemes of [["fp8"], ["int8"]]) {
}
assert.equal(
curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, {
- gpuGb: 40,
+ gpuGb: 24,
systemRamGb: 128,
denseQuantSchemes: [],
}),
false,
);
+assert.equal(
+ curatedArtifactFitsDevice(zTurboId, IMAGE_CATALOG, { gpuGb: 32, systemRamGb: 128 }),
+ true,
+);
assert.equal(
curatedArtifactFitsDevice("Qwen/Qwen-Image", IMAGE_CATALOG, {
gpuGb: 64,
@@ -1321,7 +1325,7 @@ const zTurboGroup = groupForRepoId(zTurboId, IMAGE_CATALOG);
assert.ok(zTurboGroup);
assert.equal(
pickDefaultArtifact(zTurboGroup, {
- gpuGb: 40,
+ gpuGb: 24,
systemRamGb: 128,
isDownloaded: notDownloaded,
}).format,
@@ -1342,6 +1346,15 @@ assert.equal(
systemRamGb: 128,
denseQuantSchemes: ["fp8"],
isDownloaded: notDownloaded,
+ }).repoId,
+ zTurboId,
+);
+assert.equal(
+ pickDefaultArtifact(zTurboGroup, {
+ gpuGb: 16,
+ systemRamGb: 128,
+ denseQuantSchemes: ["fp8"],
+ isDownloaded: notDownloaded,
}).format,
"bnb-4bit",
);
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
index f71b306e62d..395aebbb812 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
@@ -178,7 +178,8 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
description: "Text-to-image",
scope: "image",
artifacts: [
- bf16Mirror("Tongyi-MAI/Z-Image-Turbo", 30, {
+ // Resident bf16 GiB, not the Hub's fp32 DiT: 11.5 DiT + 7.5 Qwen3-4B + 0.2 VAE.
+ bf16Mirror("Tongyi-MAI/Z-Image-Turbo", 19.1, {
totalParams: 6154908736,
prequantRepo: "unsloth/Z-Image-Turbo-FP8",
prequantSizeGb: { fp8: 5.86, int8: 5.86 },
diff --git a/studio/frontend/tests/dense-quant-prequant-routing.test.ts b/studio/frontend/tests/dense-quant-prequant-routing.test.ts
index 56e3c58c4f5..178462a2670 100644
--- a/studio/frontend/tests/dense-quant-prequant-routing.test.ts
+++ b/studio/frontend/tests/dense-quant-prequant-routing.test.ts
@@ -63,19 +63,19 @@ test("the catalog states the hosted checkpoint each official row would be fetche
test("the fit verdict is the quantised resident size on a host that runs a scheme", () => {
assert.equal(
- curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40)),
+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24)),
false,
);
assert.equal(
- curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40, ["fp8"])),
+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24, ["fp8"])),
true,
);
assert.equal(
- curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40, ["int8"])),
+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24, ["int8"])),
true,
);
assert.equal(
- curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24, ["fp8"])),
+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(16, ["fp8"])),
false,
);
});
@@ -118,7 +118,7 @@ test("the 2512 pick is judged by the checkpoint the backend seeds for it, not by
test("a host with no scheme, and a row with no hosted checkpoint, are unchanged", () => {
assert.equal(
- curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(40, [])),
+ curatedArtifactFitsDevice(Z_TURBO, IMAGE_CATALOG, onCard(24, [])),
false,
);
for (const id of ["black-forest-labs/FLUX.1-dev", "stabilityai/sdxl-turbo"]) {
@@ -134,20 +134,20 @@ test("the router sends a card that only fits the quantised form to the official
const group = groupForRepoId(Z_TURBO, IMAGE_CATALOG);
assert.ok(group);
assert.equal(
- pickDefaultArtifact(group, { ...onCard(40), isDownloaded: notDownloaded })
+ pickDefaultArtifact(group, { ...onCard(24), isDownloaded: notDownloaded })
.format,
"bnb-4bit",
);
assert.equal(
pickDefaultArtifact(group, {
- ...onCard(40, ["fp8"]),
+ ...onCard(24, ["fp8"]),
isDownloaded: notDownloaded,
}).repoId,
Z_TURBO,
);
assert.equal(
pickDefaultArtifact(group, {
- ...onCard(24, ["fp8"]),
+ ...onCard(16, ["fp8"]),
isDownloaded: notDownloaded,
}).format,
"bnb-4bit",
@@ -314,10 +314,10 @@ test("the over-budget text never shows the size below the budget it exceeds", ()
assert.ok(budget);
return curatedBudgetText(Math.round(fit.sizeGb), gpuGb, budget);
};
- // 24.40 GB rounds to 24, under the 24.1 budget, so it is shown rounded up.
+ // 13.50 GB rounds to 13, under the 13.3 budget, so it is shown rounded up.
assert.equal(
- text(Z_TURBO, 34.5),
- "Needs ~24.4GB for weights (budget: ~24.1GB, 70% of a 34.5GB GPU)",
+ text(Z_TURBO, 19),
+ "Needs ~13.5GB for weights (budget: ~13.3GB, 70% of a 19GB GPU)",
);
// A whole 4 GB against a 3.99 allowance: the budget is rounded down so 4.0 still reads as over.
const whisper = curatedArtifactFit("unsloth/whisper-large-v3", AUDIO_CATALOG, {