Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run Qwen-Image-2.1 int8/fp8 on 24 GB cards with the transformer resident and the text encoder streamed (#11883)

Daniel Hanf2a9375055f2
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 5338fd1b92b..57b33faa4f1 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -111,6 +111,7 @@ from .diffusion_memory import (
normalize_memory_mode,
plan_diffusion_memory,
plan_fits_total_capacity,
+ plan_keeps_transformer_resident,
raise_on_image_activation_shortfall,
raise_on_unified_memory_shortfall,
reclaimable_snapshot_device_memory,
@@ -1337,6 +1338,19 @@ def _no_recast_pipeline_class(pipe_cls: Any) -> Any:
_NO_WEIGHT = object()
+def _torchao_offload_decline(what: str, plan: Any) -> str:
+ """Why a torchao build was declined under ``plan``, quoting the floor that keeps it resident."""
+ estimates = getattr(plan, "estimates", None) or {}
+ floor = estimates.get("resident_transformer_floor_mib")
+ streamed = f", {floor} MiB even with the text encoders streamed" if floor is not None else ""
+ return (
+ f"{what} '{plan.offload_policy}' offload here "
+ f"({estimates.get('resident_required_mib')} MiB required vs a "
+ f"{estimates.get('safe_device_budget_mib')} MiB budget{streamed}), "
+ "and torchao tensors cannot be offloaded"
+ )
+
+
def _has_active_lora(loras: Any) -> bool:
"""True when any adapter would actually be baked, for either shape the callers pass. Weight 0
means disabled, but /images/load passes ``(id, weight)`` tuples while /images/download-plan
@@ -3235,15 +3249,16 @@ class DiffusionBackend:
repo_id = repo_id,
fetch_base = fetch_base,
transformer_resident_override_mib = candidate.steady_transformer_mib,
- companion_override_mib = candidate.companions_mib,
- text_encoder_override_mib = candidate.text_encoders_mib,
+ **self._candidate_companion_overrides(
+ candidate, fam, base or repo_id or "", target, text_encoder_quant
+ ),
device_memory_override = replace(memory, free_mib = memory.total_mib),
)
- if planned.offload_policy != OFFLOAD_NONE:
+ if not plan_keeps_transformer_resident(planned):
logger.info(
- "diffusion.denoiser_prequant: an artifact-sized plan for %s still offloads "
- "on this card, and offload moves the denoiser via Module.to(), so the "
- "released shards are kept",
+ "diffusion.denoiser_prequant: an artifact-sized plan for %s streams the "
+ "denoiser on this card, and offload moves the denoiser via Module.to(), "
+ "so the released shards are kept",
rung,
)
declined = True
@@ -4690,14 +4705,15 @@ class DiffusionBackend:
transformer_resident_override_mib = (
seed_estimate.steady_transformer_mib
),
- companion_override_mib = seed_estimate.companions_mib,
- text_encoder_override_mib = seed_estimate.text_encoders_mib,
+ **self._candidate_companion_overrides(
+ seed_estimate, fam, base, target, text_encoder_quant
+ ),
text_encoder_quant = text_encoder_quant,
)
if seed_estimate is not None
else None
)
- if seeded_plan is None or seeded_plan.offload_policy != OFFLOAD_NONE:
+ if seeded_plan is None or not plan_keeps_transformer_resident(seeded_plan):
# Offload hooks use Module.to(), which torchao tensors reject, and live free
# memory can undercut the CAPACITY the plan settled this against.
logger.info(
@@ -4973,13 +4989,10 @@ class DiffusionBackend:
transformer_resident_override_mib = (
candidate.transient_transformer_mib
),
- # Pass the companion estimate so prefetched base shards aren't double-counted.
- companion_override_mib = candidate.companions_mib,
- # ... and its text-encoder share, so the planner can still price the streamed-encoder
- # group tier on this path. getattr: a candidate without the split passes None and
- # keeps the previous decision.
- text_encoder_override_mib = getattr(
- candidate, "text_encoders_mib", None
+ # Companion estimate avoids double-counting prefetched shards; its encoder share
+ # keeps the streamed-encoder tiers priceable.
+ **self._candidate_companion_overrides(
+ candidate, fam, base, target, text_encoder_quant
),
text_encoder_quant = text_encoder_quant,
)
@@ -4995,7 +5008,7 @@ class DiffusionBackend:
if candidate is not None:
replanned = _replan_candidate()
if (
- replanned.offload_policy != OFFLOAD_NONE
+ not plan_keeps_transformer_resident(replanned)
# Explicit balanced/low_vram picks offload BY MODE, so a fresh snapshot cannot change
# it.
and normalize_memory_mode(memory_mode)
@@ -5006,7 +5019,7 @@ class DiffusionBackend:
# (settled) and replan once rather than letting a transient foreign allocation force
# the GGUF fallback.
replanned = _replan_candidate()
- if replanned.offload_policy != OFFLOAD_NONE:
+ if not plan_keeps_transformer_resident(replanned):
logger.info(
"diffusion.transformer_quant_declined: required=%s MiB "
"budget=%s MiB free=%s MiB policy=%s (%s)",
@@ -5016,15 +5029,10 @@ class DiffusionBackend:
replanned.offload_policy,
"; ".join(replanned.reasons),
)
- transformer_quant_decline = (
- f"the quantised build still needs '{replanned.offload_policy}' "
- f"offload here ("
- f"{replanned.estimates.get('resident_required_mib')} MiB "
- f"required vs a "
- f"{replanned.estimates.get('safe_device_budget_mib')} MiB "
- "budget), and torchao tensors cannot be offloaded"
+ transformer_quant_decline = _torchao_offload_decline(
+ "the quantised build still needs", replanned
)
- if replanned.offload_policy == OFFLOAD_NONE:
+ if plan_keeps_transformer_resident(replanned):
quant_plan = replanned
# The GGUF plan declined resident; a prequant-sized replan says nothing about the
# dense build
@@ -5064,7 +5072,7 @@ class DiffusionBackend:
if retry_candidate is not None:
candidate = retry_candidate
retry_plan = _replan_candidate()
- if retry_plan.offload_policy == OFFLOAD_NONE:
+ if plan_keeps_transformer_resident(retry_plan):
logger.info(
"diffusion.transformer_quant: auto's pick does not fit "
"resident; retrying at %s, whose checkpoint is cached",
@@ -5229,6 +5237,55 @@ class DiffusionBackend:
"loading the GGUF",
prequant_candidate.transient_transformer_mib,
)
+ # The INT8 artifact outweighs the GGUF, so plan it on its own; a plan streaming the
+ # transformer cannot carry torchao.
+ if (
+ prequant is not None
+ and not dense_declined
+ and getattr(plan.device_memory, "memory_kind", None) != "unified_memory"
+ ):
+ sized_candidate = resolve_dense_quant_candidate(
+ fam = fam,
+ target = target,
+ requested = transformer_quant,
+ base_repo = base,
+ prequant_path = transformer_prequant_path,
+ force_dense = False,
+ logger = logger,
+ )
+ if sized_candidate is not None and sized_candidate.prequant:
+ sized_plan = self._plan_memory(
+ target,
+ single_file_path,
+ base,
+ fam,
+ memory_mode,
+ cpu_offload,
+ kind = kind,
+ repo_id = repo_id,
+ fetch_base = fetch_base,
+ transformer_resident_override_mib = (
+ sized_candidate.transient_transformer_mib
+ ),
+ **self._candidate_companion_overrides(
+ sized_candidate, fam, base, target, text_encoder_quant
+ ),
+ text_encoder_quant = text_encoder_quant,
+ )
+ if plan_keeps_transformer_resident(sized_plan):
+ quant_plan = sized_plan
+ if sized_plan.offload_policy != OFFLOAD_NONE:
+ dense_fallback_allowed = False
+ else:
+ dense_declined = True
+ dense_fallback_allowed = False
+ transformer_quant_decline = _torchao_offload_decline(
+ "the pre-quantised transformer needs", sized_plan
+ )
+ logger.info(
+ "diffusion.transformer_quant_declined: %s",
+ transformer_quant_decline,
+ )
# A dense misfit with a prequant source only forbids the dense fallback; with none, auto could
# still have picked a DIFFERENT scheme that does have a hosted checkpoint. auto returns one
# winner, and a winner with no published prequant that also cannot build dense would drop the
@@ -5276,7 +5333,7 @@ class DiffusionBackend:
retry_candidate.transient_transformer_mib
),
# Same pre-cast encoder pricing as the fit check above.
- companion_override_mib = self._precast_scaled_companions_mib(
+ **self._candidate_companion_overrides(
retry_candidate,
fam,
base,
@@ -5289,7 +5346,7 @@ class DiffusionBackend:
)
if (
retry_plan is not None
- and retry_plan.offload_policy == OFFLOAD_NONE
+ and plan_keeps_transformer_resident(retry_plan)
# Same unified caveat as above: 'none' is not a fit there, so the rung being retried
# has to be sized explicitly before it is pinned.
and unified_memory_shortfall_message(
@@ -5750,21 +5807,25 @@ class DiffusionBackend:
transformer_resident_override_mib = (
estimate.steady_transformer_mib
),
- companion_override_mib = estimate.companions_mib,
- text_encoder_override_mib = estimate.text_encoders_mib,
+ **self._candidate_companion_overrides(
+ estimate, fam, base, target, text_encoder_quant
+ ),
text_encoder_quant = text_encoder_quant,
)
- if replanned.offload_policy == OFFLOAD_NONE:
+ if plan_keeps_transformer_resident(replanned):
logger.info(
"diffusion.transformer_quant: %s fits resident (%d MiB "
- "steady); dropping the bf16 plan's '%s' offload",
+ "steady, encoders streamed=%s); replacing the bf16 plan's "
+ "'%s' offload",
preview_scheme,
estimate.steady_transformer_mib,
+ replanned.offload_policy != OFFLOAD_NONE,
plan.offload_policy,
)
plan = replanned
+ keeps_resident = plan_keeps_transformer_resident(plan)
if (
- plan.offload_policy != OFFLOAD_NONE
+ not keeps_resident
and native_scheme is None
and native_offload_scheme is not None
):
@@ -5775,7 +5836,7 @@ class DiffusionBackend:
native_scheme,
plan.offload_policy,
)
- if plan.offload_policy != OFFLOAD_NONE and native_scheme is None:
+ if not keeps_resident and native_scheme is None:
logger.info(
"diffusion.transformer_quant: skipped (the memory plan picked '%s' "
"offload, which moves the transformer via Module.to())",
@@ -6763,6 +6824,29 @@ class DiffusionBackend:
except Exception: # noqa: BLE001 -- unreadable cache: treat the shards as missing
return False
+ @staticmethod
+ def _candidate_companion_overrides(
+ candidate: Any,
+ fam: DiffusionFamily,
+ base: str,
+ target: Any,
+ text_encoder_quant: Optional[str],
+ ) -> dict[str, Optional[int]]:
+ """``_plan_memory`` overrides for a dense-quant candidate, pricing the encoder the load actually opens."""
+ companions = DiffusionBackend._precast_scaled_companions_mib(
+ candidate, fam, base, target, text_encoder_quant
+ )
+ encoders = getattr(candidate, "text_encoders_mib", None)
+ if encoders and companions is not None:
+ raw = getattr(candidate, "companions_mib", None)
+ if raw is not None:
+ # companions - VAE is the priced encoder share; the VAE term is unscaled in both.
+ encoders = max(0, int(encoders) - (int(raw) - int(companions)))
+ return {
+ "companion_override_mib": companions,
+ "text_encoder_override_mib": encoders,
+ }
+
@staticmethod
def _precast_scaled_companions_mib(
candidate: Any,
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index d058f05ded3..b14cb9d56f8 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -886,6 +886,34 @@ def _safe_device_budget_mib(memory: DeviceMemory) -> Optional[int]:
return max(0, int(memory.free_mib) - _reserve_mib(memory.memory_kind, base))
+def plan_keeps_transformer_resident(plan: Any) -> bool:
+ """Whether ``plan`` never moves the denoiser after placement: torchao survives placement, not per-forward hooks."""
+ policy = getattr(plan, "offload_policy", OFFLOAD_NONE)
+ if policy == OFFLOAD_NONE:
+ return True
+ return policy == OFFLOAD_GROUP and not bool(getattr(plan, "stream_transformer", True))
+
+
+def _holds_torchao_weights(module: Any) -> bool:
+ """Whether any parameter of ``module`` is a torchao tensor subclass (GGUF and native int8 are not)."""
+ try:
+ for param in module.parameters():
+ for tensor in (param, getattr(param, "data", None)):
+ if tensor is not None and type(tensor).__module__.startswith("torchao"):
+ return True
+ except Exception: # noqa: BLE001 - an unreadable module is treated as movable, today's behaviour
+ return False
+ return False
+
+
+def _pipe_denoisers_hold_torchao(pipe: Any) -> bool:
+ return any(
+ _holds_torchao_weights(getattr(pipe, name, None))
+ for name in ("transformer", "transformer_2", "unconditional_transformer")
+ if getattr(pipe, name, None) is not None
+ )
+
+
def plan_fits_total_capacity(plan: Any) -> bool:
"""Whether ``plan``'s resident requirement fits TOTAL device capacity under the standard
reserve + the 0.85 resident margin -- i.e. an offload decision can only stem from the
@@ -1351,6 +1379,11 @@ def apply_memory_plan(
if not bool(getattr(plan, "stream_transformer", True)):
group_kwargs["stream_transformer"] = False
if not _apply_group_offload(pipe, placement, logger, **group_kwargs):
+ if "stream_transformer" in group_kwargs and _pipe_denoisers_hold_torchao(pipe):
+ raise RuntimeError(
+ "the text encoder could not be streamed beside the resident quantised "
+ "transformer, and whole-module offload cannot move torchao weights"
+ )
_fallback_to_model_offload()
policy = OFFLOAD_MODEL
elif policy == OFFLOAD_STREAMING:
@@ -1675,6 +1708,8 @@ def _apply_group_offload(
_remove_group_offload_hooks(module)
raise
if not stream_transformer and not transformer_demoted:
+ if _pipe_denoisers_hold_torchao(pipe):
+ raise
# Model offload is gone once hooks exist: stream the transformer before this encoder goes resident.
for dit_name in ("transformer", "transformer_2", "unconditional_transformer"):
dit = getattr(pipe, dit_name, None)
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 502180ab115..4b3f329c6c4 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -13059,3 +13059,242 @@ def test_status_reports_cuda_graph_off_once_every_armed_step_ran_eager():
st = backend.status()
assert st["resolved"]["cuda_graph"]["value"] == "on"
assert st["speed_optims"] == ["compiled", "cuda_graph"]
+
+
+def _resident_transformer(plan):
+ """``plan`` as the tier that keeps the denoiser resident and streams only the text encoders."""
+ return dataclasses.replace(
+ plan, offload_policy = "group", stream_text_encoders = True, stream_transformer = False
+ )
+
+
+def _record_placement(monkeypatch):
+ from core.inference import diffusion as dmod
+
+ placed: list = []
+
+ def _apply(pipe, plan, **kwargs):
+ placed.append(plan)
+ return plan.offload_policy, False
+
+ monkeypatch.setattr(dmod, "apply_memory_plan", _apply)
+ return placed
+
+
+def test_a_pipeline_quantises_where_only_the_encoders_stream(fake_runtime, tmp_path, monkeypatch):
+ """torchao weights fit once the encoders stream, so the transformer is converted and placed once."""
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ real_plan = DiffusionBackend._plan_memory
+
+ def _plan(self, *args, **kwargs):
+ plan = real_plan(self, *args, **kwargs)
+ if kwargs.get("transformer_resident_override_mib") is not None:
+ return _resident_transformer(plan)
+ return dataclasses.replace(plan, offload_policy = "group")
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+ placed = _record_placement(monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ _base_local_dir = str(tmp_path),
+ )
+ assert len(calls) == 1
+ assert status["transformer_quant"] == "fp8"
+ assert status["offload_policy"] == "group"
+ assert placed and placed[-1].stream_transformer is False
+ backend.unload()
+
+
+def test_a_pipeline_quant_replan_prices_the_encoder_the_load_opens(
+ fake_runtime, tmp_path, monkeypatch
+):
+ """The in-place replan prices encoders like the other candidate replans, not the family table."""
+ backend = DiffusionBackend()
+ _stub_pipeline_dense_quant(backend, monkeypatch)
+ priced = {"companion_override_mib": 1234, "text_encoder_override_mib": 567}
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_candidate_companion_overrides",
+ staticmethod(lambda *args, **kwargs: dict(priced)),
+ )
+ real_plan = DiffusionBackend._plan_memory
+ seen = []
+
+ def _plan(self, *args, **kwargs):
+ plan = real_plan(self, *args, **kwargs)
+ if kwargs.get("transformer_resident_override_mib") is not None:
+ seen.append(kwargs)
+ return _resident_transformer(plan)
+ return dataclasses.replace(plan, offload_policy = "group")
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+ _record_placement(monkeypatch)
+ backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ _base_local_dir = str(tmp_path),
+ )
+ assert seen
+ assert seen[-1]["companion_override_mib"] == 1234
+ assert seen[-1]["text_encoder_override_mib"] == 567
+ backend.unload()
+
+
+def test_a_pipeline_whose_quantised_plan_still_streams_the_transformer_stays_dense(
+ fake_runtime, tmp_path, monkeypatch
+):
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ real_plan = DiffusionBackend._plan_memory
+
+ def _plan(self, *args, **kwargs):
+ return dataclasses.replace(real_plan(self, *args, **kwargs), offload_policy = "group")
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+ _record_placement(monkeypatch)
+ with pytest.raises(RuntimeError) as excinfo:
+ backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ _base_local_dir = str(tmp_path),
+ )
+ assert calls == []
+ assert "Module.to()" in str(excinfo.value)
+
+
+def _gguf_candidate_backend(monkeypatch, tmp_path, *, initial_policy, candidate_plan):
+ """GGUF load planned as ``initial_policy`` whose candidate plans as ``candidate_plan(real_plan)``."""
+ from core.inference import diffusion as dmod
+
+ backend = _cuda_backend(tmp_path, monkeypatch)
+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(
+ dmod, "select_transformer_quant_scheme", lambda target, mode, family = None: "int8"
+ )
+ monkeypatch.setattr(dmod, "usable_prequant_source", lambda fam, scheme, **kw: "prequant/path")
+ monkeypatch.setattr(
+ dmod,
+ "resolve_dense_quant_candidate",
+ lambda **kw: types.SimpleNamespace(
+ transient_transformer_mib = 7_000,
+ steady_transformer_mib = 7_000,
+ companions_mib = 18_000,
+ text_encoders_mib = 16_700,
+ prequant = True,
+ ),
+ )
+ orig_plan = DiffusionBackend._plan_memory
+
+ def spy_plan(
+ self,
+ *a,
+ transformer_resident_override_mib = None,
+ **k,
+ ):
+ real = orig_plan(
+ self, *a, transformer_resident_override_mib = transformer_resident_override_mib, **k
+ )
+ if transformer_resident_override_mib is None:
+ return dataclasses.replace(real, offload_policy = initial_policy)
+ return candidate_plan(real)
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", spy_plan)
+ attempted: list = []
+
+ def fake_dense_load(self, *a, **k):
+ attempted.append(k.get("allow_dense_fallback"))
+ raise RuntimeError("test: stop after reaching the fast path")
+
+ monkeypatch.setattr(DiffusionBackend, "_load_dense_quant_pipeline", fake_dense_load)
+ return backend, attempted
+
+
+def test_a_gguf_pick_builds_the_quant_where_only_the_encoders_stream(
+ fake_runtime, tmp_path, monkeypatch, allow_precision_fallback
+):
+ backend, attempted = _gguf_candidate_backend(
+ monkeypatch, tmp_path, initial_policy = "group", candidate_plan = _resident_transformer
+ )
+ _load_m(backend, tmp_path, transformer_quant = "int8")
+ assert attempted == [False]
+
+
+def test_a_gguf_pick_declines_the_quant_where_the_transformer_would_stream(
+ fake_runtime, tmp_path, monkeypatch, allow_precision_fallback
+):
+ backend, attempted = _gguf_candidate_backend(
+ monkeypatch,
+ tmp_path,
+ initial_policy = "group",
+ candidate_plan = lambda real: dataclasses.replace(real, offload_policy = "group"),
+ )
+ status = _load_m(backend, tmp_path, transformer_quant = "int8")
+ assert attempted == []
+ assert status["transformer_quant"] is None
+
+
+def test_a_resident_gguf_plan_sizes_the_prequant_that_replaces_it(
+ fake_runtime, tmp_path, monkeypatch, allow_precision_fallback
+):
+ """The INT8 artifact outgrows the GGUF, so it loads under its own plan rather than the GGUF's."""
+ sized: list = []
+
+ def _candidate(real):
+ plan = _resident_transformer(real)
+ sized.append(plan)
+ return plan
+
+ backend, attempted = _gguf_candidate_backend(
+ monkeypatch, tmp_path, initial_policy = "none", candidate_plan = _candidate
+ )
+ placed = _record_placement(monkeypatch)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_load_dense_quant_pipeline",
+ lambda self, *a, **k: attempted.append(k.get("allow_dense_fallback")) or (None, None),
+ )
+ _load_m(backend, tmp_path, transformer_quant = "int8")
+ assert sized, "the prequant was never sized"
+ assert attempted == [False]
+ assert placed[-1].offload_policy in ("none", "group")
+
+
+def test_a_resident_gguf_plan_declines_a_prequant_that_would_stream(
+ fake_runtime, tmp_path, monkeypatch, allow_precision_fallback
+):
+ backend, attempted = _gguf_candidate_backend(
+ monkeypatch,
+ tmp_path,
+ initial_policy = "none",
+ candidate_plan = lambda real: dataclasses.replace(real, offload_policy = "group"),
+ )
+ status = _load_m(backend, tmp_path, transformer_quant = "int8")
+ assert attempted == []
+ assert status["transformer_quant"] is None
+ assert (
+ "torchao tensors cannot be offloaded" in (status["resolved"]["transformer_quant"]["reason"])
+ )
+
+
+def test_candidate_overrides_price_the_encoder_the_load_opens(monkeypatch):
+ candidate = types.SimpleNamespace(companions_mib = 18_000, text_encoders_mib = 16_000)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_precast_scaled_companions_mib",
+ staticmethod(lambda cand, fam, base, target, teq: 2_000 + int(16_000 * 0.65)),
+ )
+ overrides = DiffusionBackend._candidate_companion_overrides(candidate, None, "b", None, "fp8")
+ assert overrides == {"companion_override_mib": 12_400, "text_encoder_override_mib": 10_400}
+ # No split: the encoder term stays unknown, so the planner keeps its previous tiers.
+ bare = types.SimpleNamespace(companions_mib = 18_000, text_encoders_mib = 0)
+ assert (
+ DiffusionBackend._candidate_companion_overrides(bare, None, "b", None, None)[
+ "text_encoder_override_mib"
+ ]
+ == 0
+ )
diff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.py
index 0892cf6de0c..11ed0915b52 100644
--- a/studio/backend/tests/test_diffusion_memory.py
+++ b/studio/backend/tests/test_diffusion_memory.py
@@ -2460,3 +2460,97 @@ def test_pinned_host_pricing_rounds_each_tensor_to_a_power_of_two(monkeypatch):
return [_T(1024 * 1024)]
assert mem._module_host_mib(_M()) == 13
+
+
+def test_only_resident_and_encoder_only_plans_keep_the_transformer_in_place():
+ from types import SimpleNamespace
+
+ import core.inference.diffusion_memory as mem
+
+ keeps = mem.plan_keeps_transformer_resident
+ assert keeps(SimpleNamespace(offload_policy = OFFLOAD_NONE)) is True
+ assert keeps(SimpleNamespace(offload_policy = OFFLOAD_GROUP, stream_transformer = False)) is True
+ assert keeps(SimpleNamespace(offload_policy = OFFLOAD_GROUP, stream_transformer = True)) is False
+ # Plans built before the field existed stream the transformer.
+ assert keeps(SimpleNamespace(offload_policy = OFFLOAD_GROUP)) is False
+ for policy in (OFFLOAD_MODEL, mem.OFFLOAD_STREAMING, mem.OFFLOAD_SEQUENTIAL):
+ assert keeps(SimpleNamespace(offload_policy = policy, stream_transformer = False)) is False
+ assert keeps(_q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB)) is True
+ assert keeps(_q21_plan(_16G_FREE_MIB, _16G_TOTAL_MIB)) is False
+
+
+class _TorchaoLikeTensor:
+ """Stands in for a torchao subclass: only its defining module is read."""
+
+
+_TorchaoLikeTensor.__module__ = "torchao.quantization.linear_activation_quantized_tensor"
+
+
+class _ParamsModule:
+ def __init__(self, *tensors):
+ self._tensors = tensors
+
+ def parameters(self):
+ return iter(self._tensors)
+
+
+def test_torchao_weights_are_told_apart_from_gguf_and_plain_ones():
+ import core.inference.diffusion_memory as mem
+
+ class _GGUFParameter:
+ pass
+
+ _GGUFParameter.__module__ = "diffusers.quantizers.gguf.utils"
+ assert mem._holds_torchao_weights(_ParamsModule(object(), _TorchaoLikeTensor())) is True
+ assert mem._holds_torchao_weights(_ParamsModule(_GGUFParameter(), object())) is False
+
+ class _Broken:
+ def parameters(self):
+ raise RuntimeError("no parameters")
+
+ assert mem._holds_torchao_weights(_Broken()) is False
+
+
+def test_a_refusing_encoder_never_hands_a_torchao_transformer_to_whole_module_offload(monkeypatch):
+ # Whole-module offload moves the transformer every call, which torchao weights do not survive.
+ import core.inference.diffusion_memory as mem
+
+ pipe = _RecordingPipe()
+ pipe.transformer = _ParamsModule(_TorchaoLikeTensor())
+ monkeypatch.setattr(mem, "_apply_group_offload", lambda *a, **k: False)
+ with pytest.raises(RuntimeError, match = "cannot move torchao weights"):
+ apply_memory_plan(pipe, _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB), device = "cuda")
+ assert "model_offload" not in pipe.calls
+
+
+def test_a_refusing_encoder_still_falls_back_for_a_dense_transformer(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ pipe = _RecordingPipe()
+ pipe.transformer = _ParamsModule(object())
+ monkeypatch.setattr(mem, "_apply_group_offload", lambda *a, **k: False)
+ policy, _tiling = apply_memory_plan(
+ pipe, _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB), device = "cuda"
+ )
+ assert policy == OFFLOAD_MODEL
+
+
+def test_a_late_encoder_refusal_never_streams_a_torchao_transformer(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ order: list[str] = []
+
+ def _apply(module, **kw):
+ if getattr(module, "name", "") == "text_encoder_2":
+ raise ValueError("no leaves to stream")
+ order.append("hook:" + module.name)
+
+ pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
+ _swap_group_offloading(monkeypatch, _apply)
+ monkeypatch.setattr(mem, "_pipe_denoisers_hold_torchao", lambda _pipe: True)
+ with pytest.raises(ValueError, match = "no leaves to stream"):
+ mem._apply_group_offload(
+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False
+ )
+ assert order == ["hook:text_encoder"]
+ assert te2.placed is None
diff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.py
index 6206802e794..cb12d2ef759 100644
--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py
+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py
@@ -957,6 +957,44 @@ def test_the_rebuilt_fp8_artifacts_are_listed_for_both_schemes(family, repo):
assert family_prequant_repo(fam, scheme) == repo
+def _encoders_streamed(**_k):
+ return types.SimpleNamespace(offload_policy = "group", stream_transformer = False)
+
+
+def test_an_artifact_that_fits_once_the_encoders_stream_is_seeded(monkeypatch):
+ """An artifact-sized plan streaming only the encoders keeps the torchao seed."""
+ backend = _settle_backend(monkeypatch)
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", lambda *_a, **k: _encoders_streamed(**k))
+ assert _settle(backend) == "fp8"
+
+
+def test_an_artifact_plan_that_streams_the_transformer_still_declines(monkeypatch):
+ backend = _settle_backend(monkeypatch)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_plan_memory",
+ lambda *_a, **_k: types.SimpleNamespace(offload_policy = "group", stream_transformer = True),
+ )
+ assert _settle(backend) == PIPELINE_SEED_DECLINED
+
+
+def test_a_seed_whose_load_plan_streams_only_the_encoders_is_kept(fake_runtime, monkeypatch):
+ backend, spy = _load_backend(monkeypatch, offload = "group")
+ real_plan = DiffusionBackend._plan_memory
+
+ def _plan(self, *a, **kwargs):
+ plan = real_plan(self, *a, **kwargs)
+ if kwargs.get("transformer_resident_override_mib") is not None:
+ plan.stream_transformer = False
+ return plan
+
+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)
+ status = _load(backend)
+
+ assert spy.seeds and spy.restored == []
+ assert status["transformer_quant"] == "fp8"
+
+
_MEASURED = {"safe_device_budget_mib": 170_000, "resident_required_mib": 60_000}