Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run Qwen-Image-2.1 int8/fp8 on 24 GB cards with the transformer resident and the text encoder streamed (#11883)
Daniel Hanf2a9375055f2
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 5338fd1b92b..57b33faa4f1 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -111,6 +111,7 @@ from .diffusion_memory import (normalize_memory_mode,plan_diffusion_memory,plan_fits_total_capacity,+ plan_keeps_transformer_resident,raise_on_image_activation_shortfall,raise_on_unified_memory_shortfall,reclaimable_snapshot_device_memory,@@ -1337,6 +1338,19 @@ def _no_recast_pipeline_class(pipe_cls: Any) -> Any:_NO_WEIGHT = object()+def _torchao_offload_decline(what: str, plan: Any) -> str:+ """Why a torchao build was declined under ``plan``, quoting the floor that keeps it resident."""+ estimates = getattr(plan, "estimates", None) or {}+ floor = estimates.get("resident_transformer_floor_mib")+ streamed = f", {floor} MiB even with the text encoders streamed" if floor is not None else ""+ return (+ f"{what} '{plan.offload_policy}' offload here "+ f"({estimates.get('resident_required_mib')} MiB required vs a "+ f"{estimates.get('safe_device_budget_mib')} MiB budget{streamed}), "+ "and torchao tensors cannot be offloaded"+ )++def _has_active_lora(loras: Any) -> bool:"""True when any adapter would actually be baked, for either shape the callers pass. Weight 0means disabled, but /images/load passes ``(id, weight)`` tuples while /images/download-plan@@ -3235,15 +3249,16 @@ class DiffusionBackend:repo_id = repo_id,fetch_base = fetch_base,transformer_resident_override_mib = candidate.steady_transformer_mib,- companion_override_mib = candidate.companions_mib,- text_encoder_override_mib = candidate.text_encoders_mib,+ **self._candidate_companion_overrides(+ candidate, fam, base or repo_id or "", target, text_encoder_quant+ ),device_memory_override = replace(memory, free_mib = memory.total_mib),)- if planned.offload_policy != OFFLOAD_NONE:+ if not plan_keeps_transformer_resident(planned):logger.info(- "diffusion.denoiser_prequant: an artifact-sized plan for %s still offloads "- "on this card, and offload moves the denoiser via Module.to(), so the "- "released shards are kept",+ "diffusion.denoiser_prequant: an artifact-sized plan for %s streams the "+ "denoiser on this card, and offload moves the denoiser via Module.to(), "+ "so the released shards are kept",rung,)declined = True@@ -4690,14 +4705,15 @@ class DiffusionBackend:transformer_resident_override_mib = (seed_estimate.steady_transformer_mib),- companion_override_mib = seed_estimate.companions_mib,- text_encoder_override_mib = seed_estimate.text_encoders_mib,+ **self._candidate_companion_overrides(+ seed_estimate, fam, base, target, text_encoder_quant+ ),text_encoder_quant = text_encoder_quant,)if seed_estimate is not Noneelse None)- if seeded_plan is None or seeded_plan.offload_policy != OFFLOAD_NONE:+ if seeded_plan is None or not plan_keeps_transformer_resident(seeded_plan):# Offload hooks use Module.to(), which torchao tensors reject, and live free# memory can undercut the CAPACITY the plan settled this against.logger.info(@@ -4973,13 +4989,10 @@ class DiffusionBackend:transformer_resident_override_mib = (candidate.transient_transformer_mib),- # Pass the companion estimate so prefetched base shards aren't double-counted.- companion_override_mib = candidate.companions_mib,- # ... and its text-encoder share, so the planner can still price the streamed-encoder- # group tier on this path. getattr: a candidate without the split passes None and- # keeps the previous decision.- text_encoder_override_mib = getattr(- candidate, "text_encoders_mib", None+ # Companion estimate avoids double-counting prefetched shards; its encoder share+ # keeps the streamed-encoder tiers priceable.+ **self._candidate_companion_overrides(+ candidate, fam, base, target, text_encoder_quant),text_encoder_quant = text_encoder_quant,)@@ -4995,7 +5008,7 @@ class DiffusionBackend:if candidate is not None:replanned = _replan_candidate()if (- replanned.offload_policy != OFFLOAD_NONE+ not plan_keeps_transformer_resident(replanned)# Explicit balanced/low_vram picks offload BY MODE, so a fresh snapshot cannot change# it.and normalize_memory_mode(memory_mode)@@ -5006,7 +5019,7 @@ class DiffusionBackend:# (settled) and replan once rather than letting a transient foreign allocation force# the GGUF fallback.replanned = _replan_candidate()- if replanned.offload_policy != OFFLOAD_NONE:+ if not plan_keeps_transformer_resident(replanned):logger.info("diffusion.transformer_quant_declined: required=%s MiB ""budget=%s MiB free=%s MiB policy=%s (%s)",@@ -5016,15 +5029,10 @@ class DiffusionBackend:replanned.offload_policy,"; ".join(replanned.reasons),)- transformer_quant_decline = (- f"the quantised build still needs '{replanned.offload_policy}' "- f"offload here ("- f"{replanned.estimates.get('resident_required_mib')} MiB "- f"required vs a "- f"{replanned.estimates.get('safe_device_budget_mib')} MiB "- "budget), and torchao tensors cannot be offloaded"+ transformer_quant_decline = _torchao_offload_decline(+ "the quantised build still needs", replanned)- if replanned.offload_policy == OFFLOAD_NONE:+ if plan_keeps_transformer_resident(replanned):quant_plan = replanned# The GGUF plan declined resident; a prequant-sized replan says nothing about the# dense build@@ -5064,7 +5072,7 @@ class DiffusionBackend:if retry_candidate is not None:candidate = retry_candidateretry_plan = _replan_candidate()- if retry_plan.offload_policy == OFFLOAD_NONE:+ if plan_keeps_transformer_resident(retry_plan):logger.info("diffusion.transformer_quant: auto's pick does not fit ""resident; retrying at %s, whose checkpoint is cached",@@ -5229,6 +5237,55 @@ class DiffusionBackend:"loading the GGUF",prequant_candidate.transient_transformer_mib,)+ # The INT8 artifact outweighs the GGUF, so plan it on its own; a plan streaming the+ # transformer cannot carry torchao.+ if (+ prequant is not None+ and not dense_declined+ and getattr(plan.device_memory, "memory_kind", None) != "unified_memory"+ ):+ sized_candidate = resolve_dense_quant_candidate(+ fam = fam,+ target = target,+ requested = transformer_quant,+ base_repo = base,+ prequant_path = transformer_prequant_path,+ force_dense = False,+ logger = logger,+ )+ if sized_candidate is not None and sized_candidate.prequant:+ sized_plan = self._plan_memory(+ target,+ single_file_path,+ base,+ fam,+ memory_mode,+ cpu_offload,+ kind = kind,+ repo_id = repo_id,+ fetch_base = fetch_base,+ transformer_resident_override_mib = (+ sized_candidate.transient_transformer_mib+ ),+ **self._candidate_companion_overrides(+ sized_candidate, fam, base, target, text_encoder_quant+ ),+ text_encoder_quant = text_encoder_quant,+ )+ if plan_keeps_transformer_resident(sized_plan):+ quant_plan = sized_plan+ if sized_plan.offload_policy != OFFLOAD_NONE:+ dense_fallback_allowed = False+ else:+ dense_declined = True+ dense_fallback_allowed = False+ transformer_quant_decline = _torchao_offload_decline(+ "the pre-quantised transformer needs", sized_plan+ )+ logger.info(+ "diffusion.transformer_quant_declined: %s",+ transformer_quant_decline,+ )# A dense misfit with a prequant source only forbids the dense fallback; with none, auto could# still have picked a DIFFERENT scheme that does have a hosted checkpoint. auto returns one# winner, and a winner with no published prequant that also cannot build dense would drop the@@ -5276,7 +5333,7 @@ class DiffusionBackend:retry_candidate.transient_transformer_mib),# Same pre-cast encoder pricing as the fit check above.- companion_override_mib = self._precast_scaled_companions_mib(+ **self._candidate_companion_overrides(retry_candidate,fam,base,@@ -5289,7 +5346,7 @@ class DiffusionBackend:)if (retry_plan is not None- and retry_plan.offload_policy == OFFLOAD_NONE+ and plan_keeps_transformer_resident(retry_plan)# Same unified caveat as above: 'none' is not a fit there, so the rung being retried# has to be sized explicitly before it is pinned.and unified_memory_shortfall_message(@@ -5750,21 +5807,25 @@ class DiffusionBackend:transformer_resident_override_mib = (estimate.steady_transformer_mib),- companion_override_mib = estimate.companions_mib,- text_encoder_override_mib = estimate.text_encoders_mib,+ **self._candidate_companion_overrides(+ estimate, fam, base, target, text_encoder_quant+ ),text_encoder_quant = text_encoder_quant,)- if replanned.offload_policy == OFFLOAD_NONE:+ if plan_keeps_transformer_resident(replanned):logger.info("diffusion.transformer_quant: %s fits resident (%d MiB "- "steady); dropping the bf16 plan's '%s' offload",+ "steady, encoders streamed=%s); replacing the bf16 plan's "+ "'%s' offload",preview_scheme,estimate.steady_transformer_mib,+ replanned.offload_policy != OFFLOAD_NONE,plan.offload_policy,)plan = replanned+ keeps_resident = plan_keeps_transformer_resident(plan)if (- plan.offload_policy != OFFLOAD_NONE+ not keeps_residentand native_scheme is Noneand native_offload_scheme is not None):@@ -5775,7 +5836,7 @@ class DiffusionBackend:native_scheme,plan.offload_policy,)- if plan.offload_policy != OFFLOAD_NONE and native_scheme is None:+ if not keeps_resident and native_scheme is None:logger.info("diffusion.transformer_quant: skipped (the memory plan picked '%s' ""offload, which moves the transformer via Module.to())",@@ -6763,6 +6824,29 @@ class DiffusionBackend:except Exception: # noqa: BLE001 -- unreadable cache: treat the shards as missingreturn False+ @staticmethod+ def _candidate_companion_overrides(+ candidate: Any,+ fam: DiffusionFamily,+ base: str,+ target: Any,+ text_encoder_quant: Optional[str],+ ) -> dict[str, Optional[int]]:+ """``_plan_memory`` overrides for a dense-quant candidate, pricing the encoder the load actually opens."""+ companions = DiffusionBackend._precast_scaled_companions_mib(+ candidate, fam, base, target, text_encoder_quant+ )+ encoders = getattr(candidate, "text_encoders_mib", None)+ if encoders and companions is not None:+ raw = getattr(candidate, "companions_mib", None)+ if raw is not None:+ # companions - VAE is the priced encoder share; the VAE term is unscaled in both.+ encoders = max(0, int(encoders) - (int(raw) - int(companions)))+ return {+ "companion_override_mib": companions,+ "text_encoder_override_mib": encoders,+ }+@staticmethoddef _precast_scaled_companions_mib(candidate: Any,diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex d058f05ded3..b14cb9d56f8 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -886,6 +886,34 @@ def _safe_device_budget_mib(memory: DeviceMemory) -> Optional[int]:return max(0, int(memory.free_mib) - _reserve_mib(memory.memory_kind, base))+def plan_keeps_transformer_resident(plan: Any) -> bool:+ """Whether ``plan`` never moves the denoiser after placement: torchao survives placement, not per-forward hooks."""+ policy = getattr(plan, "offload_policy", OFFLOAD_NONE)+ if policy == OFFLOAD_NONE:+ return True+ return policy == OFFLOAD_GROUP and not bool(getattr(plan, "stream_transformer", True))+++def _holds_torchao_weights(module: Any) -> bool:+ """Whether any parameter of ``module`` is a torchao tensor subclass (GGUF and native int8 are not)."""+ try:+ for param in module.parameters():+ for tensor in (param, getattr(param, "data", None)):+ if tensor is not None and type(tensor).__module__.startswith("torchao"):+ return True+ except Exception: # noqa: BLE001 - an unreadable module is treated as movable, today's behaviour+ return False+ return False+++def _pipe_denoisers_hold_torchao(pipe: Any) -> bool:+ return any(+ _holds_torchao_weights(getattr(pipe, name, None))+ for name in ("transformer", "transformer_2", "unconditional_transformer")+ if getattr(pipe, name, None) is not None+ )++def plan_fits_total_capacity(plan: Any) -> bool:"""Whether ``plan``'s resident requirement fits TOTAL device capacity under the standardreserve + the 0.85 resident margin -- i.e. an offload decision can only stem from the@@ -1351,6 +1379,11 @@ def apply_memory_plan(if not bool(getattr(plan, "stream_transformer", True)):group_kwargs["stream_transformer"] = Falseif not _apply_group_offload(pipe, placement, logger, **group_kwargs):+ if "stream_transformer" in group_kwargs and _pipe_denoisers_hold_torchao(pipe):+ raise RuntimeError(+ "the text encoder could not be streamed beside the resident quantised "+ "transformer, and whole-module offload cannot move torchao weights"+ )_fallback_to_model_offload()policy = OFFLOAD_MODELelif policy == OFFLOAD_STREAMING:@@ -1675,6 +1708,8 @@ def _apply_group_offload(_remove_group_offload_hooks(module)raiseif not stream_transformer and not transformer_demoted:+ if _pipe_denoisers_hold_torchao(pipe):+ raise# Model offload is gone once hooks exist: stream the transformer before this encoder goes resident.for dit_name in ("transformer", "transformer_2", "unconditional_transformer"):dit = getattr(pipe, dit_name, None)diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex 502180ab115..4b3f329c6c4 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -13059,3 +13059,242 @@ def test_status_reports_cuda_graph_off_once_every_armed_step_ran_eager():st = backend.status()assert st["resolved"]["cuda_graph"]["value"] == "on"assert st["speed_optims"] == ["compiled", "cuda_graph"]+++def _resident_transformer(plan):+ """``plan`` as the tier that keeps the denoiser resident and streams only the text encoders."""+ return dataclasses.replace(+ plan, offload_policy = "group", stream_text_encoders = True, stream_transformer = False+ )+++def _record_placement(monkeypatch):+ from core.inference import diffusion as dmod++ placed: list = []++ def _apply(pipe, plan, **kwargs):+ placed.append(plan)+ return plan.offload_policy, False++ monkeypatch.setattr(dmod, "apply_memory_plan", _apply)+ return placed+++def test_a_pipeline_quantises_where_only_the_encoders_stream(fake_runtime, tmp_path, monkeypatch):+ """torchao weights fit once the encoders stream, so the transformer is converted and placed once."""+ backend = DiffusionBackend()+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)+ real_plan = DiffusionBackend._plan_memory++ def _plan(self, *args, **kwargs):+ plan = real_plan(self, *args, **kwargs)+ if kwargs.get("transformer_resident_override_mib") is not None:+ return _resident_transformer(plan)+ return dataclasses.replace(plan, offload_policy = "group")++ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)+ placed = _record_placement(monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "fp8",+ _base_local_dir = str(tmp_path),+ )+ assert len(calls) == 1+ assert status["transformer_quant"] == "fp8"+ assert status["offload_policy"] == "group"+ assert placed and placed[-1].stream_transformer is False+ backend.unload()+++def test_a_pipeline_quant_replan_prices_the_encoder_the_load_opens(+ fake_runtime, tmp_path, monkeypatch+):+ """The in-place replan prices encoders like the other candidate replans, not the family table."""+ backend = DiffusionBackend()+ _stub_pipeline_dense_quant(backend, monkeypatch)+ priced = {"companion_override_mib": 1234, "text_encoder_override_mib": 567}+ monkeypatch.setattr(+ DiffusionBackend,+ "_candidate_companion_overrides",+ staticmethod(lambda *args, **kwargs: dict(priced)),+ )+ real_plan = DiffusionBackend._plan_memory+ seen = []++ def _plan(self, *args, **kwargs):+ plan = real_plan(self, *args, **kwargs)+ if kwargs.get("transformer_resident_override_mib") is not None:+ seen.append(kwargs)+ return _resident_transformer(plan)+ return dataclasses.replace(plan, offload_policy = "group")++ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)+ _record_placement(monkeypatch)+ backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "fp8",+ _base_local_dir = str(tmp_path),+ )+ assert seen+ assert seen[-1]["companion_override_mib"] == 1234+ assert seen[-1]["text_encoder_override_mib"] == 567+ backend.unload()+++def test_a_pipeline_whose_quantised_plan_still_streams_the_transformer_stays_dense(+ fake_runtime, tmp_path, monkeypatch+):+ backend = DiffusionBackend()+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)+ real_plan = DiffusionBackend._plan_memory++ def _plan(self, *args, **kwargs):+ return dataclasses.replace(real_plan(self, *args, **kwargs), offload_policy = "group")++ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)+ _record_placement(monkeypatch)+ with pytest.raises(RuntimeError) as excinfo:+ backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "fp8",+ _base_local_dir = str(tmp_path),+ )+ assert calls == []+ assert "Module.to()" in str(excinfo.value)+++def _gguf_candidate_backend(monkeypatch, tmp_path, *, initial_policy, candidate_plan):+ """GGUF load planned as ``initial_policy`` whose candidate plans as ``candidate_plan(real_plan)``."""+ from core.inference import diffusion as dmod++ backend = _cuda_backend(tmp_path, monkeypatch)+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda target: True)+ monkeypatch.setattr(+ dmod, "select_transformer_quant_scheme", lambda target, mode, family = None: "int8"+ )+ monkeypatch.setattr(dmod, "usable_prequant_source", lambda fam, scheme, **kw: "prequant/path")+ monkeypatch.setattr(+ dmod,+ "resolve_dense_quant_candidate",+ lambda **kw: types.SimpleNamespace(+ transient_transformer_mib = 7_000,+ steady_transformer_mib = 7_000,+ companions_mib = 18_000,+ text_encoders_mib = 16_700,+ prequant = True,+ ),+ )+ orig_plan = DiffusionBackend._plan_memory++ def spy_plan(+ self,+ *a,+ transformer_resident_override_mib = None,+ **k,+ ):+ real = orig_plan(+ self, *a, transformer_resident_override_mib = transformer_resident_override_mib, **k+ )+ if transformer_resident_override_mib is None:+ return dataclasses.replace(real, offload_policy = initial_policy)+ return candidate_plan(real)++ monkeypatch.setattr(DiffusionBackend, "_plan_memory", spy_plan)+ attempted: list = []++ def fake_dense_load(self, *a, **k):+ attempted.append(k.get("allow_dense_fallback"))+ raise RuntimeError("test: stop after reaching the fast path")++ monkeypatch.setattr(DiffusionBackend, "_load_dense_quant_pipeline", fake_dense_load)+ return backend, attempted+++def test_a_gguf_pick_builds_the_quant_where_only_the_encoders_stream(+ fake_runtime, tmp_path, monkeypatch, allow_precision_fallback+):+ backend, attempted = _gguf_candidate_backend(+ monkeypatch, tmp_path, initial_policy = "group", candidate_plan = _resident_transformer+ )+ _load_m(backend, tmp_path, transformer_quant = "int8")+ assert attempted == [False]+++def test_a_gguf_pick_declines_the_quant_where_the_transformer_would_stream(+ fake_runtime, tmp_path, monkeypatch, allow_precision_fallback+):+ backend, attempted = _gguf_candidate_backend(+ monkeypatch,+ tmp_path,+ initial_policy = "group",+ candidate_plan = lambda real: dataclasses.replace(real, offload_policy = "group"),+ )+ status = _load_m(backend, tmp_path, transformer_quant = "int8")+ assert attempted == []+ assert status["transformer_quant"] is None+++def test_a_resident_gguf_plan_sizes_the_prequant_that_replaces_it(+ fake_runtime, tmp_path, monkeypatch, allow_precision_fallback+):+ """The INT8 artifact outgrows the GGUF, so it loads under its own plan rather than the GGUF's."""+ sized: list = []++ def _candidate(real):+ plan = _resident_transformer(real)+ sized.append(plan)+ return plan++ backend, attempted = _gguf_candidate_backend(+ monkeypatch, tmp_path, initial_policy = "none", candidate_plan = _candidate+ )+ placed = _record_placement(monkeypatch)+ monkeypatch.setattr(+ DiffusionBackend,+ "_load_dense_quant_pipeline",+ lambda self, *a, **k: attempted.append(k.get("allow_dense_fallback")) or (None, None),+ )+ _load_m(backend, tmp_path, transformer_quant = "int8")+ assert sized, "the prequant was never sized"+ assert attempted == [False]+ assert placed[-1].offload_policy in ("none", "group")+++def test_a_resident_gguf_plan_declines_a_prequant_that_would_stream(+ fake_runtime, tmp_path, monkeypatch, allow_precision_fallback+):+ backend, attempted = _gguf_candidate_backend(+ monkeypatch,+ tmp_path,+ initial_policy = "none",+ candidate_plan = lambda real: dataclasses.replace(real, offload_policy = "group"),+ )+ status = _load_m(backend, tmp_path, transformer_quant = "int8")+ assert attempted == []+ assert status["transformer_quant"] is None+ assert (+ "torchao tensors cannot be offloaded" in (status["resolved"]["transformer_quant"]["reason"])+ )+++def test_candidate_overrides_price_the_encoder_the_load_opens(monkeypatch):+ candidate = types.SimpleNamespace(companions_mib = 18_000, text_encoders_mib = 16_000)+ monkeypatch.setattr(+ DiffusionBackend,+ "_precast_scaled_companions_mib",+ staticmethod(lambda cand, fam, base, target, teq: 2_000 + int(16_000 * 0.65)),+ )+ overrides = DiffusionBackend._candidate_companion_overrides(candidate, None, "b", None, "fp8")+ assert overrides == {"companion_override_mib": 12_400, "text_encoder_override_mib": 10_400}+ # No split: the encoder term stays unknown, so the planner keeps its previous tiers.+ bare = types.SimpleNamespace(companions_mib = 18_000, text_encoders_mib = 0)+ assert (+ DiffusionBackend._candidate_companion_overrides(bare, None, "b", None, None)[+ "text_encoder_override_mib"+ ]+ == 0+ )diff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.pyindex 0892cf6de0c..11ed0915b52 100644--- a/studio/backend/tests/test_diffusion_memory.py+++ b/studio/backend/tests/test_diffusion_memory.py@@ -2460,3 +2460,97 @@ def test_pinned_host_pricing_rounds_each_tensor_to_a_power_of_two(monkeypatch):return [_T(1024 * 1024)]assert mem._module_host_mib(_M()) == 13+++def test_only_resident_and_encoder_only_plans_keep_the_transformer_in_place():+ from types import SimpleNamespace++ import core.inference.diffusion_memory as mem++ keeps = mem.plan_keeps_transformer_resident+ assert keeps(SimpleNamespace(offload_policy = OFFLOAD_NONE)) is True+ assert keeps(SimpleNamespace(offload_policy = OFFLOAD_GROUP, stream_transformer = False)) is True+ assert keeps(SimpleNamespace(offload_policy = OFFLOAD_GROUP, stream_transformer = True)) is False+ # Plans built before the field existed stream the transformer.+ assert keeps(SimpleNamespace(offload_policy = OFFLOAD_GROUP)) is False+ for policy in (OFFLOAD_MODEL, mem.OFFLOAD_STREAMING, mem.OFFLOAD_SEQUENTIAL):+ assert keeps(SimpleNamespace(offload_policy = policy, stream_transformer = False)) is False+ assert keeps(_q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB)) is True+ assert keeps(_q21_plan(_16G_FREE_MIB, _16G_TOTAL_MIB)) is False+++class _TorchaoLikeTensor:+ """Stands in for a torchao subclass: only its defining module is read."""+++_TorchaoLikeTensor.__module__ = "torchao.quantization.linear_activation_quantized_tensor"+++class _ParamsModule:+ def __init__(self, *tensors):+ self._tensors = tensors++ def parameters(self):+ return iter(self._tensors)+++def test_torchao_weights_are_told_apart_from_gguf_and_plain_ones():+ import core.inference.diffusion_memory as mem++ class _GGUFParameter:+ pass++ _GGUFParameter.__module__ = "diffusers.quantizers.gguf.utils"+ assert mem._holds_torchao_weights(_ParamsModule(object(), _TorchaoLikeTensor())) is True+ assert mem._holds_torchao_weights(_ParamsModule(_GGUFParameter(), object())) is False++ class _Broken:+ def parameters(self):+ raise RuntimeError("no parameters")++ assert mem._holds_torchao_weights(_Broken()) is False+++def test_a_refusing_encoder_never_hands_a_torchao_transformer_to_whole_module_offload(monkeypatch):+ # Whole-module offload moves the transformer every call, which torchao weights do not survive.+ import core.inference.diffusion_memory as mem++ pipe = _RecordingPipe()+ pipe.transformer = _ParamsModule(_TorchaoLikeTensor())+ monkeypatch.setattr(mem, "_apply_group_offload", lambda *a, **k: False)+ with pytest.raises(RuntimeError, match = "cannot move torchao weights"):+ apply_memory_plan(pipe, _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB), device = "cuda")+ assert "model_offload" not in pipe.calls+++def test_a_refusing_encoder_still_falls_back_for_a_dense_transformer(monkeypatch):+ import core.inference.diffusion_memory as mem++ pipe = _RecordingPipe()+ pipe.transformer = _ParamsModule(object())+ monkeypatch.setattr(mem, "_apply_group_offload", lambda *a, **k: False)+ policy, _tiling = apply_memory_plan(+ pipe, _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB), device = "cuda"+ )+ assert policy == OFFLOAD_MODEL+++def test_a_late_encoder_refusal_never_streams_a_torchao_transformer(monkeypatch):+ import core.inference.diffusion_memory as mem++ order: list[str] = []++ def _apply(module, **kw):+ if getattr(module, "name", "") == "text_encoder_2":+ raise ValueError("no leaves to stream")+ order.append("hook:" + module.name)++ pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)+ _swap_group_offloading(monkeypatch, _apply)+ monkeypatch.setattr(mem, "_pipe_denoisers_hold_torchao", lambda _pipe: True)+ with pytest.raises(ValueError, match = "no leaves to stream"):+ mem._apply_group_offload(+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False+ )+ assert order == ["hook:text_encoder"]+ assert te2.placed is Nonediff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.pyindex 6206802e794..cb12d2ef759 100644--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py@@ -957,6 +957,44 @@ def test_the_rebuilt_fp8_artifacts_are_listed_for_both_schemes(family, repo):assert family_prequant_repo(fam, scheme) == repo+def _encoders_streamed(**_k):+ return types.SimpleNamespace(offload_policy = "group", stream_transformer = False)+++def test_an_artifact_that_fits_once_the_encoders_stream_is_seeded(monkeypatch):+ """An artifact-sized plan streaming only the encoders keeps the torchao seed."""+ backend = _settle_backend(monkeypatch)+ monkeypatch.setattr(DiffusionBackend, "_plan_memory", lambda *_a, **k: _encoders_streamed(**k))+ assert _settle(backend) == "fp8"+++def test_an_artifact_plan_that_streams_the_transformer_still_declines(monkeypatch):+ backend = _settle_backend(monkeypatch)+ monkeypatch.setattr(+ DiffusionBackend,+ "_plan_memory",+ lambda *_a, **_k: types.SimpleNamespace(offload_policy = "group", stream_transformer = True),+ )+ assert _settle(backend) == PIPELINE_SEED_DECLINED+++def test_a_seed_whose_load_plan_streams_only_the_encoders_is_kept(fake_runtime, monkeypatch):+ backend, spy = _load_backend(monkeypatch, offload = "group")+ real_plan = DiffusionBackend._plan_memory++ def _plan(self, *a, **kwargs):+ plan = real_plan(self, *a, **kwargs)+ if kwargs.get("transformer_resident_override_mib") is not None:+ plan.stream_transformer = False+ return plan++ monkeypatch.setattr(DiffusionBackend, "_plan_memory", _plan)+ status = _load(backend)++ assert spy.seeds and spy.restored == []+ assert status["transformer_quant"] == "fp8"++_MEASURED = {"safe_device_budget_mib": 170_000, "resident_required_mib": 60_000}