Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: count the hosted text encoder and keep the GGUF denoiser resident when only the encoder does not fit (#11802)
Daniel Hanea11ce28bf48
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex da2e82adc86..50f836666fe 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -4236,6 +4236,81 @@ class DiffusionBackend:staged_dir,)+ @staticmethod+ def _precast_text_encoder_mib(+ fam: Any,+ base: str,+ target: Any,+ text_encoder_quant: Optional[str],+ staged_dir: Optional[str] = None,+ ) -> Optional[tuple[int, tuple[str, ...], bool]]:+ """``(MiB, components, exact)`` for the hosted pre-cast text encoder(s) this pick loads, or None.++ Uncached checkpoints are priced from the family table (``exact=False``). Never raises."""+ try:+ from .diffusion_te_prequant import (+ TE_PREQUANT_BUDGET_SCALE,+ TE_PREQUANT_COMPONENTS,+ te_candidate_filenames,+ te_prequant_sources_for_base,+ )++ extra: dict[str, Any] = {}+ if getattr(fam, "name", None) == HIDREAM_FAMILY_NAME:+ # TE4 comes from its own repo (hidream_te4_kwargs), as in the download planner.+ extra["components"] = (*TE_PREQUANT_COMPONENTS, "text_encoder_4")+ extra["standalone_component_bases"] = {"text_encoder_4": HIDREAM_LLAMA_REPO}+ sources = te_prequant_sources_for_base(+ fam, base, te_quant_mode = text_encoder_quant, target = target, **extra+ )+ if not sources:+ return None+ total = 0+ uncached = 0+ table_priced = 0+ for component, source in sources.items():+ size = 0+ kind = getattr(source, "kind", None)+ if kind == "path":+ try:+ size = Path(str(source.location)).expanduser().stat().st_size+ except OSError:+ size = 0+ elif kind == "repo":+ names = te_candidate_filenames(source)++ def _sizes(d: Path, names = names) -> dict[str, int]:+ for name in names:+ f = d / name+ if f.is_file():+ try:+ return {"precast": f.stat().st_size}+ except OSError:+ return {}+ return {}++ size = DiffusionBackend._union_over_cached_revs(str(source.location), _sizes)+ if size > 0:+ total += int(size)+ continue+ uncached += 1+ if component == "text_encoder_4":+ # Outside the table's encoder term: priced from its own dense size.+ total += int(HIDREAM_LLAMA_BF16_BYTES * TE_PREQUANT_BUDGET_SCALE)+ else:+ table_priced += 1+ if table_priced:+ table = family_bf16_components_gb(fam, base)+ if table is None:+ return None+ share = table_priced / max(1, len(sources))+ total += int(table[1] * (1000.0**3) * TE_PREQUANT_BUDGET_SCALE * share)+ if total <= 0:+ return None+ return max(1, total // (1024 * 1024)), tuple(sources), uncached == 0+ except Exception: # noqa: BLE001 -- sizing aid only; the scanned terms stand+ return None+@staticmethoddef _safetensors_param_count(path: Path) -> int:"""Total tensor elements in a safetensors file, read from its JSON header without touching@@ -4439,6 +4514,7 @@ class DiffusionBackend:# companions.base_local_dir = _base_local_dir,fetch_base = fetch_base,+ text_encoder_quant = text_encoder_quant,)pipeline_seed_scheme: Optional[str] = Nonebf16_pipeline_plan = plan@@ -4467,6 +4543,7 @@ class DiffusionBackend:),companion_override_mib = seed_estimate.companions_mib,text_encoder_override_mib = seed_estimate.text_encoders_mib,+ text_encoder_quant = text_encoder_quant,)if seed_estimate is not Noneelse None@@ -4703,6 +4780,7 @@ class DiffusionBackend:text_encoder_override_mib = getattr(candidate, "text_encoders_mib", None),+ text_encoder_quant = text_encoder_quant,)if candidate is None:@@ -4846,6 +4924,7 @@ class DiffusionBackend:# No companion_override here, so this one reads the cache: point it at the snapshot# the load will read, not the live root alone.base_local_dir = _base_local_dir,+ text_encoder_quant = text_encoder_quant,)# On unified memory the policy cannot express a misfit (the planner returns 'none' for ANY# size there, because offload shuffles bytes within one pool) so the check above never@@ -5287,6 +5366,7 @@ class DiffusionBackend:repo_id = repo_id,base_local_dir = _base_local_dir,fetch_base = fetch_base,+ text_encoder_quant = text_encoder_quant,)bf16_pipeline_plan = planself._raise_if_load_cancelled(_load_token)@@ -5470,6 +5550,7 @@ class DiffusionBackend:),companion_override_mib = estimate.companions_mib,text_encoder_override_mib = estimate.text_encoders_mib,+ text_encoder_quant = text_encoder_quant,)if replanned.offload_policy == OFFLOAD_NONE:logger.info(@@ -6511,6 +6592,7 @@ class DiffusionBackend:base_local_dir: Optional[str] = None,fetch_base: Optional[str] = None,device_memory_override: Optional[DeviceMemory] = None,+ text_encoder_quant: Optional[str] = None,):"""Build the memory plan for this load: snapshot free device memory and estimate the model'sresident footprint, then let the planner pick an offload policy + VAE memory savers. Kept on@@ -6540,6 +6622,9 @@ class DiffusionBackend:``device_memory_override`` replaces the live reading for a plan taken BEFORE the download,where free memory still describes the OLD model; capacity bounds any later free reading.++ ``text_encoder_quant`` (resolved scheme) re-prices cache-scanned encoder terms at a hosted+ pre-cast checkpoint's size, which base-repo scans never see."""# Settled (max-over-reads) on cuda: a transient foreign allocation would make an empty card look fulldevice_memory = (@@ -6547,6 +6632,7 @@ class DiffusionBackend:if device_memory_override is not Noneelse settled_snapshot_device_memory(target))+ companions_from_cache = Falseif kind == "pipeline" and transformer_resident_override_mib is not None:# Re-planning an assembled pipeline against its dense-quant candidate. The family estimate already# splits transformer from companions; the cache scan below would price the bf16 transformer this@@ -6605,11 +6691,14 @@ class DiffusionBackend:companion_mib = int(companion // (1024 * 1024)) if companion else Nonetext_encoder = self._text_encoder_cache_bytes(fetch_base or base, base_local_dir)text_encoder_mib = int(text_encoder // (1024 * 1024)) if text_encoder else None+ companions_from_cache = Trueif is_narrow_base and table is not None:# model_dense_mib was raised to the bf16 table above; the companions upcast with it, so take their# share from the same table rather than leaving a narrow on-disk figure beside a bf16 total.companion_mib = int(sum(table[1:]) * (1000.0**3) / (1024.0 * 1024.0))text_encoder_mib = int(table[1] * (1000.0**3) / (1024.0 * 1024.0))+ # Already the dense bf16 encoders, an upper bound on any pre-cast one: swapping would add it on top.+ companions_from_cache = Falseif companion_mib is not None and model_dense_mib is not None:# The two terms come from different merges over the cache roots, so a repo only one of them can see# must not report companions larger than the model.@@ -6651,9 +6740,36 @@ class DiffusionBackend:# as nothing cached, i.e. no split.text_encoder = self._text_encoder_cache_bytes(fetch_base or base, base_local_dir)text_encoder_mib = int(text_encoder // (1024 * 1024)) if text_encoder else None+ companions_from_cache = Truemodel_dense_mib = Noneif transformer_resident is not None:model_dense_mib = transformer_resident + (companion_mib or 0)+ if companions_from_cache and text_encoder_quant is not None:+ precast = self._precast_text_encoder_mib(+ fam, base, target, text_encoder_quant, base_local_dir+ )+ if precast:+ precast_mib, precast_components, _exact = precast+ covered = frozenset(precast_components)+ scanned_te = int(+ self._union_over_cached_revs(+ fetch_base or base,+ lambda d: {+ rel: size+ for rel, size in self._local_dir_text_encoder_sizes(d).items()+ if rel.split("/", 1)[0] in covered+ },+ base_local_dir,+ )+ ) // (1024 * 1024)+ # Never below scanned dense shards: a bad checkpoint falls back to opening them.+ precast_mib = max(int(precast_mib), scanned_te)+ text_encoder_mib = max(0, int(text_encoder_mib or 0) - scanned_te) + int(+ precast_mib+ )+ companion_mib = max(0, int(companion_mib or 0) - scanned_te) + int(precast_mib)+ if model_dense_mib is not None:+ model_dense_mib = max(0, int(model_dense_mib) - scanned_te) + int(precast_mib)# Feed the variant hint so estimate_image_runtime_mib sees distilled markers (distilled needs ~15% less# headroom).variant_hint = _image_variant_hint(diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 598f60139e5..24a6018f23a 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -485,6 +485,8 @@ class MemoryPlan:# Defaulted so every existing construction is unchanged; set only where that is what makes group offload fit at# all (see plan_diffusion_memory).stream_text_encoders: bool = False+ # False only on the tier keeping the transformer resident and streaming just the text encoders.+ stream_transformer: bool = True@propertydef engages_offload(self) -> bool:@@ -500,6 +502,7 @@ class MemoryPlan:"estimates": dict(self.estimates),"reasons": list(self.reasons),"stream_text_encoders": self.stream_text_encoders,+ "stream_transformer": self.stream_transformer,}@@ -1062,8 +1065,21 @@ def plan_diffusion_memory(if companion_dense_mib is not None and text_encoder_dense_mib is not Noneelse None)+ # Floor for a resident transformer with streamed text encoders (they run once per call, not per step).+ resident_transformer_floor = (+ _sum_required(+ max(0, int(model_dense_mib) - int(text_encoder_dense_mib)),+ runtime_headroom_mib,+ base_overhead_mib,+ )+ if model_dense_mib is not None+ and text_encoder_dense_mib is not None+ and int(text_encoder_dense_mib) > 0+ else None+ )reasons: list[str] = []stream_text_encoders = False+ stream_transformer = Trueestimates: dict[str, Optional[int]] = {"safe_device_budget_mib": budget,"model_dense_mib": model_dense_mib,@@ -1074,6 +1090,7 @@ def plan_diffusion_memory("resident_required_mib": required,"group_floor_mib": group_floor,"group_floor_streamed_te_mib": group_floor_streamed_te,+ "resident_transformer_floor_mib": resident_transformer_floor,}def _group_fits() -> bool:@@ -1087,20 +1104,33 @@ def plan_diffusion_memory(and group_floor_streamed_te <= budget)+ def _resident_transformer_fits() -> bool:+ return (+ resident_transformer_floor is not None+ and budget is not None+ and resident_transformer_floor <= budget+ )+# The best tier available when the weights do not fit resident, in speed order: plain group (companions resident)# beats group with streamed encoders (one extra host-to-device pass per CALL) beats whole-module offload (every# component paged per STEP -- the 48-minute case).- def _offload_tier() -> tuple[str, bool]:+ def _offload_tier() -> tuple[str, bool, bool]:+ if _resident_transformer_fits():+ return OFFLOAD_GROUP, True, Falseif _group_fits():- return OFFLOAD_GROUP, False+ return OFFLOAD_GROUP, False, Trueif _group_fits_streamed_te():- return OFFLOAD_GROUP, True- return OFFLOAD_MODEL, False+ return OFFLOAD_GROUP, True, True+ return OFFLOAD_MODEL, False, True_STREAMED_TE_REASON = ("companions exceed budget, but they fit with the text encoders streamed too ""(they run once, before step 0); streaming them beats paging every component per step")+ _RESIDENT_TRANSFORMER_REASON = (+ "the transformer fits resident once the text encoders are streamed (they run once, "+ "before step 0); every denoise step runs at resident speed"+ )if not can_offload or device_memory.is_unified:# MPS / CPU cannot stream to a separate device; on unified memory offload just shuffles bytes within the same@@ -1113,10 +1143,11 @@ def plan_diffusion_memory(elif mode == MEMORY_MODE_FAST:policy = OFFLOAD_NONEif budget is not None and required is not None and required > budget:- # Doesn't fit resident: streamed transformer is the fastest offload.- policy, stream_text_encoders = _offload_tier()+ policy, stream_text_encoders, stream_transformer = _offload_tier()reasons.append("fast requested but weights do not fit resident; offloading")- if stream_text_encoders:+ if not stream_transformer:+ reasons.append(_RESIDENT_TRANSFORMER_REASON)+ elif stream_text_encoders:reasons.append(_STREAMED_TE_REASON)else:reasons.append("fast requested; weights resident on device")@@ -1132,6 +1163,11 @@ def plan_diffusion_memory(elif required <= int(budget * 0.85):policy = OFFLOAD_NONEreasons.append("weights fit resident with headroom")+ elif _resident_transformer_fits():+ policy = OFFLOAD_GROUP+ stream_text_encoders = True+ stream_transformer = False+ reasons.append(_RESIDENT_TRANSFORMER_REASON)elif _group_fits():policy = OFFLOAD_GROUPreasons.append("tight fit; stream the transformer, companions resident")@@ -1177,6 +1213,7 @@ def plan_diffusion_memory(reasons = tuple(reasons),# only ever meaningful under group offload; every other tier already places the encodersstream_text_encoders = stream_text_encoders and policy == OFFLOAD_GROUP,+ stream_transformer = stream_transformer or policy != OFFLOAD_GROUP,)@@ -1313,12 +1350,12 @@ def apply_memory_plan(keep_cpu_weights_on_offload(pipe, logger)elif policy == OFFLOAD_GROUP:# getattr, not attribute access: manually built / duck-typed plans predate this field.- if not _apply_group_offload(- pipe,- placement,- logger,- stream_text_encoders = bool(getattr(plan, "stream_text_encoders", False)),- ):+ group_kwargs: dict[str, Any] = {+ "stream_text_encoders": bool(getattr(plan, "stream_text_encoders", False))+ }+ if not bool(getattr(plan, "stream_transformer", True)):+ group_kwargs["stream_transformer"] = False+ if not _apply_group_offload(pipe, placement, logger, **group_kwargs):_fallback_to_model_offload()policy = OFFLOAD_MODELelif policy == OFFLOAD_STREAMING:@@ -1389,6 +1426,107 @@ def _pin_vision_embedding_device(module: Any) -> int:return patched+# ``0`` never pins the streamed-encoder tiers, ``1`` always does.+GROUP_OFFLOAD_PIN_ENV = "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN"+_PIN_RESERVE_MIN_MIB = _PIN_RESERVE_MIN_BYTES >> 20+++def _module_host_mib(module: Any) -> int:+ """Pinned host MiB for ``module``; per tensor rounded to a power of two like torch's pinned allocator."""+ try:+ seen: set[int] = set()+ total = 0+ tensors = list(module.parameters(recurse = True)) + list(module.buffers(recurse = True))+ for tensor in tensors:+ if id(tensor) in seen:+ continue+ seen.add(id(tensor))+ nbytes = int(tensor.numel()) * int(tensor.element_size())+ if nbytes > 0:+ total += 1 << (nbytes - 1).bit_length()+ return total // (1024 * 1024)+ except Exception: # noqa: BLE001 - an unsizeable module is priced as nothing to pin+ return 0+++def _pin_budget_mib() -> Optional[int]:+ """Pinnable host MiB leaving ``max(4 GiB, 15%)`` free, or None if unreadable."""+ total, _available = _system_memory_mib()+ available = _available_system_memory_mib()+ if total is None or available is None:+ return None+ # Same container sizing as _pin_host_weights: pinned pages are charged to an enforcing cgroup.+ limit = _cgroup_memory_limit_mib()+ if limit is not None:+ total = min(int(total), int(limit))+ reserve = max(_PIN_RESERVE_MIN_MIB, int(int(total) * _PIN_RESERVE_FRACTION))+ return max(0, int(available) - reserve)+++def _remove_group_offload_hooks(module: Any) -> None:+ """Undo a group offloading apply that raised part way (best effort)."""+ try:+ from diffusers.hooks import group_offloading as go+ from diffusers.hooks.hooks import HookRegistry++ registry = HookRegistry.check_if_exists_or_initialize(module)+ for name in (+ "_GROUP_OFFLOADING",+ "_LAZY_PREFETCH_GROUP_OFFLOADING",+ "_LAYER_EXECUTION_TRACKER",+ ):+ hook = getattr(go, name, None)+ if isinstance(hook, str):+ registry.remove_hook(hook, recurse = True)+ except Exception: # noqa: BLE001 - the fallback reports its own failure+ pass+++def _pinned_memory_capped() -> bool:+ """Windows (WDDM) and WSL2 cap pinned host memory near 1 GiB (NVIDIA CUDA on WSL known limitation)."""+ if sys.platform == "win32":+ return True+ try:+ with open("/proc/version", "r", encoding = "utf-8") as f:+ return "microsoft" in f.read().lower()+ except Exception: # noqa: BLE001 - not Linux, or unreadable: not WSL+ return False+++def _streamed_pin_plan(+ transformer_mib: int,+ encoder_mib: int,+ logger: Any = None,+) -> tuple[bool, bool]:+ """(pin transformer, pin encoders), transformer first."""+ forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()+ if forced in ("0", "off", "false", "no"):+ return False, False+ if forced in ("1", "on", "true", "yes"):+ return True, True+ budget = None if _pinned_memory_capped() else _pin_budget_mib()+ if budget is None or budget <= 0:+ pin_transformer = pin_encoders = False+ else:+ pin_transformer = transformer_mib <= budget+ pinned = transformer_mib if pin_transformer else 0+ pin_encoders = pinned + encoder_mib <= budget+ if logger is not None:+ try:+ logger.info(+ "diffusion.memory: streamed-encoder tier pins transformer=%s (%d MiB) encoders=%s (%d MiB) "+ "against %s MiB of pinnable host RAM",+ pin_transformer,+ transformer_mib,+ pin_encoders,+ encoder_mib,+ budget,+ )+ except Exception: # noqa: BLE001+ pass+ return pin_transformer, pin_encoders++def install_group_offload_buffer_restore() -> bool:"""diffusers stream group offload restores only parameters, leaving buffers (native int8 weights) on the GPU."""try:@@ -1430,16 +1568,21 @@ def _apply_group_offload(logger: Any,*,stream_text_encoders: bool = False,+ stream_transformer: bool = True,) -> bool:"""Stream the transformer a few blocks at a time via diffusers group offloading, keeping thesmaller components resident. Returns False (caller falls back to whole-module) on any failure.``stream_text_encoders`` extends the streamed set to every ``text_encoder*`` module. Off bydefault: keeping them resident is faster when there is room. The planner turns it on only- where it is the difference between group offload and whole-module offload."""+ where it is the difference between group offload and whole-module offload.++ ``stream_transformer=False`` (only with ``stream_text_encoders``) keeps every DiT resident."""transformer = getattr(pipe, "transformer", None)if transformer is None:return False+ if not stream_transformer and not stream_text_encoders:+ return Falseinstalled = 0try:import inspect@@ -1451,11 +1594,13 @@ def _apply_group_offload(# A dual-DiT pipeline (Ideogram 4) carries a second denoiser as large as the first, so stream every DiT and keep# only smaller companions resident.- streamed: dict[str, Any] = {"transformer": transformer}- for extra in ("transformer_2", "unconditional_transformer"):- module = getattr(pipe, extra, None)- if isinstance(module, torch.nn.Module):- streamed[extra] = module+ streamed: dict[str, Any] = {}+ if stream_transformer:+ streamed["transformer"] = transformer+ for extra in ("transformer_2", "unconditional_transformer"):+ module = getattr(pipe, extra, None)+ if isinstance(module, torch.nn.Module):+ streamed[extra] = module# The text encoders are streamed SEPARATELY from the DiTs, and tolerantly (see the apply loop below). Kept in# their own dict so the resident placement loop still skips them.streamed_encoders: dict[str, Any] = {}@@ -1486,16 +1631,17 @@ def _apply_group_offload(gkwargs["non_blocking"] = Trueif "record_stream" in _params:gkwargs["record_stream"] = True+ pin_streamed = (True, True)if stream_text_encoders and "low_cpu_mem_usage" in _params:# The streamed path PINS every offloaded parameter in host RAM when a copy stream is in use (diffusers# group_offloading `_init_cpu_param_dict`), which is a fine trade when group offload was already the plan.- # It is not a fine trade here: this tier is only ever reached as a rescue from whole-module offload, which- # pins nothing, on a card small enough that the companions did not fit. Those hosts are not reliably- # RAM-rich either, and silently converting a device-memory shortfall into ten-plus GB of unswappable host- # RAM is how #8188's machine got into trouble in the first place. low_cpu_mem_usage trades a slower- # host-to-device copy for not pinning; the encoders this tier streams run ONCE per call, so that copy is- # paid once, not per step.- gkwargs["low_cpu_mem_usage"] = True+ # Pin only what host RAM covers (unbounded pinning hurt #8188); unpinned re-pins on every onload.+ pin_streamed = _streamed_pin_plan(+ sum(_module_host_mib(m) for m in streamed.values()),+ sum(_module_host_mib(m) for m in streamed_encoders.values()),+ logger,+ )+ gkwargs["low_cpu_mem_usage"] = not pin_streamed[0]# Place the smaller components resident BEFORE attaching the transformer group-offload hooks: a companion .to()# OOM then returns False with no hooks installed, and diffusers rejects enable_model_cpu_offload once group# hooks exist.@@ -1520,12 +1666,36 @@ def _apply_group_offload(# plan said leaf and the application said block. num_blocks_per_group goes with it: leaf level has no blocks.ekwargs = {k: v for k, v in gkwargs.items() if k != "num_blocks_per_group"}ekwargs["offload_type"] = "leaf_level"+ if "low_cpu_mem_usage" in gkwargs:+ ekwargs["low_cpu_mem_usage"] = not pin_streamed[1]+ transformer_demoted = Falsefor name, module in streamed_encoders.items():try:apply_group_offloading(module, **ekwargs)installed += 1_pin_vision_embedding_device(module)except Exception as exc: # noqa: BLE001 -- degrade this encoder, never fail the load+ if not stream_transformer and installed == 0:+ # Resident encoder here would OOM; fall back to model offload, which rejects partial hooks.+ _remove_group_offload_hooks(module)+ raise+ if not stream_transformer and not transformer_demoted:+ # Model offload is gone once hooks exist: stream the transformer before this encoder goes resident.+ for dit_name in ("transformer", "transformer_2", "unconditional_transformer"):+ dit = getattr(pipe, dit_name, None)+ if isinstance(dit, torch.nn.Module):+ dkwargs = dict(gkwargs)+ if "low_cpu_mem_usage" in _params and use_stream:+ dkwargs["low_cpu_mem_usage"] = True+ apply_group_offloading(dit, **dkwargs)+ installed += 1+ transformer_demoted = True+ if logger is not None:+ logger.warning(+ "diffusion.memory: %s refused group offload on the resident-transformer tier; "+ "streaming the transformer instead",+ name,+ )if logger is not None:logger.warning("diffusion.memory: group offload unavailable for %s (%s); "diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex 4b5a2590a26..9806f601433 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -10716,8 +10716,9 @@ def test_plan_memory_hands_the_planner_the_text_encoder_split(monkeypatch, tmp_passert plan.estimates["companion_dense_mib"] == 200assert plan.estimates["text_encoder_dense_mib"] == 150assert plan.estimates["group_floor_streamed_te_mib"] == 2198- # The companions fit as they are, so the cheaper tier still wins and nothing streams.- assert plan.offload_policy == OFFLOAD_GROUP and plan.stream_text_encoders is False+ assert plan.estimates["resident_transformer_floor_mib"] == 2498+ assert plan.offload_policy == OFFLOAD_GROUP+ assert plan.stream_text_encoders is True and plan.stream_transformer is Falsedef test_plan_memory_streams_the_text_encoders_instead_of_offloading_everything(@@ -12067,6 +12068,325 @@ def test_a_prequant_repo_missing_its_artifact_marks_the_plan_incomplete(monkeypaassert "prequant artifact missing" in str(failures[0])+def test_plan_memory_prices_the_hosted_precast_text_encoder(monkeypatch, tmp_path):+ snapshot = _base_snapshot_with_sizes(+ tmp_path, monkeypatch, {"vae/diffusion_pytorch_model.safetensors": 50}+ )+ target = _small_card(monkeypatch)+ seen = {}++ def _precast(+ fam,+ base,+ tgt,+ text_encoder_quant,+ staged_dir = None,+ ):+ seen["quant"] = text_encoder_quant+ return (2800, ("text_encoder",), True) if text_encoder_quant == "fp8" else None++ monkeypatch.setattr(DiffusionBackend, "_precast_text_encoder_mib", staticmethod(_precast))++ def _plan(**kw):+ return DiffusionBackend()._plan_memory(+ target,+ None,+ "bfl/base",+ types.SimpleNamespace(name = "flux.1"),+ None,+ False,+ kind = "gguf",+ transformer_resident_override_mib = 300,+ base_local_dir = str(snapshot),+ **kw,+ )++ before = _plan()+ assert before.estimates["companion_dense_mib"] == 50+ assert before.estimates["text_encoder_dense_mib"] is None+ after = _plan(text_encoder_quant = "fp8")+ assert seen["quant"] == "fp8"+ assert after.estimates["text_encoder_dense_mib"] == 2800+ assert after.estimates["companion_dense_mib"] == 2850+ assert after.estimates["model_dense_mib"] == 3150+++def test_plan_memory_prices_cached_dense_shards_over_a_cached_precast(monkeypatch, tmp_path):+ snapshot = _base_snapshot_with_sizes(+ tmp_path,+ monkeypatch,+ {+ "text_encoder/model.safetensors": 4000,+ "vae/diffusion_pytorch_model.safetensors": 50,+ },+ )+ target = _small_card(monkeypatch)+ monkeypatch.setattr(+ DiffusionBackend,+ "_precast_text_encoder_mib",+ staticmethod(lambda *a, **k: (2600, ("text_encoder",), True)),+ )+ plan = DiffusionBackend()._plan_memory(+ target,+ None,+ "bfl/base",+ types.SimpleNamespace(name = "flux.1"),+ None,+ False,+ kind = "gguf",+ transformer_resident_override_mib = 300,+ base_local_dir = str(snapshot),+ text_encoder_quant = "fp8",+ )+ assert plan.estimates["text_encoder_dense_mib"] == 4000+ assert plan.estimates["companion_dense_mib"] == 4050+ assert plan.estimates["model_dense_mib"] == 4350+++def test_cached_dense_shards_keep_the_24g_denoiser_resident(monkeypatch, tmp_path):+ from core.inference import diffusion as dmod+ from core.inference.diffusion_memory import OFFLOAD_GROUP, DeviceMemory++ snapshot = _base_snapshot_with_sizes(+ tmp_path,+ monkeypatch,+ {+ "text_encoder/model.safetensors": 16689,+ "vae/diffusion_pytorch_model.safetensors": 1288,+ },+ )+ monkeypatch.setattr(+ dmod,+ "settled_snapshot_device_memory",+ lambda t: DeviceMemory("cuda", "cuda", "discrete_vram", 23000, 24576),+ )+ monkeypatch.setattr(dmod, "estimate_image_runtime_mib", lambda **kw: 8192)+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", supports_model_cpu_offload = True)+ monkeypatch.setattr(+ DiffusionBackend,+ "_precast_text_encoder_mib",+ staticmethod(lambda *a, **k: (8959, ("text_encoder",), True)),+ )+ plan = DiffusionBackend()._plan_memory(+ target,+ None,+ "bfl/base",+ types.SimpleNamespace(name = "qwen-image"),+ None,+ False,+ kind = "gguf",+ transformer_resident_override_mib = 7650,+ base_local_dir = str(snapshot),+ text_encoder_quant = "fp8",+ )+ assert plan.estimates["text_encoder_dense_mib"] == 16689+ assert plan.offload_policy == OFFLOAD_GROUP+ assert plan.stream_transformer is False and plan.stream_text_encoders is True+++def _flux_like_plan(tmp_path, monkeypatch, precast):+ snapshot = _base_snapshot_with_sizes(+ tmp_path,+ monkeypatch,+ {+ "text_encoder/model.safetensors": 235,+ "text_encoder_2/model.safetensors": 4000,+ "vae/diffusion_pytorch_model.safetensors": 50,+ },+ )+ target = _small_card(monkeypatch)+ monkeypatch.setattr(+ DiffusionBackend, "_precast_text_encoder_mib", staticmethod(lambda *a, **k: precast)+ )+ return DiffusionBackend()._plan_memory(+ target,+ None,+ "bfl/base",+ types.SimpleNamespace(name = "flux.1"),+ None,+ False,+ kind = "gguf",+ transformer_resident_override_mib = 300,+ base_local_dir = str(snapshot),+ text_encoder_quant = "fp8",+ )+++def test_plan_memory_swaps_only_the_encoder_the_precast_checkpoint_replaces(monkeypatch, tmp_path):+ plan = _flux_like_plan(tmp_path, monkeypatch, (4200, ("text_encoder_2",), True))+ assert plan.estimates["text_encoder_dense_mib"] == 235 + 4200+ assert plan.estimates["companion_dense_mib"] == 50 + 235 + 4200+++def test_plan_memory_never_prices_an_uncached_precast_below_the_dense_shards(monkeypatch, tmp_path):+ plan = _flux_like_plan(tmp_path, monkeypatch, (2600, ("text_encoder_2",), False))+ assert plan.estimates["text_encoder_dense_mib"] == 235 + 4000+++def test_a_table_priced_pipeline_does_not_add_the_precast_encoder_on_top(monkeypatch, tmp_path):+ # A base-repo pipeline prices its encoders from the bf16 table, which already covers the dense encoder.+ snapshot = _base_snapshot_with_sizes(+ tmp_path,+ monkeypatch,+ {+ "transformer/diffusion_pytorch_model.safetensors": 13500,+ "vae/diffusion_pytorch_model.safetensors": 1288,+ },+ )+ target = _small_card(monkeypatch)+ monkeypatch.setattr(+ DiffusionBackend,+ "_precast_text_encoder_mib",+ staticmethod(lambda *a, **k: (8959, ("text_encoder",), True)),+ )+ fam = types.SimpleNamespace(name = "qwen-image-2.1", base_repo = "bfl/base")++ def _plan(quant):+ return (+ DiffusionBackend()+ ._plan_memory(+ target,+ None,+ "bfl/base",+ fam,+ None,+ False,+ kind = "pipeline",+ repo_id = "bfl/base",+ base_local_dir = str(snapshot),+ text_encoder_quant = quant,+ )+ .estimates+ )++ dense, precast = _plan(None), _plan("fp8")+ assert dense["text_encoder_dense_mib"] == 16689+ for key in ("model_dense_mib", "companion_dense_mib", "text_encoder_dense_mib"):+ assert precast[key] == dense[key]+++def test_plan_memory_leaves_a_callers_companion_override_alone(monkeypatch, tmp_path):+ target = _small_card(monkeypatch)+ monkeypatch.setattr(+ DiffusionBackend, "_precast_text_encoder_mib", staticmethod(lambda *a, **k: 2600)+ )+ plan = DiffusionBackend()._plan_memory(+ target,+ None,+ "bfl/base",+ types.SimpleNamespace(name = "flux.1"),+ None,+ False,+ kind = "gguf",+ transformer_resident_override_mib = 300,+ companion_override_mib = 700,+ text_encoder_override_mib = 600,+ text_encoder_quant = "fp8",+ )+ assert plan.estimates["companion_dense_mib"] == 700+ assert plan.estimates["text_encoder_dense_mib"] == 600+++def _q21_precast_cache(tmp_path, monkeypatch, *, mib):+ """unsloth/Qwen-Image-2.1-FP8 cached under the live root, holding only the pre-cast encoder."""+ live, _other = _split_cache_roots(tmp_path, monkeypatch)+ repo = live / "models--unsloth--Qwen-Image-2.1-FP8"+ rev = "b" * 40+ (repo / "refs").mkdir(parents = True)+ (repo / "refs" / "main").write_text(rev)+ snap = repo / "snapshots" / rev+ snap.mkdir(parents = True)+ if mib:+ with open(snap / "Qwen-Image-2.1-text_encoder-FP8.safetensors", "wb") as fh:+ fh.truncate(mib * 1024 * 1024)+ return snap+++def _bf16_cuda_target():+ import torch+ return types.SimpleNamespace(+ device = "cuda", backend = "cuda", dtype = torch.bfloat16, supports_model_cpu_offload = True+ )+++def test_precast_text_encoder_mib_reads_the_cached_checkpoint(monkeypatch, tmp_path):+ from core.inference.diffusion_families import detect_family++ fam = detect_family("Qwen/Qwen-Image-2.1")+ assert fam is not None and fam.name == "qwen-image-2.1"+ _q21_precast_cache(tmp_path, monkeypatch, mib = 8959)+ target = _bf16_cuda_target()+ assert DiffusionBackend._precast_text_encoder_mib(+ fam, "Qwen/Qwen-Image-2.1", target, "fp8"+ ) == (+ 8959,+ ("text_encoder",),+ True,+ )+ assert (+ DiffusionBackend._precast_text_encoder_mib(fam, "Qwen/Qwen-Image-2.1", target, None) is None+ )+ assert (+ DiffusionBackend._precast_text_encoder_mib(fam, "Qwen/Qwen-Image-2.1", target, "none")+ is None+ )+++def test_precast_text_encoder_mib_prices_hidreams_standalone_fourth_encoder(monkeypatch, tmp_path):+ from core.inference.diffusion_families import detect_family+ from core.inference.diffusion_te_prequant import te_candidate_filenames, te_prequant_sources++ fam = detect_family("HiDream-ai/HiDream-I1-Full")+ assert fam is not None and fam.name == "hidream-i1"+ target = _bf16_cuda_target()+ source = te_prequant_sources(+ fam, te_quant_mode = "fp8", target = target, components = ("text_encoder_4",)+ )["text_encoder_4"]+ live, _other = _split_cache_roots(tmp_path, monkeypatch)+ repo = live / ("models--" + source.location.replace("/", "--"))+ rev = "c" * 40+ (repo / "refs").mkdir(parents = True)+ (repo / "refs" / "main").write_text(rev)+ snap = repo / "snapshots" / rev+ snap.mkdir(parents = True)+ with open(snap / te_candidate_filenames(source)[0], "wb") as fh:+ fh.truncate(7700 * 1024 * 1024)+ assert DiffusionBackend._precast_text_encoder_mib(+ fam, "HiDream-ai/HiDream-I1-Full", target, "fp8"+ ) == (7700, ("text_encoder_4",), True)++ (snap / te_candidate_filenames(source)[0]).unlink()+ from core.inference.diffusion_hidream import HIDREAM_LLAMA_BF16_BYTES+ from core.inference.diffusion_te_prequant import TE_PREQUANT_BUDGET_SCALE++ assert DiffusionBackend._precast_text_encoder_mib(+ fam, "HiDream-ai/HiDream-I1-Full", target, "fp8"+ ) == (+ int(HIDREAM_LLAMA_BF16_BYTES * TE_PREQUANT_BUDGET_SCALE) // (1024 * 1024),+ ("text_encoder_4",),+ False,+ )+++def test_precast_text_encoder_mib_prices_an_uncached_checkpoint_from_the_family_table(+ monkeypatch, tmp_path+):+ from core.inference.diffusion_families import detect_family+ from core.inference.diffusion_te_prequant import TE_PREQUANT_BUDGET_SCALE++ fam = detect_family("Qwen/Qwen-Image-2.1")+ _q21_precast_cache(tmp_path, monkeypatch, mib = 0)+ got = DiffusionBackend._precast_text_encoder_mib(+ fam, "Qwen/Qwen-Image-2.1", _bf16_cuda_target(), "fp8"+ )+ assert got == (+ int(17.5 * 1000**3 * TE_PREQUANT_BUDGET_SCALE) // (1024 * 1024),+ ("text_encoder",),+ False,+ )+ assert got[0] > 8959++def _stub_amd_weight_only_host(backend, monkeypatch):"""ROCm / torchao-stub host; records the quantise calls."""from core.inference import diffusion as dmoddiff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.pyindex ff77828225e..0892cf6de0c 100644--- a/studio/backend/tests/test_diffusion_memory.py+++ b/studio/backend/tests/test_diffusion_memory.py@@ -1628,7 +1628,10 @@ def test_streaming_the_text_encoders_does_not_pin_host_memory(monkeypatch):# whole-module offload, which pins nothing, on a card too small to hold the companions. Those# hosts are not reliably RAM-rich, and #8188's machine got into trouble precisely by turning a# device shortfall into unswappable host memory. The encoders run once per call, so the slower- # unpinned copy is paid once rather than per step.+ import core.inference.diffusion_memory as mem++ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 0)seen = _stream_te_kwargs(monkeypatch, stream_text_encoders = True)assert seen, "the applier never reached diffusers"assert all(kw["low_cpu_mem_usage"] is True for kw in seen.values()), seen@@ -2117,3 +2120,343 @@ def test_the_same_load_reaches_group_offload_once_the_split_is_known():)assert plan.offload_policy == OFFLOAD_GROUPassert plan.vae_tiling is False+++_Q21_TRANSFORMER_MIB = 7_650+_Q21_VAE_MIB = 1_288+_Q21_TE_MIB = 8_959+_24G_FREE_MIB = 23_000+_24G_TOTAL_MIB = 24_576 # budget 23000 - 2457 = 20543+++def _q21_plan(+ free_mib,+ total_mib,+ *,+ text_encoder_dense_mib = _Q21_TE_MIB,+ mode = None,+):+ return plan_diffusion_memory(+ target = _target(),+ device_memory = _discrete(free_mib, total_mib),+ model_dense_mib = _Q21_TRANSFORMER_MIB + _Q21_VAE_MIB + _Q21_TE_MIB,+ companion_dense_mib = _Q21_VAE_MIB + _Q21_TE_MIB,+ text_encoder_dense_mib = text_encoder_dense_mib,+ runtime_headroom_mib = 8192,+ base_overhead_mib = 2048,+ requested_mode = mode,+ )+++def test_the_transformer_stays_resident_when_only_the_encoder_does_not_fit():+ plan = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB)+ assert plan.estimates["resident_transformer_floor_mib"] == 19_178+ assert plan.offload_policy == OFFLOAD_GROUP+ assert plan.stream_transformer is False and plan.stream_text_encoders is True+ assert plan.estimates["group_floor_mib"] <= plan.estimates["safe_device_budget_mib"]+ assert plan.as_public_dict()["stream_transformer"] is False+++def test_the_resident_transformer_tier_needs_the_encoder_split():+ plan = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB, text_encoder_dense_mib = None)+ assert plan.estimates["resident_transformer_floor_mib"] is None+ assert plan.offload_policy == OFFLOAD_GROUP+ assert plan.stream_transformer is True and plan.stream_text_encoders is False+++def test_a_card_too_small_for_the_resident_transformer_still_streams_it():+ plan = _q21_plan(_16G_FREE_MIB, _16G_TOTAL_MIB)+ assert plan.offload_policy == OFFLOAD_GROUP+ assert plan.stream_transformer is True and plan.stream_text_encoders is True+++def test_fast_mode_prefers_the_resident_transformer_over_streaming_it():+ plan = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB, mode = MEMORY_MODE_FAST)+ assert plan.offload_policy == OFFLOAD_GROUP+ assert plan.stream_transformer is False and plan.stream_text_encoders is True+++def test_every_other_tier_reports_a_streamed_or_resident_transformer_consistently():+ roomy = _q21_plan(80_000, 81_920)+ assert roomy.offload_policy == OFFLOAD_NONE and roomy.stream_transformer is True+ tiny = _q21_plan(6_000, 8_192)+ assert tiny.offload_policy == OFFLOAD_MODEL and tiny.stream_transformer is True+ balanced = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB, mode = MEMORY_MODE_BALANCED)+ assert balanced.offload_policy == OFFLOAD_GROUP and balanced.stream_transformer is True+++def test_apply_group_offload_can_keep_the_transformer_resident(monkeypatch):+ import core.inference.diffusion_memory as mem++ pipe, applied, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)+ assert (+ mem._apply_group_offload(+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False+ )+ is True+ )+ assert applied == [te, te2]+ assert transformer.placed is not None and vae.placed is not None+ assert te.placed is None and te2.placed is None+++def test_the_resident_transformer_tier_streams_encoders_at_leaf_level(monkeypatch):+ seen = _stream_te_kwargs(monkeypatch, stream_text_encoders = True, stream_transformer = False)+ assert "transformer" not in seen+ assert seen["text_encoder"]["offload_type"] == "leaf_level"+ assert seen["text_encoder_2"]["offload_type"] == "leaf_level"+++def test_a_refusing_encoder_hands_the_resident_transformer_tier_to_whole_module_offload(+ monkeypatch,+):+ import core.inference.diffusion_memory as mem++ def _apply(module, **kw):+ raise ValueError("no leaves to stream")++ pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)+ _swap_group_offloading(monkeypatch, _apply)+ assert (+ mem._apply_group_offload(+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False+ )+ is False+ )+++def test_apply_group_offload_refuses_a_plan_that_streams_nothing(monkeypatch):+ import core.inference.diffusion_memory as mem++ pipe, applied, *_ = _stream_te_pipe(monkeypatch)+ assert mem._apply_group_offload(pipe, "cuda", logger = None, stream_transformer = False) is False+ assert applied == []+++def test_apply_memory_plan_threads_the_resident_transformer_flag(monkeypatch):+ import core.inference.diffusion_memory as mem++ seen = {}++ def _fake(pipe, device, logger, **kw):+ seen.clear()+ seen.update(kw)+ return True++ monkeypatch.setattr(mem, "_apply_group_offload", _fake)+ policy, _tiling = apply_memory_plan(+ _RecordingPipe(), _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB), device = "cuda"+ )+ assert policy == OFFLOAD_GROUP+ assert seen == {"stream_text_encoders": True, "stream_transformer": False}+ apply_memory_plan(_RecordingPipe(), _q21_plan(_16G_FREE_MIB, _16G_TOTAL_MIB), device = "cuda")+ assert seen == {"stream_text_encoders": True}+++def _sized_stream_te_kwargs(monkeypatch, sizes, budget, **call_kw):+ import core.inference.diffusion_memory as mem++ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False, raising = False)+ monkeypatch.setattr(mem, "_module_host_mib", lambda m: sizes.get(getattr(m, "name", ""), 0))+ return _stream_te_kwargs(monkeypatch, **call_kw)+++def test_a_ram_rich_host_pins_everything_the_tier_streams(monkeypatch):+ seen = _sized_stream_te_kwargs(+ monkeypatch,+ {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},+ 40_000,+ stream_text_encoders = True,+ )+ assert all(kw["low_cpu_mem_usage"] is False for kw in seen.values()), seen+++def test_the_transformer_is_pinned_before_the_encoders(monkeypatch):+ seen = _sized_stream_te_kwargs(+ monkeypatch,+ {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},+ 10_000,+ stream_text_encoders = True,+ )+ assert seen["transformer"]["low_cpu_mem_usage"] is False+ assert seen["text_encoder"]["low_cpu_mem_usage"] is True+ assert seen["text_encoder_2"]["low_cpu_mem_usage"] is True+++def test_the_resident_transformer_tier_pins_its_encoders_when_ram_allows(monkeypatch):+ seen = _sized_stream_te_kwargs(+ monkeypatch,+ {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},+ 10_000,+ stream_text_encoders = True,+ stream_transformer = False,+ )+ assert "transformer" not in seen+ assert seen["text_encoder"]["low_cpu_mem_usage"] is False+ assert seen["text_encoder_2"]["low_cpu_mem_usage"] is False++[email protected]("stream_transformer", [True, False])+def test_windows_and_wsl_never_pin_the_streamed_tiers(monkeypatch, stream_transformer):+ import core.inference.diffusion_memory as mem++ seen = _sized_stream_te_kwargs(+ monkeypatch,+ {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},+ 1_000_000,+ stream_text_encoders = True,+ stream_transformer = stream_transformer,+ )+ assert all(+ kw["low_cpu_mem_usage"] is False for kw in seen.values()+ ), seen # control: RAM-rich Linux pins+ monkeypatch.delattr(mem, "_pinned_memory_capped", raising = False)+ monkeypatch.undo()+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1_000_000)+ monkeypatch.setattr(mem.sys, "platform", "win32")+ seen = _stream_te_kwargs(+ monkeypatch, stream_text_encoders = True, stream_transformer = stream_transformer+ )+ assert seen and all(kw["low_cpu_mem_usage"] is True for kw in seen.values()), seen+ monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, "1")+ assert mem._streamed_pin_plan(7000, 8000) == (True, True)+++def test_a_first_encoder_refusal_is_unhooked_before_whole_module_offload(monkeypatch):+ import core.inference.diffusion_memory as mem++ def _apply(module, **kw):+ if getattr(module, "name", "") == "text_encoder":+ raise RuntimeError("CUDA error: out of memory (pin_memory)")++ pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)+ _swap_group_offloading(monkeypatch, _apply)+ unhooked: list[str] = []+ monkeypatch.setattr(+ mem, "_remove_group_offload_hooks", lambda m: unhooked.append(m.name), raising = False+ )++ assert (+ mem._apply_group_offload(+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False+ )+ is False+ )+ assert unhooked == ["text_encoder"]+++def test_pinned_memory_cap_detection(monkeypatch, tmp_path):+ import builtins++ import core.inference.diffusion_memory as mem++ monkeypatch.setattr(mem.sys, "platform", "win32")+ assert mem._pinned_memory_capped() is True+ monkeypatch.setattr(mem.sys, "platform", "linux")+ real_open = builtins.open+ for text, capped in (+ ("Linux version 6.6.87.2-microsoft-standard-WSL2 (gcc)", True),+ ("Linux version 6.17.0-1007-aws (gcc)", False),+ ):+ monkeypatch.setattr(+ builtins,+ "open",+ lambda path, *a, _t = text, **k: (+ __import__("io").StringIO(_t)+ if path == "/proc/version"+ else real_open(path, *a, **k)+ ),+ )+ assert mem._pinned_memory_capped() is capped+++def test_unknown_host_memory_never_pins(monkeypatch):+ seen = _sized_stream_te_kwargs(+ monkeypatch, {"transformer": 1, "text_encoder": 1}, None, stream_text_encoders = True+ )+ assert all(kw["low_cpu_mem_usage"] is True for kw in seen.values()), seen++[email protected]("value, pinned", [("0", True), ("1", False)])+def test_the_pin_env_overrides_the_ram_gate(monkeypatch, value, pinned):+ import core.inference.diffusion_memory as mem++ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 40_000 if value == "0" else 0)+ monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, value)+ seen = _stream_te_kwargs(monkeypatch, stream_text_encoders = True)+ assert all(kw["low_cpu_mem_usage"] is pinned for kw in seen.values()), seen+++def test_the_pin_budget_leaves_the_reserve_free(monkeypatch):+ import core.inference.diffusion_memory as mem++ monkeypatch.setattr(mem, "_cgroup_memory_limit_mib", lambda: None)+ monkeypatch.setattr(mem, "_system_memory_mib", lambda: (65_536, 30_000))+ monkeypatch.setattr(mem, "_available_system_memory_mib", lambda: 30_000)+ assert mem._pin_budget_mib() == 30_000 - int(65_536 * 0.15)+ # A 64 GiB container on a 512 GiB host: the reserve comes from the container, as _pin_host_weights does.+ monkeypatch.setattr(mem, "_cgroup_memory_limit_mib", lambda: 65_536)+ monkeypatch.setattr(mem, "_system_memory_mib", lambda: (524_288, 60_000))+ monkeypatch.setattr(mem, "_available_system_memory_mib", lambda: 60_000)+ assert mem._pin_budget_mib() == 60_000 - int(65_536 * 0.15)+ monkeypatch.setattr(mem, "_cgroup_memory_limit_mib", lambda: None)+ monkeypatch.setattr(mem, "_system_memory_mib", lambda: (16_384, 3_000))+ monkeypatch.setattr(mem, "_available_system_memory_mib", lambda: 3_000)+ assert mem._pin_budget_mib() == 0+ monkeypatch.setattr(mem, "_system_memory_mib", lambda: (None, None))+ assert mem._pin_budget_mib() is None+++def test_a_late_encoder_refusal_streams_the_transformer_before_placing_the_encoder(monkeypatch):+ import core.inference.diffusion_memory as mem++ order: list[str] = []++ def _apply(module, **kw):+ if getattr(module, "name", "") == "text_encoder_2":+ raise ValueError("no leaves to stream")+ order.append("hook:" + module.name)++ pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)+ _swap_group_offloading(monkeypatch, _apply)+ original_to = type(te2).to++ def _to(self, device):+ if self is te2:+ order.append("place:text_encoder_2")+ return original_to(self, device)++ monkeypatch.setattr(type(te2), "to", _to)+ assert (+ mem._apply_group_offload(+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False+ )+ is True+ )+ assert order == ["hook:text_encoder", "hook:transformer", "place:text_encoder_2"]+++def test_pinned_host_pricing_rounds_each_tensor_to_a_power_of_two(monkeypatch):+ import core.inference.diffusion_memory as mem++ class _T:+ def __init__(self, nbytes):+ self._n = nbytes++ def numel(self):+ return self._n++ def element_size(self):+ return 1++ class _M:+ def parameters(self, recurse = True):+ return [_T(3 * 1024 * 1024), _T(5 * 1024 * 1024)]++ def buffers(self, recurse = True):+ return [_T(1024 * 1024)]++ assert mem._module_host_mib(_M()) == 13