Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: count the hosted text encoder and keep the GGUF denoiser resident when only the encoder does not fit (#11802)

Daniel Hanea11ce28bf48
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index da2e82adc86..50f836666fe 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -4236,6 +4236,81 @@ class DiffusionBackend:
staged_dir,
)
+ @staticmethod
+ def _precast_text_encoder_mib(
+ fam: Any,
+ base: str,
+ target: Any,
+ text_encoder_quant: Optional[str],
+ staged_dir: Optional[str] = None,
+ ) -> Optional[tuple[int, tuple[str, ...], bool]]:
+ """``(MiB, components, exact)`` for the hosted pre-cast text encoder(s) this pick loads, or None.
+
+ Uncached checkpoints are priced from the family table (``exact=False``). Never raises."""
+ try:
+ from .diffusion_te_prequant import (
+ TE_PREQUANT_BUDGET_SCALE,
+ TE_PREQUANT_COMPONENTS,
+ te_candidate_filenames,
+ te_prequant_sources_for_base,
+ )
+
+ extra: dict[str, Any] = {}
+ if getattr(fam, "name", None) == HIDREAM_FAMILY_NAME:
+ # TE4 comes from its own repo (hidream_te4_kwargs), as in the download planner.
+ extra["components"] = (*TE_PREQUANT_COMPONENTS, "text_encoder_4")
+ extra["standalone_component_bases"] = {"text_encoder_4": HIDREAM_LLAMA_REPO}
+ sources = te_prequant_sources_for_base(
+ fam, base, te_quant_mode = text_encoder_quant, target = target, **extra
+ )
+ if not sources:
+ return None
+ total = 0
+ uncached = 0
+ table_priced = 0
+ for component, source in sources.items():
+ size = 0
+ kind = getattr(source, "kind", None)
+ if kind == "path":
+ try:
+ size = Path(str(source.location)).expanduser().stat().st_size
+ except OSError:
+ size = 0
+ elif kind == "repo":
+ names = te_candidate_filenames(source)
+
+ def _sizes(d: Path, names = names) -> dict[str, int]:
+ for name in names:
+ f = d / name
+ if f.is_file():
+ try:
+ return {"precast": f.stat().st_size}
+ except OSError:
+ return {}
+ return {}
+
+ size = DiffusionBackend._union_over_cached_revs(str(source.location), _sizes)
+ if size > 0:
+ total += int(size)
+ continue
+ uncached += 1
+ if component == "text_encoder_4":
+ # Outside the table's encoder term: priced from its own dense size.
+ total += int(HIDREAM_LLAMA_BF16_BYTES * TE_PREQUANT_BUDGET_SCALE)
+ else:
+ table_priced += 1
+ if table_priced:
+ table = family_bf16_components_gb(fam, base)
+ if table is None:
+ return None
+ share = table_priced / max(1, len(sources))
+ total += int(table[1] * (1000.0**3) * TE_PREQUANT_BUDGET_SCALE * share)
+ if total <= 0:
+ return None
+ return max(1, total // (1024 * 1024)), tuple(sources), uncached == 0
+ except Exception: # noqa: BLE001 -- sizing aid only; the scanned terms stand
+ return None
+
@staticmethod
def _safetensors_param_count(path: Path) -> int:
"""Total tensor elements in a safetensors file, read from its JSON header without touching
@@ -4439,6 +4514,7 @@ class DiffusionBackend:
# companions.
base_local_dir = _base_local_dir,
fetch_base = fetch_base,
+ text_encoder_quant = text_encoder_quant,
)
pipeline_seed_scheme: Optional[str] = None
bf16_pipeline_plan = plan
@@ -4467,6 +4543,7 @@ class DiffusionBackend:
),
companion_override_mib = seed_estimate.companions_mib,
text_encoder_override_mib = seed_estimate.text_encoders_mib,
+ text_encoder_quant = text_encoder_quant,
)
if seed_estimate is not None
else None
@@ -4703,6 +4780,7 @@ class DiffusionBackend:
text_encoder_override_mib = getattr(
candidate, "text_encoders_mib", None
),
+ text_encoder_quant = text_encoder_quant,
)
if candidate is None:
@@ -4846,6 +4924,7 @@ class DiffusionBackend:
# No companion_override here, so this one reads the cache: point it at the snapshot
# the load will read, not the live root alone.
base_local_dir = _base_local_dir,
+ text_encoder_quant = text_encoder_quant,
)
# On unified memory the policy cannot express a misfit (the planner returns 'none' for ANY
# size there, because offload shuffles bytes within one pool) so the check above never
@@ -5287,6 +5366,7 @@ class DiffusionBackend:
repo_id = repo_id,
base_local_dir = _base_local_dir,
fetch_base = fetch_base,
+ text_encoder_quant = text_encoder_quant,
)
bf16_pipeline_plan = plan
self._raise_if_load_cancelled(_load_token)
@@ -5470,6 +5550,7 @@ class DiffusionBackend:
),
companion_override_mib = estimate.companions_mib,
text_encoder_override_mib = estimate.text_encoders_mib,
+ text_encoder_quant = text_encoder_quant,
)
if replanned.offload_policy == OFFLOAD_NONE:
logger.info(
@@ -6511,6 +6592,7 @@ class DiffusionBackend:
base_local_dir: Optional[str] = None,
fetch_base: Optional[str] = None,
device_memory_override: Optional[DeviceMemory] = None,
+ text_encoder_quant: Optional[str] = None,
):
"""Build the memory plan for this load: snapshot free device memory and estimate the model's
resident footprint, then let the planner pick an offload policy + VAE memory savers. Kept on
@@ -6540,6 +6622,9 @@ class DiffusionBackend:
``device_memory_override`` replaces the live reading for a plan taken BEFORE the download,
where free memory still describes the OLD model; capacity bounds any later free reading.
+
+ ``text_encoder_quant`` (resolved scheme) re-prices cache-scanned encoder terms at a hosted
+ pre-cast checkpoint's size, which base-repo scans never see.
"""
# Settled (max-over-reads) on cuda: a transient foreign allocation would make an empty card look full
device_memory = (
@@ -6547,6 +6632,7 @@ class DiffusionBackend:
if device_memory_override is not None
else settled_snapshot_device_memory(target)
)
+ companions_from_cache = False
if kind == "pipeline" and transformer_resident_override_mib is not None:
# Re-planning an assembled pipeline against its dense-quant candidate. The family estimate already
# splits transformer from companions; the cache scan below would price the bf16 transformer this
@@ -6605,11 +6691,14 @@ class DiffusionBackend:
companion_mib = int(companion // (1024 * 1024)) if companion else None
text_encoder = self._text_encoder_cache_bytes(fetch_base or base, base_local_dir)
text_encoder_mib = int(text_encoder // (1024 * 1024)) if text_encoder else None
+ companions_from_cache = True
if is_narrow_base and table is not None:
# model_dense_mib was raised to the bf16 table above; the companions upcast with it, so take their
# share from the same table rather than leaving a narrow on-disk figure beside a bf16 total.
companion_mib = int(sum(table[1:]) * (1000.0**3) / (1024.0 * 1024.0))
text_encoder_mib = int(table[1] * (1000.0**3) / (1024.0 * 1024.0))
+ # Already the dense bf16 encoders, an upper bound on any pre-cast one: swapping would add it on top.
+ companions_from_cache = False
if companion_mib is not None and model_dense_mib is not None:
# The two terms come from different merges over the cache roots, so a repo only one of them can see
# must not report companions larger than the model.
@@ -6651,9 +6740,36 @@ class DiffusionBackend:
# as nothing cached, i.e. no split.
text_encoder = self._text_encoder_cache_bytes(fetch_base or base, base_local_dir)
text_encoder_mib = int(text_encoder // (1024 * 1024)) if text_encoder else None
+ companions_from_cache = True
model_dense_mib = None
if transformer_resident is not None:
model_dense_mib = transformer_resident + (companion_mib or 0)
+ if companions_from_cache and text_encoder_quant is not None:
+ precast = self._precast_text_encoder_mib(
+ fam, base, target, text_encoder_quant, base_local_dir
+ )
+ if precast:
+ precast_mib, precast_components, _exact = precast
+ covered = frozenset(precast_components)
+ scanned_te = int(
+ self._union_over_cached_revs(
+ fetch_base or base,
+ lambda d: {
+ rel: size
+ for rel, size in self._local_dir_text_encoder_sizes(d).items()
+ if rel.split("/", 1)[0] in covered
+ },
+ base_local_dir,
+ )
+ ) // (1024 * 1024)
+ # Never below scanned dense shards: a bad checkpoint falls back to opening them.
+ precast_mib = max(int(precast_mib), scanned_te)
+ text_encoder_mib = max(0, int(text_encoder_mib or 0) - scanned_te) + int(
+ precast_mib
+ )
+ companion_mib = max(0, int(companion_mib or 0) - scanned_te) + int(precast_mib)
+ if model_dense_mib is not None:
+ model_dense_mib = max(0, int(model_dense_mib) - scanned_te) + int(precast_mib)
# Feed the variant hint so estimate_image_runtime_mib sees distilled markers (distilled needs ~15% less
# headroom).
variant_hint = _image_variant_hint(
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 598f60139e5..24a6018f23a 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -485,6 +485,8 @@ class MemoryPlan:
# Defaulted so every existing construction is unchanged; set only where that is what makes group offload fit at
# all (see plan_diffusion_memory).
stream_text_encoders: bool = False
+ # False only on the tier keeping the transformer resident and streaming just the text encoders.
+ stream_transformer: bool = True
@property
def engages_offload(self) -> bool:
@@ -500,6 +502,7 @@ class MemoryPlan:
"estimates": dict(self.estimates),
"reasons": list(self.reasons),
"stream_text_encoders": self.stream_text_encoders,
+ "stream_transformer": self.stream_transformer,
}
@@ -1062,8 +1065,21 @@ def plan_diffusion_memory(
if companion_dense_mib is not None and text_encoder_dense_mib is not None
else None
)
+ # Floor for a resident transformer with streamed text encoders (they run once per call, not per step).
+ resident_transformer_floor = (
+ _sum_required(
+ max(0, int(model_dense_mib) - int(text_encoder_dense_mib)),
+ runtime_headroom_mib,
+ base_overhead_mib,
+ )
+ if model_dense_mib is not None
+ and text_encoder_dense_mib is not None
+ and int(text_encoder_dense_mib) > 0
+ else None
+ )
reasons: list[str] = []
stream_text_encoders = False
+ stream_transformer = True
estimates: dict[str, Optional[int]] = {
"safe_device_budget_mib": budget,
"model_dense_mib": model_dense_mib,
@@ -1074,6 +1090,7 @@ def plan_diffusion_memory(
"resident_required_mib": required,
"group_floor_mib": group_floor,
"group_floor_streamed_te_mib": group_floor_streamed_te,
+ "resident_transformer_floor_mib": resident_transformer_floor,
}
def _group_fits() -> bool:
@@ -1087,20 +1104,33 @@ def plan_diffusion_memory(
and group_floor_streamed_te <= budget
)
+ def _resident_transformer_fits() -> bool:
+ return (
+ resident_transformer_floor is not None
+ and budget is not None
+ and resident_transformer_floor <= budget
+ )
+
# The best tier available when the weights do not fit resident, in speed order: plain group (companions resident)
# beats group with streamed encoders (one extra host-to-device pass per CALL) beats whole-module offload (every
# component paged per STEP -- the 48-minute case).
- def _offload_tier() -> tuple[str, bool]:
+ def _offload_tier() -> tuple[str, bool, bool]:
+ if _resident_transformer_fits():
+ return OFFLOAD_GROUP, True, False
if _group_fits():
- return OFFLOAD_GROUP, False
+ return OFFLOAD_GROUP, False, True
if _group_fits_streamed_te():
- return OFFLOAD_GROUP, True
- return OFFLOAD_MODEL, False
+ return OFFLOAD_GROUP, True, True
+ return OFFLOAD_MODEL, False, True
_STREAMED_TE_REASON = (
"companions exceed budget, but they fit with the text encoders streamed too "
"(they run once, before step 0); streaming them beats paging every component per step"
)
+ _RESIDENT_TRANSFORMER_REASON = (
+ "the transformer fits resident once the text encoders are streamed (they run once, "
+ "before step 0); every denoise step runs at resident speed"
+ )
if not can_offload or device_memory.is_unified:
# MPS / CPU cannot stream to a separate device; on unified memory offload just shuffles bytes within the same
@@ -1113,10 +1143,11 @@ def plan_diffusion_memory(
elif mode == MEMORY_MODE_FAST:
policy = OFFLOAD_NONE
if budget is not None and required is not None and required > budget:
- # Doesn't fit resident: streamed transformer is the fastest offload.
- policy, stream_text_encoders = _offload_tier()
+ policy, stream_text_encoders, stream_transformer = _offload_tier()
reasons.append("fast requested but weights do not fit resident; offloading")
- if stream_text_encoders:
+ if not stream_transformer:
+ reasons.append(_RESIDENT_TRANSFORMER_REASON)
+ elif stream_text_encoders:
reasons.append(_STREAMED_TE_REASON)
else:
reasons.append("fast requested; weights resident on device")
@@ -1132,6 +1163,11 @@ def plan_diffusion_memory(
elif required <= int(budget * 0.85):
policy = OFFLOAD_NONE
reasons.append("weights fit resident with headroom")
+ elif _resident_transformer_fits():
+ policy = OFFLOAD_GROUP
+ stream_text_encoders = True
+ stream_transformer = False
+ reasons.append(_RESIDENT_TRANSFORMER_REASON)
elif _group_fits():
policy = OFFLOAD_GROUP
reasons.append("tight fit; stream the transformer, companions resident")
@@ -1177,6 +1213,7 @@ def plan_diffusion_memory(
reasons = tuple(reasons),
# only ever meaningful under group offload; every other tier already places the encoders
stream_text_encoders = stream_text_encoders and policy == OFFLOAD_GROUP,
+ stream_transformer = stream_transformer or policy != OFFLOAD_GROUP,
)
@@ -1313,12 +1350,12 @@ def apply_memory_plan(
keep_cpu_weights_on_offload(pipe, logger)
elif policy == OFFLOAD_GROUP:
# getattr, not attribute access: manually built / duck-typed plans predate this field.
- if not _apply_group_offload(
- pipe,
- placement,
- logger,
- stream_text_encoders = bool(getattr(plan, "stream_text_encoders", False)),
- ):
+ group_kwargs: dict[str, Any] = {
+ "stream_text_encoders": bool(getattr(plan, "stream_text_encoders", False))
+ }
+ if not bool(getattr(plan, "stream_transformer", True)):
+ group_kwargs["stream_transformer"] = False
+ if not _apply_group_offload(pipe, placement, logger, **group_kwargs):
_fallback_to_model_offload()
policy = OFFLOAD_MODEL
elif policy == OFFLOAD_STREAMING:
@@ -1389,6 +1426,107 @@ def _pin_vision_embedding_device(module: Any) -> int:
return patched
+# ``0`` never pins the streamed-encoder tiers, ``1`` always does.
+GROUP_OFFLOAD_PIN_ENV = "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN"
+_PIN_RESERVE_MIN_MIB = _PIN_RESERVE_MIN_BYTES >> 20
+
+
+def _module_host_mib(module: Any) -> int:
+ """Pinned host MiB for ``module``; per tensor rounded to a power of two like torch's pinned allocator."""
+ try:
+ seen: set[int] = set()
+ total = 0
+ tensors = list(module.parameters(recurse = True)) + list(module.buffers(recurse = True))
+ for tensor in tensors:
+ if id(tensor) in seen:
+ continue
+ seen.add(id(tensor))
+ nbytes = int(tensor.numel()) * int(tensor.element_size())
+ if nbytes > 0:
+ total += 1 << (nbytes - 1).bit_length()
+ return total // (1024 * 1024)
+ except Exception: # noqa: BLE001 - an unsizeable module is priced as nothing to pin
+ return 0
+
+
+def _pin_budget_mib() -> Optional[int]:
+ """Pinnable host MiB leaving ``max(4 GiB, 15%)`` free, or None if unreadable."""
+ total, _available = _system_memory_mib()
+ available = _available_system_memory_mib()
+ if total is None or available is None:
+ return None
+ # Same container sizing as _pin_host_weights: pinned pages are charged to an enforcing cgroup.
+ limit = _cgroup_memory_limit_mib()
+ if limit is not None:
+ total = min(int(total), int(limit))
+ reserve = max(_PIN_RESERVE_MIN_MIB, int(int(total) * _PIN_RESERVE_FRACTION))
+ return max(0, int(available) - reserve)
+
+
+def _remove_group_offload_hooks(module: Any) -> None:
+ """Undo a group offloading apply that raised part way (best effort)."""
+ try:
+ from diffusers.hooks import group_offloading as go
+ from diffusers.hooks.hooks import HookRegistry
+
+ registry = HookRegistry.check_if_exists_or_initialize(module)
+ for name in (
+ "_GROUP_OFFLOADING",
+ "_LAZY_PREFETCH_GROUP_OFFLOADING",
+ "_LAYER_EXECUTION_TRACKER",
+ ):
+ hook = getattr(go, name, None)
+ if isinstance(hook, str):
+ registry.remove_hook(hook, recurse = True)
+ except Exception: # noqa: BLE001 - the fallback reports its own failure
+ pass
+
+
+def _pinned_memory_capped() -> bool:
+ """Windows (WDDM) and WSL2 cap pinned host memory near 1 GiB (NVIDIA CUDA on WSL known limitation)."""
+ if sys.platform == "win32":
+ return True
+ try:
+ with open("/proc/version", "r", encoding = "utf-8") as f:
+ return "microsoft" in f.read().lower()
+ except Exception: # noqa: BLE001 - not Linux, or unreadable: not WSL
+ return False
+
+
+def _streamed_pin_plan(
+ transformer_mib: int,
+ encoder_mib: int,
+ logger: Any = None,
+) -> tuple[bool, bool]:
+ """(pin transformer, pin encoders), transformer first."""
+ forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()
+ if forced in ("0", "off", "false", "no"):
+ return False, False
+ if forced in ("1", "on", "true", "yes"):
+ return True, True
+ budget = None if _pinned_memory_capped() else _pin_budget_mib()
+ if budget is None or budget <= 0:
+ pin_transformer = pin_encoders = False
+ else:
+ pin_transformer = transformer_mib <= budget
+ pinned = transformer_mib if pin_transformer else 0
+ pin_encoders = pinned + encoder_mib <= budget
+ if logger is not None:
+ try:
+ logger.info(
+ "diffusion.memory: streamed-encoder tier pins transformer=%s (%d MiB) encoders=%s (%d MiB) "
+ "against %s MiB of pinnable host RAM",
+ pin_transformer,
+ transformer_mib,
+ pin_encoders,
+ encoder_mib,
+ budget,
+ )
+ except Exception: # noqa: BLE001
+ pass
+ return pin_transformer, pin_encoders
+
+
def install_group_offload_buffer_restore() -> bool:
"""diffusers stream group offload restores only parameters, leaving buffers (native int8 weights) on the GPU."""
try:
@@ -1430,16 +1568,21 @@ def _apply_group_offload(
logger: Any,
*,
stream_text_encoders: bool = False,
+ stream_transformer: bool = True,
) -> bool:
"""Stream the transformer a few blocks at a time via diffusers group offloading, keeping the
smaller components resident. Returns False (caller falls back to whole-module) on any failure.
``stream_text_encoders`` extends the streamed set to every ``text_encoder*`` module. Off by
default: keeping them resident is faster when there is room. The planner turns it on only
- where it is the difference between group offload and whole-module offload."""
+ where it is the difference between group offload and whole-module offload.
+
+ ``stream_transformer=False`` (only with ``stream_text_encoders``) keeps every DiT resident."""
transformer = getattr(pipe, "transformer", None)
if transformer is None:
return False
+ if not stream_transformer and not stream_text_encoders:
+ return False
installed = 0
try:
import inspect
@@ -1451,11 +1594,13 @@ def _apply_group_offload(
# A dual-DiT pipeline (Ideogram 4) carries a second denoiser as large as the first, so stream every DiT and keep
# only smaller companions resident.
- streamed: dict[str, Any] = {"transformer": transformer}
- for extra in ("transformer_2", "unconditional_transformer"):
- module = getattr(pipe, extra, None)
- if isinstance(module, torch.nn.Module):
- streamed[extra] = module
+ streamed: dict[str, Any] = {}
+ if stream_transformer:
+ streamed["transformer"] = transformer
+ for extra in ("transformer_2", "unconditional_transformer"):
+ module = getattr(pipe, extra, None)
+ if isinstance(module, torch.nn.Module):
+ streamed[extra] = module
# The text encoders are streamed SEPARATELY from the DiTs, and tolerantly (see the apply loop below). Kept in
# their own dict so the resident placement loop still skips them.
streamed_encoders: dict[str, Any] = {}
@@ -1486,16 +1631,17 @@ def _apply_group_offload(
gkwargs["non_blocking"] = True
if "record_stream" in _params:
gkwargs["record_stream"] = True
+ pin_streamed = (True, True)
if stream_text_encoders and "low_cpu_mem_usage" in _params:
# The streamed path PINS every offloaded parameter in host RAM when a copy stream is in use (diffusers
# group_offloading `_init_cpu_param_dict`), which is a fine trade when group offload was already the plan.
- # It is not a fine trade here: this tier is only ever reached as a rescue from whole-module offload, which
- # pins nothing, on a card small enough that the companions did not fit. Those hosts are not reliably
- # RAM-rich either, and silently converting a device-memory shortfall into ten-plus GB of unswappable host
- # RAM is how #8188's machine got into trouble in the first place. low_cpu_mem_usage trades a slower
- # host-to-device copy for not pinning; the encoders this tier streams run ONCE per call, so that copy is
- # paid once, not per step.
- gkwargs["low_cpu_mem_usage"] = True
+ # Pin only what host RAM covers (unbounded pinning hurt #8188); unpinned re-pins on every onload.
+ pin_streamed = _streamed_pin_plan(
+ sum(_module_host_mib(m) for m in streamed.values()),
+ sum(_module_host_mib(m) for m in streamed_encoders.values()),
+ logger,
+ )
+ gkwargs["low_cpu_mem_usage"] = not pin_streamed[0]
# Place the smaller components resident BEFORE attaching the transformer group-offload hooks: a companion .to()
# OOM then returns False with no hooks installed, and diffusers rejects enable_model_cpu_offload once group
# hooks exist.
@@ -1520,12 +1666,36 @@ def _apply_group_offload(
# plan said leaf and the application said block. num_blocks_per_group goes with it: leaf level has no blocks.
ekwargs = {k: v for k, v in gkwargs.items() if k != "num_blocks_per_group"}
ekwargs["offload_type"] = "leaf_level"
+ if "low_cpu_mem_usage" in gkwargs:
+ ekwargs["low_cpu_mem_usage"] = not pin_streamed[1]
+ transformer_demoted = False
for name, module in streamed_encoders.items():
try:
apply_group_offloading(module, **ekwargs)
installed += 1
_pin_vision_embedding_device(module)
except Exception as exc: # noqa: BLE001 -- degrade this encoder, never fail the load
+ if not stream_transformer and installed == 0:
+ # Resident encoder here would OOM; fall back to model offload, which rejects partial hooks.
+ _remove_group_offload_hooks(module)
+ raise
+ if not stream_transformer and not transformer_demoted:
+ # Model offload is gone once hooks exist: stream the transformer before this encoder goes resident.
+ for dit_name in ("transformer", "transformer_2", "unconditional_transformer"):
+ dit = getattr(pipe, dit_name, None)
+ if isinstance(dit, torch.nn.Module):
+ dkwargs = dict(gkwargs)
+ if "low_cpu_mem_usage" in _params and use_stream:
+ dkwargs["low_cpu_mem_usage"] = True
+ apply_group_offloading(dit, **dkwargs)
+ installed += 1
+ transformer_demoted = True
+ if logger is not None:
+ logger.warning(
+ "diffusion.memory: %s refused group offload on the resident-transformer tier; "
+ "streaming the transformer instead",
+ name,
+ )
if logger is not None:
logger.warning(
"diffusion.memory: group offload unavailable for %s (%s); "
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 4b5a2590a26..9806f601433 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -10716,8 +10716,9 @@ def test_plan_memory_hands_the_planner_the_text_encoder_split(monkeypatch, tmp_p
assert plan.estimates["companion_dense_mib"] == 200
assert plan.estimates["text_encoder_dense_mib"] == 150
assert plan.estimates["group_floor_streamed_te_mib"] == 2198
- # The companions fit as they are, so the cheaper tier still wins and nothing streams.
- assert plan.offload_policy == OFFLOAD_GROUP and plan.stream_text_encoders is False
+ assert plan.estimates["resident_transformer_floor_mib"] == 2498
+ assert plan.offload_policy == OFFLOAD_GROUP
+ assert plan.stream_text_encoders is True and plan.stream_transformer is False
def test_plan_memory_streams_the_text_encoders_instead_of_offloading_everything(
@@ -12067,6 +12068,325 @@ def test_a_prequant_repo_missing_its_artifact_marks_the_plan_incomplete(monkeypa
assert "prequant artifact missing" in str(failures[0])
+def test_plan_memory_prices_the_hosted_precast_text_encoder(monkeypatch, tmp_path):
+ snapshot = _base_snapshot_with_sizes(
+ tmp_path, monkeypatch, {"vae/diffusion_pytorch_model.safetensors": 50}
+ )
+ target = _small_card(monkeypatch)
+ seen = {}
+
+ def _precast(
+ fam,
+ base,
+ tgt,
+ text_encoder_quant,
+ staged_dir = None,
+ ):
+ seen["quant"] = text_encoder_quant
+ return (2800, ("text_encoder",), True) if text_encoder_quant == "fp8" else None
+
+ monkeypatch.setattr(DiffusionBackend, "_precast_text_encoder_mib", staticmethod(_precast))
+
+ def _plan(**kw):
+ return DiffusionBackend()._plan_memory(
+ target,
+ None,
+ "bfl/base",
+ types.SimpleNamespace(name = "flux.1"),
+ None,
+ False,
+ kind = "gguf",
+ transformer_resident_override_mib = 300,
+ base_local_dir = str(snapshot),
+ **kw,
+ )
+
+ before = _plan()
+ assert before.estimates["companion_dense_mib"] == 50
+ assert before.estimates["text_encoder_dense_mib"] is None
+ after = _plan(text_encoder_quant = "fp8")
+ assert seen["quant"] == "fp8"
+ assert after.estimates["text_encoder_dense_mib"] == 2800
+ assert after.estimates["companion_dense_mib"] == 2850
+ assert after.estimates["model_dense_mib"] == 3150
+
+
+def test_plan_memory_prices_cached_dense_shards_over_a_cached_precast(monkeypatch, tmp_path):
+ snapshot = _base_snapshot_with_sizes(
+ tmp_path,
+ monkeypatch,
+ {
+ "text_encoder/model.safetensors": 4000,
+ "vae/diffusion_pytorch_model.safetensors": 50,
+ },
+ )
+ target = _small_card(monkeypatch)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_precast_text_encoder_mib",
+ staticmethod(lambda *a, **k: (2600, ("text_encoder",), True)),
+ )
+ plan = DiffusionBackend()._plan_memory(
+ target,
+ None,
+ "bfl/base",
+ types.SimpleNamespace(name = "flux.1"),
+ None,
+ False,
+ kind = "gguf",
+ transformer_resident_override_mib = 300,
+ base_local_dir = str(snapshot),
+ text_encoder_quant = "fp8",
+ )
+ assert plan.estimates["text_encoder_dense_mib"] == 4000
+ assert plan.estimates["companion_dense_mib"] == 4050
+ assert plan.estimates["model_dense_mib"] == 4350
+
+
+def test_cached_dense_shards_keep_the_24g_denoiser_resident(monkeypatch, tmp_path):
+ from core.inference import diffusion as dmod
+ from core.inference.diffusion_memory import OFFLOAD_GROUP, DeviceMemory
+
+ snapshot = _base_snapshot_with_sizes(
+ tmp_path,
+ monkeypatch,
+ {
+ "text_encoder/model.safetensors": 16689,
+ "vae/diffusion_pytorch_model.safetensors": 1288,
+ },
+ )
+ monkeypatch.setattr(
+ dmod,
+ "settled_snapshot_device_memory",
+ lambda t: DeviceMemory("cuda", "cuda", "discrete_vram", 23000, 24576),
+ )
+ monkeypatch.setattr(dmod, "estimate_image_runtime_mib", lambda **kw: 8192)
+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", supports_model_cpu_offload = True)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_precast_text_encoder_mib",
+ staticmethod(lambda *a, **k: (8959, ("text_encoder",), True)),
+ )
+ plan = DiffusionBackend()._plan_memory(
+ target,
+ None,
+ "bfl/base",
+ types.SimpleNamespace(name = "qwen-image"),
+ None,
+ False,
+ kind = "gguf",
+ transformer_resident_override_mib = 7650,
+ base_local_dir = str(snapshot),
+ text_encoder_quant = "fp8",
+ )
+ assert plan.estimates["text_encoder_dense_mib"] == 16689
+ assert plan.offload_policy == OFFLOAD_GROUP
+ assert plan.stream_transformer is False and plan.stream_text_encoders is True
+
+
+def _flux_like_plan(tmp_path, monkeypatch, precast):
+ snapshot = _base_snapshot_with_sizes(
+ tmp_path,
+ monkeypatch,
+ {
+ "text_encoder/model.safetensors": 235,
+ "text_encoder_2/model.safetensors": 4000,
+ "vae/diffusion_pytorch_model.safetensors": 50,
+ },
+ )
+ target = _small_card(monkeypatch)
+ monkeypatch.setattr(
+ DiffusionBackend, "_precast_text_encoder_mib", staticmethod(lambda *a, **k: precast)
+ )
+ return DiffusionBackend()._plan_memory(
+ target,
+ None,
+ "bfl/base",
+ types.SimpleNamespace(name = "flux.1"),
+ None,
+ False,
+ kind = "gguf",
+ transformer_resident_override_mib = 300,
+ base_local_dir = str(snapshot),
+ text_encoder_quant = "fp8",
+ )
+
+
+def test_plan_memory_swaps_only_the_encoder_the_precast_checkpoint_replaces(monkeypatch, tmp_path):
+ plan = _flux_like_plan(tmp_path, monkeypatch, (4200, ("text_encoder_2",), True))
+ assert plan.estimates["text_encoder_dense_mib"] == 235 + 4200
+ assert plan.estimates["companion_dense_mib"] == 50 + 235 + 4200
+
+
+def test_plan_memory_never_prices_an_uncached_precast_below_the_dense_shards(monkeypatch, tmp_path):
+ plan = _flux_like_plan(tmp_path, monkeypatch, (2600, ("text_encoder_2",), False))
+ assert plan.estimates["text_encoder_dense_mib"] == 235 + 4000
+
+
+def test_a_table_priced_pipeline_does_not_add_the_precast_encoder_on_top(monkeypatch, tmp_path):
+ # A base-repo pipeline prices its encoders from the bf16 table, which already covers the dense encoder.
+ snapshot = _base_snapshot_with_sizes(
+ tmp_path,
+ monkeypatch,
+ {
+ "transformer/diffusion_pytorch_model.safetensors": 13500,
+ "vae/diffusion_pytorch_model.safetensors": 1288,
+ },
+ )
+ target = _small_card(monkeypatch)
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_precast_text_encoder_mib",
+ staticmethod(lambda *a, **k: (8959, ("text_encoder",), True)),
+ )
+ fam = types.SimpleNamespace(name = "qwen-image-2.1", base_repo = "bfl/base")
+
+ def _plan(quant):
+ return (
+ DiffusionBackend()
+ ._plan_memory(
+ target,
+ None,
+ "bfl/base",
+ fam,
+ None,
+ False,
+ kind = "pipeline",
+ repo_id = "bfl/base",
+ base_local_dir = str(snapshot),
+ text_encoder_quant = quant,
+ )
+ .estimates
+ )
+
+ dense, precast = _plan(None), _plan("fp8")
+ assert dense["text_encoder_dense_mib"] == 16689
+ for key in ("model_dense_mib", "companion_dense_mib", "text_encoder_dense_mib"):
+ assert precast[key] == dense[key]
+
+
+def test_plan_memory_leaves_a_callers_companion_override_alone(monkeypatch, tmp_path):
+ target = _small_card(monkeypatch)
+ monkeypatch.setattr(
+ DiffusionBackend, "_precast_text_encoder_mib", staticmethod(lambda *a, **k: 2600)
+ )
+ plan = DiffusionBackend()._plan_memory(
+ target,
+ None,
+ "bfl/base",
+ types.SimpleNamespace(name = "flux.1"),
+ None,
+ False,
+ kind = "gguf",
+ transformer_resident_override_mib = 300,
+ companion_override_mib = 700,
+ text_encoder_override_mib = 600,
+ text_encoder_quant = "fp8",
+ )
+ assert plan.estimates["companion_dense_mib"] == 700
+ assert plan.estimates["text_encoder_dense_mib"] == 600
+
+
+def _q21_precast_cache(tmp_path, monkeypatch, *, mib):
+ """unsloth/Qwen-Image-2.1-FP8 cached under the live root, holding only the pre-cast encoder."""
+ live, _other = _split_cache_roots(tmp_path, monkeypatch)
+ repo = live / "models--unsloth--Qwen-Image-2.1-FP8"
+ rev = "b" * 40
+ (repo / "refs").mkdir(parents = True)
+ (repo / "refs" / "main").write_text(rev)
+ snap = repo / "snapshots" / rev
+ snap.mkdir(parents = True)
+ if mib:
+ with open(snap / "Qwen-Image-2.1-text_encoder-FP8.safetensors", "wb") as fh:
+ fh.truncate(mib * 1024 * 1024)
+ return snap
+
+
+def _bf16_cuda_target():
+ import torch
+ return types.SimpleNamespace(
+ device = "cuda", backend = "cuda", dtype = torch.bfloat16, supports_model_cpu_offload = True
+ )
+
+
+def test_precast_text_encoder_mib_reads_the_cached_checkpoint(monkeypatch, tmp_path):
+ from core.inference.diffusion_families import detect_family
+
+ fam = detect_family("Qwen/Qwen-Image-2.1")
+ assert fam is not None and fam.name == "qwen-image-2.1"
+ _q21_precast_cache(tmp_path, monkeypatch, mib = 8959)
+ target = _bf16_cuda_target()
+ assert DiffusionBackend._precast_text_encoder_mib(
+ fam, "Qwen/Qwen-Image-2.1", target, "fp8"
+ ) == (
+ 8959,
+ ("text_encoder",),
+ True,
+ )
+ assert (
+ DiffusionBackend._precast_text_encoder_mib(fam, "Qwen/Qwen-Image-2.1", target, None) is None
+ )
+ assert (
+ DiffusionBackend._precast_text_encoder_mib(fam, "Qwen/Qwen-Image-2.1", target, "none")
+ is None
+ )
+
+
+def test_precast_text_encoder_mib_prices_hidreams_standalone_fourth_encoder(monkeypatch, tmp_path):
+ from core.inference.diffusion_families import detect_family
+ from core.inference.diffusion_te_prequant import te_candidate_filenames, te_prequant_sources
+
+ fam = detect_family("HiDream-ai/HiDream-I1-Full")
+ assert fam is not None and fam.name == "hidream-i1"
+ target = _bf16_cuda_target()
+ source = te_prequant_sources(
+ fam, te_quant_mode = "fp8", target = target, components = ("text_encoder_4",)
+ )["text_encoder_4"]
+ live, _other = _split_cache_roots(tmp_path, monkeypatch)
+ repo = live / ("models--" + source.location.replace("/", "--"))
+ rev = "c" * 40
+ (repo / "refs").mkdir(parents = True)
+ (repo / "refs" / "main").write_text(rev)
+ snap = repo / "snapshots" / rev
+ snap.mkdir(parents = True)
+ with open(snap / te_candidate_filenames(source)[0], "wb") as fh:
+ fh.truncate(7700 * 1024 * 1024)
+ assert DiffusionBackend._precast_text_encoder_mib(
+ fam, "HiDream-ai/HiDream-I1-Full", target, "fp8"
+ ) == (7700, ("text_encoder_4",), True)
+
+ (snap / te_candidate_filenames(source)[0]).unlink()
+ from core.inference.diffusion_hidream import HIDREAM_LLAMA_BF16_BYTES
+ from core.inference.diffusion_te_prequant import TE_PREQUANT_BUDGET_SCALE
+
+ assert DiffusionBackend._precast_text_encoder_mib(
+ fam, "HiDream-ai/HiDream-I1-Full", target, "fp8"
+ ) == (
+ int(HIDREAM_LLAMA_BF16_BYTES * TE_PREQUANT_BUDGET_SCALE) // (1024 * 1024),
+ ("text_encoder_4",),
+ False,
+ )
+
+
+def test_precast_text_encoder_mib_prices_an_uncached_checkpoint_from_the_family_table(
+ monkeypatch, tmp_path
+):
+ from core.inference.diffusion_families import detect_family
+ from core.inference.diffusion_te_prequant import TE_PREQUANT_BUDGET_SCALE
+
+ fam = detect_family("Qwen/Qwen-Image-2.1")
+ _q21_precast_cache(tmp_path, monkeypatch, mib = 0)
+ got = DiffusionBackend._precast_text_encoder_mib(
+ fam, "Qwen/Qwen-Image-2.1", _bf16_cuda_target(), "fp8"
+ )
+ assert got == (
+ int(17.5 * 1000**3 * TE_PREQUANT_BUDGET_SCALE) // (1024 * 1024),
+ ("text_encoder",),
+ False,
+ )
+ assert got[0] > 8959
+
+
def _stub_amd_weight_only_host(backend, monkeypatch):
"""ROCm / torchao-stub host; records the quantise calls."""
from core.inference import diffusion as dmod
diff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.py
index ff77828225e..0892cf6de0c 100644
--- a/studio/backend/tests/test_diffusion_memory.py
+++ b/studio/backend/tests/test_diffusion_memory.py
@@ -1628,7 +1628,10 @@ def test_streaming_the_text_encoders_does_not_pin_host_memory(monkeypatch):
# whole-module offload, which pins nothing, on a card too small to hold the companions. Those
# hosts are not reliably RAM-rich, and #8188's machine got into trouble precisely by turning a
# device shortfall into unswappable host memory. The encoders run once per call, so the slower
- # unpinned copy is paid once rather than per step.
+ import core.inference.diffusion_memory as mem
+
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 0)
seen = _stream_te_kwargs(monkeypatch, stream_text_encoders = True)
assert seen, "the applier never reached diffusers"
assert all(kw["low_cpu_mem_usage"] is True for kw in seen.values()), seen
@@ -2117,3 +2120,343 @@ def test_the_same_load_reaches_group_offload_once_the_split_is_known():
)
assert plan.offload_policy == OFFLOAD_GROUP
assert plan.vae_tiling is False
+
+
+_Q21_TRANSFORMER_MIB = 7_650
+_Q21_VAE_MIB = 1_288
+_Q21_TE_MIB = 8_959
+_24G_FREE_MIB = 23_000
+_24G_TOTAL_MIB = 24_576 # budget 23000 - 2457 = 20543
+
+
+def _q21_plan(
+ free_mib,
+ total_mib,
+ *,
+ text_encoder_dense_mib = _Q21_TE_MIB,
+ mode = None,
+):
+ return plan_diffusion_memory(
+ target = _target(),
+ device_memory = _discrete(free_mib, total_mib),
+ model_dense_mib = _Q21_TRANSFORMER_MIB + _Q21_VAE_MIB + _Q21_TE_MIB,
+ companion_dense_mib = _Q21_VAE_MIB + _Q21_TE_MIB,
+ text_encoder_dense_mib = text_encoder_dense_mib,
+ runtime_headroom_mib = 8192,
+ base_overhead_mib = 2048,
+ requested_mode = mode,
+ )
+
+
+def test_the_transformer_stays_resident_when_only_the_encoder_does_not_fit():
+ plan = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB)
+ assert plan.estimates["resident_transformer_floor_mib"] == 19_178
+ assert plan.offload_policy == OFFLOAD_GROUP
+ assert plan.stream_transformer is False and plan.stream_text_encoders is True
+ assert plan.estimates["group_floor_mib"] <= plan.estimates["safe_device_budget_mib"]
+ assert plan.as_public_dict()["stream_transformer"] is False
+
+
+def test_the_resident_transformer_tier_needs_the_encoder_split():
+ plan = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB, text_encoder_dense_mib = None)
+ assert plan.estimates["resident_transformer_floor_mib"] is None
+ assert plan.offload_policy == OFFLOAD_GROUP
+ assert plan.stream_transformer is True and plan.stream_text_encoders is False
+
+
+def test_a_card_too_small_for_the_resident_transformer_still_streams_it():
+ plan = _q21_plan(_16G_FREE_MIB, _16G_TOTAL_MIB)
+ assert plan.offload_policy == OFFLOAD_GROUP
+ assert plan.stream_transformer is True and plan.stream_text_encoders is True
+
+
+def test_fast_mode_prefers_the_resident_transformer_over_streaming_it():
+ plan = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB, mode = MEMORY_MODE_FAST)
+ assert plan.offload_policy == OFFLOAD_GROUP
+ assert plan.stream_transformer is False and plan.stream_text_encoders is True
+
+
+def test_every_other_tier_reports_a_streamed_or_resident_transformer_consistently():
+ roomy = _q21_plan(80_000, 81_920)
+ assert roomy.offload_policy == OFFLOAD_NONE and roomy.stream_transformer is True
+ tiny = _q21_plan(6_000, 8_192)
+ assert tiny.offload_policy == OFFLOAD_MODEL and tiny.stream_transformer is True
+ balanced = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB, mode = MEMORY_MODE_BALANCED)
+ assert balanced.offload_policy == OFFLOAD_GROUP and balanced.stream_transformer is True
+
+
+def test_apply_group_offload_can_keep_the_transformer_resident(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ pipe, applied, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
+ assert (
+ mem._apply_group_offload(
+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False
+ )
+ is True
+ )
+ assert applied == [te, te2]
+ assert transformer.placed is not None and vae.placed is not None
+ assert te.placed is None and te2.placed is None
+
+
+def test_the_resident_transformer_tier_streams_encoders_at_leaf_level(monkeypatch):
+ seen = _stream_te_kwargs(monkeypatch, stream_text_encoders = True, stream_transformer = False)
+ assert "transformer" not in seen
+ assert seen["text_encoder"]["offload_type"] == "leaf_level"
+ assert seen["text_encoder_2"]["offload_type"] == "leaf_level"
+
+
+def test_a_refusing_encoder_hands_the_resident_transformer_tier_to_whole_module_offload(
+ monkeypatch,
+):
+ import core.inference.diffusion_memory as mem
+
+ def _apply(module, **kw):
+ raise ValueError("no leaves to stream")
+
+ pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
+ _swap_group_offloading(monkeypatch, _apply)
+ assert (
+ mem._apply_group_offload(
+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False
+ )
+ is False
+ )
+
+
+def test_apply_group_offload_refuses_a_plan_that_streams_nothing(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ pipe, applied, *_ = _stream_te_pipe(monkeypatch)
+ assert mem._apply_group_offload(pipe, "cuda", logger = None, stream_transformer = False) is False
+ assert applied == []
+
+
+def test_apply_memory_plan_threads_the_resident_transformer_flag(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ seen = {}
+
+ def _fake(pipe, device, logger, **kw):
+ seen.clear()
+ seen.update(kw)
+ return True
+
+ monkeypatch.setattr(mem, "_apply_group_offload", _fake)
+ policy, _tiling = apply_memory_plan(
+ _RecordingPipe(), _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB), device = "cuda"
+ )
+ assert policy == OFFLOAD_GROUP
+ assert seen == {"stream_text_encoders": True, "stream_transformer": False}
+ apply_memory_plan(_RecordingPipe(), _q21_plan(_16G_FREE_MIB, _16G_TOTAL_MIB), device = "cuda")
+ assert seen == {"stream_text_encoders": True}
+
+
+def _sized_stream_te_kwargs(monkeypatch, sizes, budget, **call_kw):
+ import core.inference.diffusion_memory as mem
+
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False, raising = False)
+ monkeypatch.setattr(mem, "_module_host_mib", lambda m: sizes.get(getattr(m, "name", ""), 0))
+ return _stream_te_kwargs(monkeypatch, **call_kw)
+
+
+def test_a_ram_rich_host_pins_everything_the_tier_streams(monkeypatch):
+ seen = _sized_stream_te_kwargs(
+ monkeypatch,
+ {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},
+ 40_000,
+ stream_text_encoders = True,
+ )
+ assert all(kw["low_cpu_mem_usage"] is False for kw in seen.values()), seen
+
+
+def test_the_transformer_is_pinned_before_the_encoders(monkeypatch):
+ seen = _sized_stream_te_kwargs(
+ monkeypatch,
+ {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},
+ 10_000,
+ stream_text_encoders = True,
+ )
+ assert seen["transformer"]["low_cpu_mem_usage"] is False
+ assert seen["text_encoder"]["low_cpu_mem_usage"] is True
+ assert seen["text_encoder_2"]["low_cpu_mem_usage"] is True
+
+
+def test_the_resident_transformer_tier_pins_its_encoders_when_ram_allows(monkeypatch):
+ seen = _sized_stream_te_kwargs(
+ monkeypatch,
+ {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},
+ 10_000,
+ stream_text_encoders = True,
+ stream_transformer = False,
+ )
+ assert "transformer" not in seen
+ assert seen["text_encoder"]["low_cpu_mem_usage"] is False
+ assert seen["text_encoder_2"]["low_cpu_mem_usage"] is False
+
+
[email protected]("stream_transformer", [True, False])
+def test_windows_and_wsl_never_pin_the_streamed_tiers(monkeypatch, stream_transformer):
+ import core.inference.diffusion_memory as mem
+
+ seen = _sized_stream_te_kwargs(
+ monkeypatch,
+ {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},
+ 1_000_000,
+ stream_text_encoders = True,
+ stream_transformer = stream_transformer,
+ )
+ assert all(
+ kw["low_cpu_mem_usage"] is False for kw in seen.values()
+ ), seen # control: RAM-rich Linux pins
+ monkeypatch.delattr(mem, "_pinned_memory_capped", raising = False)
+ monkeypatch.undo()
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1_000_000)
+ monkeypatch.setattr(mem.sys, "platform", "win32")
+ seen = _stream_te_kwargs(
+ monkeypatch, stream_text_encoders = True, stream_transformer = stream_transformer
+ )
+ assert seen and all(kw["low_cpu_mem_usage"] is True for kw in seen.values()), seen
+ monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, "1")
+ assert mem._streamed_pin_plan(7000, 8000) == (True, True)
+
+
+def test_a_first_encoder_refusal_is_unhooked_before_whole_module_offload(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ def _apply(module, **kw):
+ if getattr(module, "name", "") == "text_encoder":
+ raise RuntimeError("CUDA error: out of memory (pin_memory)")
+
+ pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
+ _swap_group_offloading(monkeypatch, _apply)
+ unhooked: list[str] = []
+ monkeypatch.setattr(
+ mem, "_remove_group_offload_hooks", lambda m: unhooked.append(m.name), raising = False
+ )
+
+ assert (
+ mem._apply_group_offload(
+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False
+ )
+ is False
+ )
+ assert unhooked == ["text_encoder"]
+
+
+def test_pinned_memory_cap_detection(monkeypatch, tmp_path):
+ import builtins
+
+ import core.inference.diffusion_memory as mem
+
+ monkeypatch.setattr(mem.sys, "platform", "win32")
+ assert mem._pinned_memory_capped() is True
+ monkeypatch.setattr(mem.sys, "platform", "linux")
+ real_open = builtins.open
+ for text, capped in (
+ ("Linux version 6.6.87.2-microsoft-standard-WSL2 (gcc)", True),
+ ("Linux version 6.17.0-1007-aws (gcc)", False),
+ ):
+ monkeypatch.setattr(
+ builtins,
+ "open",
+ lambda path, *a, _t = text, **k: (
+ __import__("io").StringIO(_t)
+ if path == "/proc/version"
+ else real_open(path, *a, **k)
+ ),
+ )
+ assert mem._pinned_memory_capped() is capped
+
+
+def test_unknown_host_memory_never_pins(monkeypatch):
+ seen = _sized_stream_te_kwargs(
+ monkeypatch, {"transformer": 1, "text_encoder": 1}, None, stream_text_encoders = True
+ )
+ assert all(kw["low_cpu_mem_usage"] is True for kw in seen.values()), seen
+
+
[email protected]("value, pinned", [("0", True), ("1", False)])
+def test_the_pin_env_overrides_the_ram_gate(monkeypatch, value, pinned):
+ import core.inference.diffusion_memory as mem
+
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 40_000 if value == "0" else 0)
+ monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, value)
+ seen = _stream_te_kwargs(monkeypatch, stream_text_encoders = True)
+ assert all(kw["low_cpu_mem_usage"] is pinned for kw in seen.values()), seen
+
+
+def test_the_pin_budget_leaves_the_reserve_free(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ monkeypatch.setattr(mem, "_cgroup_memory_limit_mib", lambda: None)
+ monkeypatch.setattr(mem, "_system_memory_mib", lambda: (65_536, 30_000))
+ monkeypatch.setattr(mem, "_available_system_memory_mib", lambda: 30_000)
+ assert mem._pin_budget_mib() == 30_000 - int(65_536 * 0.15)
+ # A 64 GiB container on a 512 GiB host: the reserve comes from the container, as _pin_host_weights does.
+ monkeypatch.setattr(mem, "_cgroup_memory_limit_mib", lambda: 65_536)
+ monkeypatch.setattr(mem, "_system_memory_mib", lambda: (524_288, 60_000))
+ monkeypatch.setattr(mem, "_available_system_memory_mib", lambda: 60_000)
+ assert mem._pin_budget_mib() == 60_000 - int(65_536 * 0.15)
+ monkeypatch.setattr(mem, "_cgroup_memory_limit_mib", lambda: None)
+ monkeypatch.setattr(mem, "_system_memory_mib", lambda: (16_384, 3_000))
+ monkeypatch.setattr(mem, "_available_system_memory_mib", lambda: 3_000)
+ assert mem._pin_budget_mib() == 0
+ monkeypatch.setattr(mem, "_system_memory_mib", lambda: (None, None))
+ assert mem._pin_budget_mib() is None
+
+
+def test_a_late_encoder_refusal_streams_the_transformer_before_placing_the_encoder(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ order: list[str] = []
+
+ def _apply(module, **kw):
+ if getattr(module, "name", "") == "text_encoder_2":
+ raise ValueError("no leaves to stream")
+ order.append("hook:" + module.name)
+
+ pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
+ _swap_group_offloading(monkeypatch, _apply)
+ original_to = type(te2).to
+
+ def _to(self, device):
+ if self is te2:
+ order.append("place:text_encoder_2")
+ return original_to(self, device)
+
+ monkeypatch.setattr(type(te2), "to", _to)
+ assert (
+ mem._apply_group_offload(
+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False
+ )
+ is True
+ )
+ assert order == ["hook:text_encoder", "hook:transformer", "place:text_encoder_2"]
+
+
+def test_pinned_host_pricing_rounds_each_tensor_to_a_power_of_two(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ class _T:
+ def __init__(self, nbytes):
+ self._n = nbytes
+
+ def numel(self):
+ return self._n
+
+ def element_size(self):
+ return 1
+
+ class _M:
+ def parameters(self, recurse = True):
+ return [_T(3 * 1024 * 1024), _T(5 * 1024 * 1024)]
+
+ def buffers(self, recurse = True):
+ return [_T(1024 * 1024)]
+
+ assert mem._module_host_mib(_M()) == 13