Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: learned peaks per render phase, so later loads keep text encoders resident (#14979)

Daniel Han93fc9c730f61
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index ea7cbab442d..27d2fe9fa68 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -222,9 +222,13 @@ from .diffusion_memory import (
with_label_part,
applied_plan,
release_resident_groups,
+ release_for_decode,
resident_group_mib,
hook_resident_denoiser,
measured_request_extra_mib,
+ DECODE_LEND_ATTR,
+ decode_lend_from_plan,
+ with_workflow,
settled_snapshot_device_memory,
snapshot_device_memory,
_torchao_stream_pinnable,
@@ -2531,6 +2535,9 @@ def _drop_measured_reserve_for_layer_split(pipe: Any, plan: Any) -> bool:
return False
try:
pipe._unsloth_measured_reserve = None
+ setattr(
+ pipe, DECODE_LEND_ATTR, None
+ ) # the lent encoder residency was priced on the same one-card plan
except Exception: # noqa: BLE001
return False
return True
@@ -2939,17 +2946,30 @@ def _record_peak_probe(
height: Optional[int],
batch_size: int,
target: Any = None,
+ workflow: Optional[str] = None,
) -> None:
- """Persist a completed render's peak above its starting allocation under the load's peak identity."""
+ """Persist a completed render's peak above its starting allocation under the load's peak identity, keyed for
+ ``workflow`` (``with_workflow``: text-to-image keeps the load's key), with its pre-decode / decode phases."""
if probe is None or not width or not height:
return
identity = getattr(getattr(state, "pipe", None), "_unsloth_peak_identity", None)
try:
- from .diffusion_measured_peaks import device_name, record
+ from .diffusion_measured_peaks import (
+ PHASE_DECODE,
+ PHASE_DECODE_UNPADDED,
+ PHASE_PRE_DECODE,
+ PHASE_UNPADDED,
+ device_name,
+ record,
+ )
peak = probe.finish()
if peak is None or not isinstance(identity, tuple) or len(identity) != 3:
return
+ if getattr(probe, "decode_released", False):
+ # the decode streamed lent groups out, so its peak is net of their bytes: a sample would under-size the
+ # bucket (the placement that lent them already held a sample covering this decode)
+ return
if not _peak_identity_matches(state, identity):
# the configuration moved during the render (a compile fallback, a toggled cache): no key names it
return
@@ -2972,7 +2992,19 @@ def _record_peak_probe(
index = getattr(probe.device, "index", None)
graphs = cuda_graph.live_graph_count(index)
generation = cuda_graph.capture_generation(index)
- peak += int(getattr(probe, "window_pad", 0) or 0)
+ pad = int(getattr(probe, "window_pad", 0) or 0)
+ peak += pad
+ split = getattr(probe, "phase_peaks", None) or {}
+ phases = {}
+ if PHASE_PRE_DECODE in split and PHASE_DECODE in split:
+ pre, dec = int(split[PHASE_PRE_DECODE]), int(split[PHASE_DECODE])
+ phases = {
+ PHASE_PRE_DECODE: pre + pad,
+ PHASE_DECODE: dec + pad,
+ PHASE_DECODE_UNPADDED: dec,
+ PHASE_UNPADDED: max(pre, dec),
+ }
+ quant = with_workflow(quant, _peak_workflow(getattr(state, "family", None), workflow))
held: list = []
raised = record(
family = family,
@@ -2985,6 +3017,7 @@ def _record_peak_probe(
device_name = device_name(probe.device),
quadratic = _quadratic_attention(target, engaged, state.pipe),
held = held,
+ phases = phases,
)
if held:
# the store holds this sample, so it covers the graphs the render recorded; unpersisted, they stay dirty
@@ -9993,6 +10026,7 @@ class DiffusionBackend:
plan.estimates.get("measured_static_seed", True),
plan.estimates.get("measured_learned_cfg_off", False),
)
+ setattr(pipe, DECODE_LEND_ATTR, decode_lend_from_plan(plan))
from .diffusion_qwenimage21_vision import configure_vision_attention
@@ -13231,6 +13265,7 @@ class DiffusionBackend:
condition_pixels = guard_condition_pixels,
cfg = guard_cfg,
unpriced_references = guard_unpriced_refs,
+ workflow = _peak_workflow(fam, workflow),
)
if extra_mib > 0:
releasable_mib = resident_group_mib(state.pipe)
@@ -13471,6 +13506,7 @@ class DiffusionBackend:
except Exception: # noqa: BLE001 - an unknown guidance mode records nothing
state.pipe._unsloth_peak_cfg = None
peak_probe = _start_peak_probe(state)
+ decode_restores: list = []
try:
previewer = _start_previewer()
pending = list(chunks)
@@ -13548,6 +13584,22 @@ class DiffusionBackend:
"""The decoder was entered: a HOST position, possibly far ahead of the GPU. Mark the
boundary in the stream and let the poller flip once the GPU reaches it; with no event
to wait on (CPU, MPS) there is no queue to be ahead of, so flip now."""
+ if peak_probe is not None:
+ peak_probe.mark_decode()
+ # encoder residency lent to the decode streams again only if free memory is short now
+ restore = release_for_decode(
+ state.pipe,
+ width = guard_width,
+ height = guard_height,
+ batch_size = len(chunk),
+ device = getattr(guard_target, "torch_device", None),
+ logger = logger,
+ )
+ if restore is not None:
+ decode_restores.append(restore)
+ if peak_probe is not None:
+ # the decode ran with fewer weights than the probe's baseline: its peak understates
+ peak_probe.decode_released = True
marked = False
for _ in range(_BOUNDARY_MARK_ATTEMPTS):
with _hold_off_cuda_graph_capture() as clear:
@@ -13614,6 +13666,14 @@ class DiffusionBackend:
)
continue
finally:
+ while decode_restores:
+ try:
+ decode_restores.pop()()
+ except Exception as restore_exc: # noqa: BLE001 - the groups keep streaming
+ logger.warning(
+ "diffusion.memory: re-pinning groups after the decode failed (%s)",
+ restore_exc,
+ )
# A guarded block that fell back to eager (compile failed at its first forward) no longer
# runs compiled: report it on every exit, cancel and error included, so status, LoRA gating
# and the compile-cache shape registry stop treating it as compiled.
@@ -13662,6 +13722,7 @@ class DiffusionBackend:
height = guard_height,
batch_size = max(chunk_shapes or [1]),
target = guard_target,
+ workflow = workflow,
)
if static_skip:
logger.debug("diffusion.step_skip: %s", static_skip_stats(state.pipe))
@@ -14207,6 +14268,17 @@ def _hires_status(fam: DiffusionFamily) -> dict[str, Any]:
}
+def _peak_workflow(fam: Any, workflow: Optional[str]) -> Optional[str]:
+ """``workflow`` for the learned-peak key, None when it is the family's only workflow (an edit-only family): those
+ renders are what its loads place from, so they stay on the load key."""
+ try:
+ if fam is not None and _family_workflows(fam) == [str(workflow or "").strip().lower()]:
+ return None
+ except Exception: # noqa: BLE001
+ pass
+ return workflow
+
+
def _family_workflows(fam: DiffusionFamily) -> list[str]:
"""The workflow ids the diffusers engine can run for ``fam`` (drives UI gating)."""
# Instruction-editing families have no txt2img mode, so expose only "edit".
diff --git a/studio/backend/core/inference/diffusion_measured_peaks.py b/studio/backend/core/inference/diffusion_measured_peaks.py
index 1b631288676..86008d6945d 100644
--- a/studio/backend/core/inference/diffusion_measured_peaks.py
+++ b/studio/backend/core/inference/diffusion_measured_peaks.py
@@ -11,6 +11,10 @@ measurement above the seed means this machine needs more) and supplies one where
seed, because one render does not exercise every phase (a prompt-cache hit skips the encoder, a smaller request
decodes less), and the seed was measured on the worst phase.
+Phases: ``pre_decode`` / ``decode`` on the whole-render scale (prefetch window pad included), ``decode_unpadded`` /
+``unpadded`` without it (only the VAE runs at decode; resident weights hold no window). Pre-phase stores place from the
+whole-render peak. ``UNSLOTH_DIFFUSION_PHASE_PEAKS=0`` ignores the phases.
+
``UNSLOTH_DIFFUSION_LEARNED_PEAKS=0`` turns lookups and recording off; ``UNSLOTH_DIFFUSION_LEARNED_PEAKS_FILE``
names the JSON file (default ``<studio cache>/diffusion_measured_peaks.json``).
"""
@@ -29,8 +33,17 @@ from typing import Any, Optional
LEARNED_PEAKS_ENV = "UNSLOTH_DIFFUSION_LEARNED_PEAKS"
LEARNED_PEAKS_FILE_ENV = "UNSLOTH_DIFFUSION_LEARNED_PEAKS_FILE"
+PHASE_PEAKS_ENV = "UNSLOTH_DIFFUSION_PHASE_PEAKS"
+PHASE_PRE_DECODE = "pre_decode"
+PHASE_DECODE = "decode"
+PHASE_DECODE_UNPADDED = "decode_unpadded"
+PHASE_UNPADDED = "unpadded"
+PHASES = (PHASE_PRE_DECODE, PHASE_DECODE, PHASE_DECODE_UNPADDED, PHASE_UNPADDED)
_FILE_NAME = "diffusion_measured_peaks.json"
-_VERSION = 1
+# 2: non-txt2img workflows record under their own ``wf:`` key. Version 1 kept every workflow's max-wins peak on the
+# load key (an edit's larger peak inflated txt2img forever), so ``_migrate_v1`` keeps only what the phase format wrote.
+_VERSION = 2
+_WORKFLOW_KEY_PART = "|wf:"
_MAX_ENTRIES = 512
_MEGAPIXEL = 1024 * 1024
# Smallest bucket: a quarter megapixel. Below it the activation is dominated by fixed costs.
@@ -49,6 +62,15 @@ def enabled() -> bool:
)
+def phases_enabled() -> bool:
+ return enabled() and (os.environ.get(PHASE_PEAKS_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
def store_path() -> Optional[Path]:
explicit = (os.environ.get(LEARNED_PEAKS_FILE_ENV) or "").strip()
if explicit:
@@ -137,6 +159,34 @@ def _empty() -> dict:
return {"version": _VERSION, "runtime": runtime_stamp(), "entries": {}}
+def _migrate_v1(entries: dict) -> dict:
+ """A version 1 store's entries on the version 2 scale. ``wf:`` keys and the phase tables only ever came from the
+ per-workflow format and stay as they are. A load key's whole-render peaks may hold another workflow's sample, so
+ a bucket keeps only the larger of its pre-decode / decode phases (each a whole-render peak of a phase-recorded
+ render, which ran under the per-workflow key); a bucket or entry with no phase is dropped and relearned, the
+ load placing from its seed or the flat estimate meanwhile, as on a fresh store."""
+ kept: dict = {}
+ for key, entry in entries.items():
+ if not isinstance(entry, dict):
+ continue
+ if _WORKFLOW_KEY_PART in str(key):
+ kept[key] = entry
+ continue
+ table = entry.get("phases") if isinstance(entry.get("phases"), dict) else {}
+ peaks: dict = {}
+ for name in (PHASE_PRE_DECODE, PHASE_DECODE):
+ for bucket, value in (table.get(name) or {}).items():
+ try:
+ value = int(value)
+ except (TypeError, ValueError):
+ continue
+ if value > int(peaks.get(bucket) or 0):
+ peaks[str(bucket)] = value
+ if peaks:
+ kept[key] = dict(entry, peaks = peaks)
+ return kept
+
+
def _read(path: Path, fresh: bool = False) -> dict:
try:
mtime = path.stat().st_mtime_ns
@@ -151,6 +201,13 @@ def _read(path: Path, fresh: bool = False) -> dict:
return _cache["data"]
try:
data = json.loads(path.read_text(encoding = "utf-8"))
+ if (
+ isinstance(data, dict)
+ and data.get("version") == 1
+ and data.get("runtime") == runtime_stamp()
+ and isinstance(data.get("entries"), dict)
+ ):
+ data = dict(data, version = _VERSION, entries = _migrate_v1(data["entries"]))
if (
not isinstance(data, dict)
or data.get("version") != _VERSION
@@ -212,11 +269,13 @@ def record(
device_name: Optional[str] = None,
quadratic: bool = False,
held: Optional[list] = None,
+ phases: Optional[dict] = None,
) -> bool:
"""Keep the max peak seen for this key and bucket, scaled to the bucket's ceiling so a smaller render in the bucket
still answers a full-size one (squared with ``quadratic``: SDPA math attention grows with pixels squared, and the
key's entry keeps scaling that way). Returns True when the stored value rose. ``held`` (a list) gets True appended
- when the store holds at least this peak afterwards (raised, or already covered): False when nothing was persisted."""
+ when the store holds at least this peak afterwards (raised, or already covered): False when nothing was persisted.
+ ``phases`` ({name: MiB}, capped at ``peak_mib``) are kept max-wins and never affect the return value."""
if (
not enabled()
or peak_mib is None
@@ -235,12 +294,13 @@ def record(
bucket_pixels = resolution_bucket(width, height, batch_size)
bucket = str(bucket_pixels)
pixels = max(1, int(width or 1024)) * max(1, int(height or 1024)) * max(1, int(batch_size or 1))
- peak_mib = int(
- math.ceil(
- int(peak_mib)
- * (bucket_pixels / float(max(pixels, bucket_pixels // 2))) ** (2 if quadratic else 1)
- )
- )
+ factor = (bucket_pixels / float(max(pixels, bucket_pixels // 2))) ** (2 if quadratic else 1)
+ peak_mib = int(math.ceil(int(peak_mib) * factor))
+ phase_mib = {
+ name: min(peak_mib, int(math.ceil(int(value) * factor)))
+ for name, value in (phases or {}).items()
+ if name in PHASES and value is not None and int(value) > 0
+ }
try:
with _lock, _file_lock(path):
data = _read(path, fresh = True)
@@ -249,7 +309,17 @@ def record(
entries[key] = entry # most recently raised last: the cap evicts from the front
peaks = entry.setdefault("peaks", {})
old = int(peaks.get(bucket) or 0)
+ phase_raised = False
+ if phase_mib:
+ table = entry.setdefault("phases", {})
+ for name, value in phase_mib.items():
+ by_bucket = table.setdefault(name, {})
+ if int(value) > int(by_bucket.get(bucket) or 0):
+ by_bucket[bucket] = int(value)
+ phase_raised = True
if int(peak_mib) <= old:
+ if phase_raised:
+ _write(path, data)
if held is not None:
held.append(True)
return False
@@ -279,9 +349,11 @@ def lookup(
batch_size: int = 1,
frames: int = 1,
device: Optional[str] = None,
+ phase: Optional[str] = None,
) -> Optional[int]:
"""Learned peak MiB (no margin) for a request of this size, or None when nothing was recorded.
- ``device`` (GPU name) defaults to the current CUDA device, the one the planner snapshots.
+ ``device`` (GPU name) defaults to the current CUDA device, the one the planner snapshots. ``phase``
+ (one of ``PHASES``) reads that phase's peak instead of the whole render's: None when the entry has none.
The smallest recorded bucket at or above the request's bucket answers as is (an overestimate); with only smaller
buckets recorded, the largest one is scaled linearly by pixel count."""
@@ -294,6 +366,7 @@ def lookup(
batch_size = batch_size,
frames = frames,
device = device,
+ phase = phase,
)
return None if found is None else found[0]
@@ -317,6 +390,7 @@ def lookup_bucket(
batch_size: int = 1,
frames: int = 1,
device: Optional[str] = None,
+ phase: Optional[str] = None,
) -> Optional[tuple[int, int]]:
"""``lookup`` plus the pixels the peak covers and whether it scales squared: (peak MiB, bucket pixels,
quadratic), or None. A ``graphs:on`` label (a load placed before it knows whether placement keeps the whole-step
@@ -335,6 +409,7 @@ def lookup_bucket(
batch_size = batch_size,
frames = frames,
device = device,
+ phase = phase,
)
for mode in ("step", "blocks")
]
@@ -350,6 +425,8 @@ def lookup_bucket(
)
if not enabled() or not family or not identified(quant):
return None
+ if phase is not None and (phase not in PHASES or not phases_enabled()):
+ return None
path = store_path()
if path is None:
return None
@@ -357,9 +434,12 @@ def lookup_bucket(
with _lock:
key = model_key(family, quant, speed_mode, frames, device or current_device_name())
entry = (_read(path).get("entries") or {}).get(key)
- peaks = {
- int(k): int(v) for k, v in ((entry or {}).get("peaks") or {}).items() if int(v) > 0
- }
+ table = (
+ (entry or {}).get("peaks")
+ if phase is None
+ else ((entry or {}).get("phases") or {}).get(phase)
+ )
+ peaks = {int(k): int(v) for k, v in (table or {}).items() if int(v) > 0}
except Exception: # noqa: BLE001
return None
if not peaks:
@@ -489,6 +569,25 @@ class PeakProbe:
self.device = dev
except Exception: # noqa: BLE001 - measurement only
self.device = None
+ self.pre_decode = None
+ self.decodes = 0
+ self.phase_peaks: dict = {}
+
+ def mark_decode(self) -> None:
+ """Read the pre-decode peak without a reset (other measurements share the counters). A second decode (split
+ batch) interleaves phases, so no split is reported."""
+ if self.device is None or self.base is None:
+ return
+ self.decodes += 1
+ if self.decodes > 1:
+ self.pre_decode = None
+ return
+ try:
+ import torch
+ peak = int(torch.cuda.max_memory_allocated(self.device)) - self.base
+ except Exception: # noqa: BLE001
+ return
+ self.pre_decode = -(-peak // (1024 * 1024)) if peak > 0 else 0
def finish(self) -> Optional[int]:
if self.device is None or self.base is None:
@@ -508,7 +607,14 @@ class PeakProbe:
# closed only after the read: a render arming in between would otherwise reset the peak unseen
if not close_window(self.device, getattr(self, "window", None)) or peak is None:
return None
- return -(-peak // (1024 * 1024)) if peak > 0 else None
+ mib = -(-peak // (1024 * 1024)) if peak > 0 else None
+ if mib is not None and self.decodes == 1 and self.pre_decode is not None:
+ # the decode's own peak is unknown when it stayed under the earlier phases: the whole render bounds it
+ self.phase_peaks = {
+ PHASE_PRE_DECODE: max(1, min(int(self.pre_decode), mib)),
+ PHASE_DECODE: mib,
+ }
+ return mib
def _range_bytes(dev: Any) -> int:
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 5133f7fa090..782a9aba80e 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import contextlib
import contextvars
import functools
+import math
import os
import sys
import threading
@@ -1026,6 +1027,23 @@ def _metal_working_set_mib() -> tuple[Optional[int], int]:
return (working_set, in_use) if working_set > 0 else (None, 0)
+def _graph_held_cache_bytes(device: Optional[int] = None) -> int:
+ """Bytes inside ``memory_reserved() - memory_allocated()`` that ordinary allocations cannot reuse: the CUDA graph
+ pools (per-block pools whole, the whole-step graphs' shared pool's unallocated part)."""
+ held = 0
+ try:
+ from .diffusion_block_graph import pool_bytes as _block_graph_pool_bytes
+ held += int(_block_graph_pool_bytes(device))
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ from .diffusion_cuda_graph import live_pool_free_bytes
+ held += int(live_pool_free_bytes())
+ except Exception: # noqa: BLE001 -- no graph layer: nothing held
+ pass
+ return held
+
+
def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:
"""``snapshot_device_memory`` with the caching allocator's RECLAIMABLE bytes credited back as free,
without flushing it.
@@ -1062,21 +1080,10 @@ def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:
except Exception: # noqa: BLE001 -- no allocator reading: the plain snapshot still stands
return snapshot
try:
- # Per-block graph pools are reserved, mostly unallocated between replays, and never reusable by anything else.
- from .diffusion_block_graph import pool_bytes as _block_graph_pool_bytes
- try:
- device = torch.cuda.current_device()
- except Exception: # noqa: BLE001 - no current CUDA device: count every pool
- device = None
- reclaimable -= _block_graph_pool_bytes(device)
- except Exception: # noqa: BLE001
- pass
- try:
- # The whole-step graphs' shared pool: only its unallocated part sits inside the difference above.
- from .diffusion_cuda_graph import live_pool_free_bytes
- reclaimable -= live_pool_free_bytes()
- except Exception: # noqa: BLE001 -- no graph layer: nothing held
- pass
+ device = torch.cuda.current_device()
+ except Exception: # noqa: BLE001 - no current CUDA device: count every pool
+ device = None
+ reclaimable -= _graph_held_cache_bytes(device)
if reclaimable <= 0:
return snapshot
free = int(snapshot.free_mib) + reclaimable // (1024 * 1024)
@@ -3065,8 +3072,10 @@ def measured_image_runtime_mib(
phase_seeds: bool = False,
static_seed: bool = True,
dense_tier_seeds: bool = False,
+ phase: Optional[str] = None,
) -> Optional[int]:
"""Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate).
+ ``phase`` (``diffusion_measured_peaks.PHASES``) reads that phase's learned peak instead of the whole render's.
``dense_transformer_mib`` selects the dense eager table; ``compute_bytes`` scales it (fp32 Qwen-Image: 8480 vs 4248).
Seeds: the peak tables above, then the per-phase calibration (``_MEASURED_IMAGE_ACTIVATION_MIB``, the same
@@ -3104,7 +3113,13 @@ def measured_image_runtime_mib(
if quant is not None:
from .diffusion_measured_peaks import lookup as learned_peak
learned = learned_peak(
- family = family, quant = quant, speed_mode = speed_mode, width = w, height = h, batch_size = batch
+ family = family,
+ quant = quant,
+ speed_mode = speed_mode,
+ width = w,
+ height = h,
+ batch_size = batch,
+ phase = phase,
)
if seed is None and learned is None:
return None
@@ -3179,6 +3194,43 @@ def with_label_part(label: str, name: str, value: str) -> str:
)
+# Non-txt2img workflows record under ``wf:<workflow>`` so an edit's larger peak never inflates the next txt2img load.
+WORKFLOW_PART = "wf"
+TXT2IMG_WORKFLOW = "txt2img"
+
+
+def with_workflow(label: str, workflow: Optional[str]) -> str:
+ """``label`` keyed for ``workflow``: unchanged for text-to-image (and an unknown workflow), else ``|wf:<name>``."""
+ name = str(workflow or "").strip().lower()
+ if not name or name == TXT2IMG_WORKFLOW:
+ return str(label)
+ return f"{label}|{WORKFLOW_PART}:{name}"
+
+
+def learned_phase_split(
+ family: Optional[str], quant: Optional[str], speed_mode: Optional[str]
+) -> Optional[dict]:
+ """Learned phases (+ ``pixels`` / ``quadratic``) at the default bucket, or None unless every phase shares the
+ whole-render bucket and the larger padded phase reaches the whole-render peak."""
+ from .diffusion_measured_peaks import PHASE_DECODE, PHASE_PRE_DECODE, PHASES, lookup_bucket
+
+ if quant is None:
+ return None
+ total = lookup_bucket(family = family, quant = quant, speed_mode = speed_mode)
+ if total is None:
+ return None
+ found = {}
+ for phase in PHASES:
+ value = lookup_bucket(family = family, quant = quant, speed_mode = speed_mode, phase = phase)
+ if value is None or value[1] != total[1]:
+ return None
+ found[phase] = int(value[0])
+ if max(found[PHASE_PRE_DECODE], found[PHASE_DECODE]) < int(total[0]):
+ return None
+ found.update(pixels = int(total[1]), quadratic = bool(total[2]))
+ return found
+
+
_VAE_SAVERS = (
("use_tiling", "enable_vae_tiling", "enable_tiling"),
("use_slicing", "enable_vae_slicing", "enable_slicing"),
@@ -3642,6 +3694,33 @@ def refine_plan_from_loaded_weights(
# The flat plan's hooks stay installed and whole groups are kept resident within the room, so an oversized
# request can stream them again (release_resident_groups) and run exactly as the flat plan would.
stream_te = bool(getattr(plan, "stream_text_encoders", False))
+ # The whole-render peak includes the stream prefetch window, which an all-resident placement never allocates:
+ # all-resident is checked against the unpadded peak, else encoders against the padded pre-decode phase. Extra
+ # residency is lent to the decode; release_for_decode re-streams it only if free memory is short then.
+ pre_headroom = resident_headroom = None
+ split = None
+ if stream_te and policy == OFFLOAD_GROUP:
+ split = learned_phase_split(family, read_quant, speed_mode)
+ if split is not None:
+ from .diffusion_measured_peaks import PHASE_PRE_DECODE, PHASE_UNPADDED
+
+ def _phase_headroom(phase: str) -> Optional[int]:
+ value = measured_image_runtime_mib(
+ family,
+ speed_mode,
+ dense_transformer_mib = dense_mib,
+ compute_bytes = compute_bytes or 2,
+ quant = read_quant,
+ phase_seeds = bool(getattr(pipe, "_unsloth_phase_seeds", False)),
+ dense_tier_seeds = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False)),
+ static_seed = static_seed,
+ phase = phase,
+ )
+ return value if value is not None and value < headroom else None
+
+ pre_headroom = _phase_headroom(PHASE_PRE_DECODE)
+ if not bool(getattr(plan, "stream_transformer", True)):
+ resident_headroom = _phase_headroom(PHASE_UNPADDED)
room = budget - floor
if policy == OFFLOAD_GROUP and not stream_te:
room -= encoders # resident companions
@@ -3674,6 +3753,30 @@ def refine_plan_from_loaded_weights(
te_room = (
min(max(0, room - dit_room), encoders) if stream_te and policy == OFFLOAD_GROUP else 0
)
+ lend = 0
+ phase_headroom = None
+ if split is not None and not whole_dit and not wide_dit:
+ phase_te_room = te_room
+ if (
+ resident_headroom is not None
+ and room + (headroom - resident_headroom) - dit_room >= encoders
+ ):
+ phase_headroom, phase_te_room = resident_headroom, encoders
+ elif pre_headroom is not None:
+ phase_headroom = pre_headroom
+ phase_te_room = min(max(0, room + (headroom - pre_headroom) - dit_room), encoders)
+ lend = max(0, phase_te_room - te_room)
+ if lend > 0:
+ from .diffusion_measured_peaks import PHASE_DECODE_UNPADDED
+ te_room = phase_te_room
+ estimates.update(
+ measured_phase_headroom_mib = phase_headroom,
+ measured_phase_all_resident = phase_headroom == resident_headroom,
+ measured_decode_peak_mib = split[PHASE_DECODE_UNPADDED],
+ measured_decode_peak_pixels = split["pixels"],
+ measured_decode_peak_quadratic = split["quadratic"],
+ decode_lend_text_encoder_mib = lend,
+ )
if dit_room <= 0 and te_room <= 0:
return plan
if whole_dit or wide_dit:
@@ -3696,7 +3799,18 @@ def refine_plan_from_loaded_weights(
else f"{int(dit_room)} MiB of the {dit} MiB transformer"
)
+ f" and {int(te_room)} MiB of the {encoders} MiB "
- f"encoders stay resident (measured peak {headroom} MiB); the rest streams"
+ f"encoders stay resident (measured peak {headroom} MiB"
+ + (
+ (
+ f"; {phase_headroom} MiB without the streaming window, every weight resident"
+ if phase_headroom == resident_headroom
+ else f"; {phase_headroom} MiB before the VAE decode"
+ )
+ + f", the decode may stream {lend} MiB of them again"
+ if lend > 0
+ else ""
+ )
+ + "); the rest streams"
+ (
f" (whole transformer within free memory less a {_resident_dit_slack_mib(memory)} MiB slack;"
f" {encode_room} MiB of it while the encoders run)"
@@ -4079,6 +4193,96 @@ def release_resident_groups(
return restore if released else None
+DECODE_LEND_ATTR = "_unsloth_decode_lend"
+# Kept free beside the decode's learned peak x margin when deciding whether lent encoder groups must stream again.
+_DECODE_SLACK_MIB = 256
+
+
+def decode_lend_from_plan(plan: Any) -> Optional[tuple[int, int, int, bool]]:
+ """``(lent MiB, decode peak MiB, bucket pixels, quadratic)`` when ``plan`` lent encoder residency to the decode."""
+ estimates = getattr(plan, "estimates", None) or {}
+ lend = estimates.get("decode_lend_text_encoder_mib")
+ if not lend or not getattr(plan, "resident_text_encoder_mib", None):
+ return None
+ return (
+ int(lend),
+ int(estimates.get("measured_decode_peak_mib") or 0),
+ int(
+ estimates.get("measured_decode_peak_pixels")
+ or DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT
+ ),
+ bool(estimates.get("measured_decode_peak_quadratic")),
+ )
+
+
+def decode_need_mib(
+ lend: Any,
+ *,
+ width: Optional[int],
+ height: Optional[int],
+ batch_size: int = 1,
+) -> int:
+ """MiB the decode of a ``width x height x batch`` request needs above the allocation at its start (margin
+ included), scaled from the learned decode peak the way the store scales (``diffusion_measured_peaks``)."""
+ from .diffusion_measured_peaks import resolution_bucket
+
+ _, decode_mib, covered, quadratic = lend
+ w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
+ h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
+ batch = max(1, int(batch_size or 1))
+ covered = max(1, int(covered))
+ scale = 1.0
+ if quadratic:
+ if w * h > covered:
+ scale = (resolution_bucket(w, h, 1) / float(covered)) ** 2
+ scale *= batch
+ elif w * h * batch > covered:
+ scale = resolution_bucket(w, h, batch) / float(covered)
+ return int(math.ceil(float(decode_mib) * scale * _MEASURED_PEAK_MARGIN))
+
+
+def release_for_decode(
+ pipe: Any,
+ *,
+ width: Optional[int],
+ height: Optional[int],
+ batch_size: int = 1,
+ device: Any = None,
+ logger: Any = None,
+) -> Optional[Callable[[], None]]:
+ """If free memory (driver + the allocator cache outside CUDA graph pools) is short of the learned decode peak x margin, stream the shortfall of
+ lent groups again (text encoders first) and return the restore; else None. Mirrors ComfyUI's
+ ``load_models_gpu(memory_required=...)``. The decode is untouched, so output is bit-identical."""
+ lend = getattr(pipe, DECODE_LEND_ATTR, None)
+ if not lend:
+ return None
+ try:
+ import torch
+
+ dev = torch.device(device if device is not None else "cuda")
+ if dev.type != "cuda":
+ return None
+ if dev.index is None:
+ dev = torch.device("cuda", torch.cuda.current_device())
+ need = decode_need_mib(lend, width = width, height = height, batch_size = batch_size)
+ free = int(torch.cuda.mem_get_info(dev)[0])
+ # only the cache an ordinary decode allocation can reuse: a CUDA graph pool's free blocks are not
+ cached = max(
+ 0,
+ int(torch.cuda.memory_reserved(dev))
+ - int(torch.cuda.memory_allocated(dev))
+ - _graph_held_cache_bytes(dev.index),
+ )
+ short = need + _DECODE_SLACK_MIB - ((free + cached) >> 20)
+ except Exception as exc: # noqa: BLE001 - unreadable: the decode's OOM retry still frees the card
+ if logger is not None:
+ logger.debug("diffusion.memory: decode residency check skipped (%s)", exc)
+ return None
+ if short <= 0:
+ return None
+ return release_resident_groups(pipe, short, logger, reason = "the VAE decode", device = dev)
+
+
def resident_group_mib(pipe: Any) -> int:
"""MiB an oversized request could release: every resident offload group of the pipeline's components."""
total = 0
@@ -4382,8 +4586,10 @@ def measured_request_extra_mib(
condition_pixels: int = 0,
cfg: Optional[bool] = None,
unpriced_references: bool = False,
+ workflow: Optional[str] = None,
) -> int:
- """MiB a request needs beyond what the measured placement reserved (0 when it fits or none was made). ``cfg``:
+ """MiB a request needs beyond what the measured placement reserved (0 when it fits or none was made). ``workflow``:
+ a workflow other than text-to-image also reads the peak recorded under its own key (``with_workflow``). ``cfg``:
whether the request runs classifier-free guidance (None: unknown, treated as running it). ``unpriced_references``:
the request carries reference images whose latents join the denoiser's sequence but are not in
``condition_pixels`` (FLUX.2 Klein's primary reference): a placement made from a dense default / max tier seed,
@@ -4467,12 +4673,32 @@ def measured_request_extra_mib(
need = max(need, stored)
if need is None and isinstance(identity, tuple) and len(identity) == 3:
need = measured_image_runtime_mib(family, speed_mode, quant = identity[1], **sized)
- if need is None:
- return 0
cond = max(0, int(condition_pixels or 0)) * max(1, int(batch_size or 1))
- need += int(
+ allowance = int(
8192 * cond / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT) * _MEASURED_PEAK_MARGIN
)
+ if need is not None:
+ need += allowance
+ if isinstance(identity, tuple) and len(identity) == 3:
+ keyed = with_workflow(identity[1], workflow)
+ if keyed != identity[1]:
+ own = None
+ if (
+ cfg is False
+ and getattr(pipe, "_unsloth_peak_expects_cfg", False)
+ and cfg_off_peaks_enabled()
+ ):
+ # a guidance-off render on a CFG-capable family records ``|cfg:off`` before the workflow part
+ off = with_workflow(f"{without_cfg_off(identity[1])}|{CFG_OFF_PART}", workflow)
+ own = measured_image_runtime_mib(family, speed_mode, quant = off, **sized)
+ if own is None:
+ own = measured_image_runtime_mib(family, speed_mode, quant = keyed, **sized)
+ if own is not None:
+ # the learned workflow peak is the other complete estimate: it was measured with the condition images
+ # resident, so the allowance is not charged on top of it
+ need = own if need is None else max(need, own)
+ if need is None:
+ return 0
return max(0, int(need) - int(headroom))
diff --git a/studio/backend/tests/test_diffusion_phase_peaks.py b/studio/backend/tests/test_diffusion_phase_peaks.py
new file mode 100644
index 00000000000..cfedc564ee5
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_phase_peaks.py
@@ -0,0 +1,657 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Learned peaks per render phase (``diffusion_measured_peaks``) and the encoder residency the planner lends to the
+VAE decode (``diffusion_memory.refine_plan_from_loaded_weights`` / ``release_for_decode``), plus the per-workflow key.
+
+The Qwen-Image-2.1 case uses what an auto load logged on an RTX 3090: int8 DiT 6922 MiB resident, 8917 MiB of text
+encoders, a 21013 MiB safe budget. The first load (empty store) placed from the 2304 MiB seed and kept every encoder
+resident; after one 1024x1024 render the store held ~5.9 GB and every later load kept only part of the encoders
+resident, so every new prompt streamed the rest. The phases are the ones a 24 GiB-capped B200 render recorded: 1864 MiB
+real peak (the VAE decode; encode + steps ~900) plus a 4187 MiB streamed-window pad (the encoders' largest leaf and the
+transformer's largest block, times the prefetch depth + 1), 6051 MiB on the whole-render scale. CPU-only.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import types
+
+import pytest
+
+import core.inference.diffusion_measured_peaks as mp
+import core.inference.diffusion_memory as dm
+
+QWEN_LOADED = {
+ "transformer": (6922, "dit"),
+ "text_encoder": (8917, "text_encoder"),
+ "vae": (254, "other"),
+}
+KEY = dict(family = "qwen-image-2.1", quant = "torchao:Int8Tensor", speed_mode = "default")
+
+
+def _memory(budget_mib: int, total_mib: int) -> "dm.DeviceMemory":
+ reserve = dm._reserve_mib("discrete_vram", total_mib)
+ return dm.DeviceMemory("cuda", "cuda", "discrete_vram", budget_mib + reserve, total_mib)
+
+
+def _qwen_flat_plan(budget_mib: int = 21013, total_mib: int = 24576):
+ plan = dm.plan_diffusion_memory(
+ target = types.SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = _memory(budget_mib, total_mib),
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(
+ width = None, height = None, family = "qwen-image-2.1"
+ ),
+ requested_mode = None,
+ model_dense_mib = 6922 + 8917 + 254,
+ companion_dense_mib = 8917 + 254,
+ text_encoder_dense_mib = 8917,
+ )
+ if plan.offload_policy == dm.OFFLOAD_MODEL:
+ plan = dm.torchao_streaming_plan(plan)
+ return plan
+
+
+def store(tmp_path, monkeypatch):
+ path = tmp_path / "peaks.json"
+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(path))
+ for env in (
+ mp.LEARNED_PEAKS_ENV,
+ mp.PHASE_PEAKS_ENV,
+ dm.MEASURED_ACTIVATION_ENV,
+ dm.PARTIAL_RESIDENT_ENV,
+ ):
+ monkeypatch.delenv(env, raising = False)
+ mp._cache.update(path = None, mtime = None, data = None)
+ monkeypatch.setattr(mp, "current_device_name", lambda: "NVIDIA GeForce RTX 3090")
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA GeForce RTX 3090")
+ return path
+
+
+def qwen_pipe(monkeypatch, store):
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(QWEN_LOADED))
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ monkeypatch.setattr(dm, "denoiser_quant_label", lambda pipe: "torchao:Int8Tensor")
+ return types.SimpleNamespace(_unsloth_phase_seeds = True)
+
+
+def _refine(pipe, plan):
+ return dm.refine_plan_from_loaded_weights(
+ pipe, plan, family = "qwen-image-2.1", speed_mode = "default"
+ )
+
+
+PAD = 4187
+
+
+def _phases(
+ pre,
+ dec,
+ pad = PAD,
+):
+ """What ``diffusion._record_peak_probe`` stores for real pre-decode / decode peaks ``pre`` / ``dec``."""
+ return {
+ mp.PHASE_PRE_DECODE: pre + pad,
+ mp.PHASE_DECODE: dec + pad,
+ mp.PHASE_DECODE_UNPADDED: dec,
+ mp.PHASE_UNPADDED: max(pre, dec),
+ }
+
+
+B200 = dict(peak = 1864 + PAD, phases = _phases(900, 1864))
+
+
+def _record(
+ peak,
+ phases = None,
+ **extra,
+):
+ kw = dict(KEY, width = 1024, height = 1024)
+ kw.update(extra)
+ return mp.record(**kw, peak_mib = peak, phases = phases)
+
+
+# ---------------------------------------------------------------------------------------------------- planner
+
+
+def test_decode_phase_peak_keeps_the_encoders_resident_on_24gb(qwen_pipe):
+ plan = _qwen_flat_plan()
+ assert plan.offload_policy == dm.OFFLOAD_GROUP and not plan.stream_transformer
+ assert plan.stream_text_encoders
+ first = _refine(qwen_pipe, plan)
+ assert first.resident_text_encoder_mib == 8917 # the seeded first load
+ _record(**B200)
+ later = _refine(qwen_pipe, plan)
+ assert (
+ later.estimates["measured_runtime_headroom_mib"] == 7168
+ ) # the whole-render reserve is unchanged
+ assert later.resident_text_encoder_mib == 8917 # main: 4621
+ assert later.estimates["measured_phase_headroom_mib"] == 2304 # max(seed 1849, 1864) x 1.15
+ assert later.estimates["measured_phase_all_resident"]
+ assert later.estimates["decode_lend_text_encoder_mib"] == 8917 - 4621
+ assert "every weight resident" in later.reasons[-1]
+ assert dm.decode_lend_from_plan(later) == (4296, 1864, 1024 * 1024, False)
+
+
+def test_whole_render_peaks_alone_place_as_before(qwen_pipe):
+ """A store written before phases existed (or a render that recorded none) keeps today's placement."""
+ plan = _qwen_flat_plan()
+ _record(B200["peak"])
+ later = _refine(qwen_pipe, plan)
+ assert later.resident_text_encoder_mib == 4621
+ assert "decode_lend_text_encoder_mib" not in later.estimates
+ assert dm.decode_lend_from_plan(later) is None
+
+
+def test_a_peak_no_phase_recorded_render_explains_keeps_the_whole_render_reserve(qwen_pipe):
+ plan = _qwen_flat_plan()
+ _record(6800) # an older render, phase unknown, above what the phase-recorded one saw
+ _record(**B200)
+ assert _refine(qwen_pipe, plan).resident_text_encoder_mib < 8917
+ assert dm.learned_phase_split(KEY["family"], KEY["quant"], KEY["speed_mode"]) is None
+
+
+def test_encoders_that_cannot_all_stay_resident_size_against_the_pre_decode_phase(qwen_pipe):
+ """16 GiB: the transformer streams, so the unpadded check does not apply; the encoders size against encode +
+ steps (padded), and the decode borrows the difference."""
+ plan = _qwen_flat_plan(budget_mib = 13132, total_mib = 16384)
+ _record(B200["peak"])
+ before = _refine(qwen_pipe, plan)
+ _record(**B200)
+ after = _refine(qwen_pipe, plan)
+ assert not before.resident_text_encoder_mib
+ assert after.resident_text_encoder_mib == 7168 - 5888
+ assert after.resident_transformer_mib == before.resident_transformer_mib
+ assert not after.estimates["measured_phase_all_resident"]
+ assert after.estimates["measured_phase_headroom_mib"] == 5888
+
+
+def test_pre_decode_phase_at_the_whole_render_peak_lends_nothing(qwen_pipe):
+ plan = _qwen_flat_plan(budget_mib = 13132, total_mib = 16384)
+ _record(B200["peak"], _phases(1864, 1864))
+ assert not _refine(qwen_pipe, plan).resident_text_encoder_mib
+
+
+def test_phase_kill_switch_restores_the_whole_render_placement(qwen_pipe, monkeypatch):
+ plan = _qwen_flat_plan()
+ _record(**B200)
+ monkeypatch.setenv(mp.PHASE_PEAKS_ENV, "0")
+ assert _refine(qwen_pipe, plan).resident_text_encoder_mib == 4621
+
+
[email protected]("budget,total", [(9300, 12288), (13132, 16384), (21013, 24576)])
+def test_12_16_and_24gb_never_keep_less_resident(qwen_pipe, budget, total):
+ plan = _qwen_flat_plan(budget_mib = budget, total_mib = total)
+ _record(B200["peak"])
+ before = _refine(qwen_pipe, plan)
+ _record(**B200)
+ after = _refine(qwen_pipe, plan)
+ assert (after.resident_text_encoder_mib or 0) >= (before.resident_text_encoder_mib or 0)
+ assert after.resident_transformer_mib == before.resident_transformer_mib
+ assert after.offload_policy == before.offload_policy
+ assert after.stream_transformer == before.stream_transformer
+
+
+# ---------------------------------------------------------------------------------------------------- store
+
+
+def test_phases_are_stored_beside_the_whole_render_peak(store):
+ assert _record(5900, {mp.PHASE_PRE_DECODE: 2000, mp.PHASE_DECODE: 5900})
+ assert mp.lookup(**KEY, phase = mp.PHASE_UNPADDED) is None # not given
+ assert mp.lookup(**KEY) == 5900
+ assert mp.lookup(**KEY, phase = mp.PHASE_PRE_DECODE) == 2000
+ assert mp.lookup(**KEY, phase = mp.PHASE_DECODE) == 5900
+ # a lower whole-render peak still raises a phase (max wins per phase), the whole-render value stays
+ assert not _record(5000, {mp.PHASE_PRE_DECODE: 2400, mp.PHASE_DECODE: 5000})
+ assert mp.lookup(**KEY) == 5900
+ assert mp.lookup(**KEY, phase = mp.PHASE_PRE_DECODE) == 2400
+ entry = json.loads(store.read_text(encoding = "utf-8"))["entries"]
+ (only,) = entry.values()
+ assert only["phases"][mp.PHASE_DECODE] == {str(1024 * 1024): 5900}
+ # a phase never exceeds the render's own peak, unknown names are dropped
+ _record(100, {mp.PHASE_PRE_DECODE: 9000, "bogus": 1}, width = 512, height = 512)
+ assert mp.lookup(**KEY, width = 512, height = 512, phase = mp.PHASE_PRE_DECODE) == 100
+
+
+def test_store_without_phases_answers_none_per_phase(store):
+ _record(5900)
+ assert mp.lookup(**KEY) == 5900
+ assert mp.lookup(**KEY, phase = mp.PHASE_DECODE) is None
+ assert dm.learned_phase_split(KEY["family"], KEY["quant"], KEY["speed_mode"]) is None
+ mp._cache.update(path = None, mtime = None, data = None)
+ store.unlink()
+ _record(**B200)
+ split = dm.learned_phase_split(KEY["family"], KEY["quant"], KEY["speed_mode"])
+ assert split[mp.PHASE_DECODE] == 6051 and split[mp.PHASE_UNPADDED] == 1864
+
+
+class _FakeCudaTorch:
+ def __init__(self):
+ self.allocated = 0
+ self.peak = 0
+ torch_self = self
+
+ class _Dev:
+ def __init__(self, *args):
+ self.type = "cuda"
+ self.index = 0
+
+ def __str__(self):
+ return "cuda:0"
+
+ class cuda: # noqa: N801
+ is_available = staticmethod(lambda: True)
+ current_device = staticmethod(lambda: 0)
+ synchronize = staticmethod(lambda dev = None: None)
+ memory_allocated = staticmethod(lambda dev = None: torch_self.allocated)
+ max_memory_allocated = staticmethod(lambda dev = None: torch_self.peak)
+
+ @staticmethod
+ def reset_peak_memory_stats(dev = None):
+ torch_self.peak = torch_self.allocated
+
+ self.device = _Dev
+ self.cuda = cuda
+
+ def alloc(self, mib):
+ self.allocated += mib << 20
+ self.peak = max(self.peak, self.allocated)
+
+ def free(self, mib):
+ self.allocated -= mib << 20
+
+
+def test_probe_splits_the_render_at_the_decode_without_resetting(monkeypatch):
+ fake = _FakeCudaTorch()
+ monkeypatch.setitem(sys.modules, "torch", fake)
+ fake.alloc(15000) # resident weights before the render
+ probe = mp.PeakProbe("cuda:0")
+ fake.alloc(2000) # encode
+ fake.free(2000)
+ fake.alloc(1500) # denoise
+ fake.free(1500)
+ probe.mark_decode()
+ peak_before = fake.peak
+ fake.alloc(5900) # untiled decode
+ fake.free(5900)
+ assert fake.peak >= peak_before # nothing reset the shared counters
+ assert probe.finish() == 5900
+ assert probe.phase_peaks == {mp.PHASE_PRE_DECODE: 2000, mp.PHASE_DECODE: 5900}
+
+
+def test_probe_with_two_decodes_reports_no_split(monkeypatch):
+ fake = _FakeCudaTorch()
+ monkeypatch.setitem(sys.modules, "torch", fake)
+ probe = mp.PeakProbe("cuda:0")
+ probe.mark_decode()
+ fake.alloc(800)
+ fake.free(800)
+ probe.mark_decode() # a split batch: chunk 2 encodes after chunk 1 decoded
+ fake.alloc(3000)
+ assert probe.finish() == 3000
+ assert probe.phase_peaks == {}
+
+
+def test_render_records_phases_and_keys_non_txt2img_workflows(store, monkeypatch):
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)
+ # recorded without a device target: the SDPA verdict counts as known
+ monkeypatch.setattr(diffusion, "_attention_scaling_known", lambda target, engaged = None: True)
+ pipe = types.SimpleNamespace(_unsloth_peak_identity = (KEY["family"], KEY["quant"], "default"))
+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default")
+
+ class _Probe:
+ device = "cuda:0"
+ failed_captures = 0
+ window_pad = 64
+
+ def __init__(self, peak, phases):
+ self._peak = peak
+ self.phase_peaks = phases
+
+ def finish(self):
+ return self._peak
+
+ diffusion._record_peak_probe(
+ state,
+ _Probe(5900, {mp.PHASE_PRE_DECODE: 2000, mp.PHASE_DECODE: 5900}),
+ width = 1024,
+ height = 1024,
+ batch_size = 1,
+ workflow = "txt2img",
+ )
+ assert mp.lookup(**KEY) == 5964
+ assert mp.lookup(**KEY, phase = mp.PHASE_PRE_DECODE) == 2064
+ assert mp.lookup(**KEY, phase = mp.PHASE_DECODE) == 5964
+ # nothing streams during the decode, and a load keeping every weight resident holds no window: no pad
+ assert mp.lookup(**KEY, phase = mp.PHASE_DECODE_UNPADDED) == 5900
+ assert mp.lookup(**KEY, phase = mp.PHASE_UNPADDED) == 5900
+ diffusion._record_peak_probe(
+ state, _Probe(7800, {}), width = 1024, height = 1024, batch_size = 1, workflow = "edit"
+ )
+ assert mp.lookup(**KEY) == 5964 # an edit no longer raises the text-to-image key
+ assert mp.lookup(**dict(KEY, quant = KEY["quant"] + "|wf:edit")) == 7864
+
+
+def test_with_workflow_keeps_txt2img_on_the_load_key():
+ assert dm.with_workflow("a|b", None) == "a|b"
+ assert dm.with_workflow("a|b", "txt2img") == "a|b"
+ assert dm.with_workflow("a|b", "edit") == "a|b|wf:edit"
+
+
+def test_edit_request_is_sized_from_its_own_key(qwen_pipe):
+ plan = _qwen_flat_plan()
+ _record(**B200)
+ later = _refine(qwen_pipe, plan)
+ qwen_pipe._unsloth_measured_reserve = (
+ later.estimates["measured_runtime_headroom_mib"],
+ "qwen-image-2.1",
+ "default",
+ None,
+ None,
+ later.estimates["measured_learned_peak_mib"],
+ later.estimates["measured_learned_peak_pixels"],
+ False,
+ True,
+ False,
+ )
+ kw = dict(width = 1024, height = 1024, batch_size = 1, cfg = False)
+ assert dm.measured_request_extra_mib(qwen_pipe, **kw, workflow = "txt2img") == 0
+ assert (
+ dm.measured_request_extra_mib(qwen_pipe, **kw, workflow = "edit") == 0
+ ) # nothing learned for edits yet
+ _record(7800, quant = KEY["quant"] + "|wf:edit")
+ # 7800 x 1.15 rounded to 256 = 9216, past the 7168 the load reserved
+ assert dm.measured_request_extra_mib(qwen_pipe, **kw, workflow = "edit") == 9216 - 7168
+ assert dm.measured_request_extra_mib(qwen_pipe, **kw, workflow = "txt2img") == 0
+
+
+# ---------------------------------------------------------------------------------------------------- decode guard
+
+
+class _GuardTorch:
+ def __init__(
+ self,
+ free_mib,
+ reserved_mib = 0,
+ allocated_mib = 0,
+ ):
+ outer = self
+ self.free = free_mib
+ self.reserved = reserved_mib
+ self.allocated = allocated_mib
+
+ class device: # noqa: N801
+ def __init__(
+ self,
+ spec,
+ index = None,
+ ):
+ spec = str(spec)
+ self.type = spec.split(":")[0]
+ self.index = (
+ index
+ if index is not None
+ else (int(spec.split(":")[1]) if ":" in spec else None)
+ )
+
+ class cuda: # noqa: N801
+ current_device = staticmethod(lambda: 0)
+ mem_get_info = staticmethod(lambda dev = None: (outer.free << 20, 24576 << 20))
+ memory_reserved = staticmethod(lambda dev = None: outer.reserved << 20)
+ memory_allocated = staticmethod(lambda dev = None: outer.allocated << 20)
+
+ self.device = device
+ self.cuda = cuda
+
+
+def test_decode_streams_lent_encoders_only_when_free_memory_is_short(monkeypatch):
+ calls = []
+ monkeypatch.setattr(
+ dm,
+ "release_resident_groups",
+ lambda pipe, need, logger = None, reason = "", device = None, **kw: calls.append(need)
+ or (lambda: None),
+ )
+ pipe = types.SimpleNamespace()
+ pipe._unsloth_decode_lend = (4040, 5900, 1024 * 1024, False)
+ need = dm.decode_need_mib(pipe._unsloth_decode_lend, width = 1024, height = 1024)
+ assert need == 6785 # 5900 x 1.15
+ monkeypatch.setitem(sys.modules, "torch", _GuardTorch(free_mib = 7500))
+ assert dm.release_for_decode(pipe, width = 1024, height = 1024, device = "cuda:0") is None
+ assert calls == []
+ # cached allocator blocks count as free
+ monkeypatch.setitem(
+ sys.modules, "torch", _GuardTorch(free_mib = 6000, reserved_mib = 3000, allocated_mib = 1000)
+ )
+ assert dm.release_for_decode(pipe, width = 1024, height = 1024, device = "cuda:0") is None
+ monkeypatch.setitem(sys.modules, "torch", _GuardTorch(free_mib = 5000))
+ assert dm.release_for_decode(pipe, width = 1024, height = 1024, device = "cuda:0") is not None
+ assert calls == [6785 + dm._DECODE_SLACK_MIB - 5000]
+ # a larger canvas scales the learned decode the way the store does
+ assert dm.decode_need_mib(pipe._unsloth_decode_lend, width = 2048, height = 2048) == 6785 * 4
+ # no lend, no check
+ assert dm.release_for_decode(types.SimpleNamespace(), width = 1024, height = 1024) is None
+
+
+def test_edit_only_family_records_under_the_load_key(store, monkeypatch):
+ # an edit-only family runs nothing but edits: its renders keep teaching the key its next load places from
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)
+ monkeypatch.setattr(diffusion, "_attention_scaling_known", lambda target, engaged = None: True)
+ pipe = types.SimpleNamespace(_unsloth_peak_identity = (KEY["family"], KEY["quant"], "default"))
+ family = types.SimpleNamespace(edit = True)
+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default", family = family)
+
+ class _Probe:
+ device = "cuda:0"
+ failed_captures = 0
+ window_pad = 0
+ phase_peaks = {}
+
+ def finish(self):
+ return 7800
+
+ diffusion._record_peak_probe(
+ state, _Probe(), width = 1024, height = 1024, batch_size = 1, workflow = "edit"
+ )
+ assert mp.lookup(**KEY) == 7800
+ assert mp.lookup(**dict(KEY, quant = KEY["quant"] + "|wf:edit")) is None
+ assert diffusion._peak_workflow(family, "edit") is None
+ assert diffusion._peak_workflow(types.SimpleNamespace(unified_edit = True), "edit") == "edit"
+
+
+def test_cfg_off_edit_request_reads_its_cfg_off_workflow_key(qwen_pipe):
+ plan = _qwen_flat_plan()
+ _record(**B200)
+ later = _refine(qwen_pipe, plan)
+ qwen_pipe._unsloth_peak_expects_cfg = True
+ qwen_pipe._unsloth_peak_identity = (KEY["family"], KEY["quant"], "default")
+ qwen_pipe._unsloth_measured_reserve = (
+ later.estimates["measured_runtime_headroom_mib"],
+ "qwen-image-2.1",
+ "default",
+ None,
+ None,
+ later.estimates["measured_learned_peak_mib"],
+ later.estimates["measured_learned_peak_pixels"],
+ False,
+ True,
+ False,
+ )
+ kw = dict(width = 1024, height = 1024, batch_size = 1, workflow = "edit")
+ # what the recorder writes for a guidance-off edit on a CFG-capable family
+ _record(7800, quant = KEY["quant"] + "|cfg:off|wf:edit")
+ assert dm.measured_request_extra_mib(qwen_pipe, **kw, cfg = False) == 9216 - 7168
+ # a CFG request is not covered by the guidance-off sample
+ assert dm.measured_request_extra_mib(qwen_pipe, **kw, cfg = True) == 0
+
+
+def _seeded_edit_pipe(pipe):
+ """A phase-seeded load that reserved 7168 MiB from a 6051 MiB learned whole-render peak."""
+ pipe._unsloth_peak_expects_cfg = True
+ pipe._unsloth_peak_identity = (KEY["family"], KEY["quant"], "default")
+ pipe._unsloth_measured_reserve = (
+ 7168,
+ "qwen-image-2.1",
+ "default",
+ None,
+ None,
+ 6051,
+ 1024 * 1024,
+ False,
+ True,
+ False,
+ )
+ return pipe
+
+
+def test_edit_workflow_peak_does_not_charge_the_condition_images_twice(qwen_pipe):
+ pipe = _seeded_edit_pipe(qwen_pipe)
+ kw = dict(
+ width = 1024,
+ height = 1024,
+ batch_size = 1,
+ cfg = False,
+ workflow = "edit",
+ condition_pixels = int(1024 * 1024 * 0.32),
+ )
+ # 7168 seed reserve + 3014 condition allowance - 7168 headroom: what main charges, store or no store
+ assert dm.measured_request_extra_mib(pipe, **kw) == 3014
+ # a learned edit peak (7800 x 1.15 -> 9216) already holds the condition images; it is an alternative to the seed
+ # estimate plus allowance (10182), not a base the allowance is added on top of (that gave 5062)
+ _record(7800, quant = KEY["quant"] + "|cfg:off|wf:edit")
+ assert dm.measured_request_extra_mib(pipe, **kw) == 3014
+
+
+def test_edit_workflow_peak_above_seed_plus_allowance_is_followed(qwen_pipe):
+ pipe = _seeded_edit_pipe(qwen_pipe)
+ kw = dict(width = 1024, height = 1024, batch_size = 1, cfg = False, workflow = "edit")
+ cond = int(1024 * 1024 * 0.32)
+ # 12000 x 1.15 rounded to 256 = 13824, past 7168 + 3014
+ _record(12000, quant = KEY["quant"] + "|cfg:off|wf:edit")
+ assert dm.measured_request_extra_mib(pipe, **kw, condition_pixels = cond) == 13824 - 7168
+ assert dm.measured_request_extra_mib(pipe, **kw, condition_pixels = 0) == 13824 - 7168
+
+
+def test_decode_headroom_does_not_credit_cuda_graph_pool_cache(monkeypatch):
+ # a graph pool's free blocks sit inside reserved - allocated yet an ordinary VAE allocation cannot reuse them
+ import core.inference.diffusion_cuda_graph as cg
+
+ calls = []
+ monkeypatch.setattr(
+ dm,
+ "release_resident_groups",
+ lambda pipe, need, logger = None, reason = "", device = None, **kw: calls.append(need)
+ or (lambda: None),
+ )
+ pipe = types.SimpleNamespace(_unsloth_decode_lend = (4040, 5900, 1024 * 1024, False))
+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 2000 << 20)
+ monkeypatch.setitem(
+ sys.modules, "torch", _GuardTorch(free_mib = 6000, reserved_mib = 3000, allocated_mib = 1000)
+ )
+ assert dm.release_for_decode(pipe, width = 1024, height = 1024, device = "cuda:0") is not None
+ assert calls == [6785 + dm._DECODE_SLACK_MIB - 6000]
+
+
+def test_render_whose_decode_released_lent_groups_records_nothing(store, monkeypatch):
+ # the decode ran net of the streamed groups' bytes: its peak (and whole-render peak) would under-size the bucket
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)
+ monkeypatch.setattr(diffusion, "_attention_scaling_known", lambda target, engaged = None: True)
+ pipe = types.SimpleNamespace(_unsloth_peak_identity = (KEY["family"], KEY["quant"], "default"))
+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default")
+ finished = []
+
+ class _Probe:
+ device = "cuda:0"
+ failed_captures = 0
+ window_pad = 0
+ decode_released = True
+ phase_peaks = {mp.PHASE_PRE_DECODE: 2000, mp.PHASE_DECODE: 2000}
+
+ def finish(self):
+ finished.append(1)
+ return 2000
+
+ diffusion._record_peak_probe(
+ state, _Probe(), width = 1024, height = 1024, batch_size = 1, workflow = "txt2img"
+ )
+ assert finished == [1] # the probe's window still closes
+ assert mp.lookup(**KEY) is None
+ assert mp.lookup(**KEY, phase = mp.PHASE_DECODE_UNPADDED) is None
+
+
+# ---------------------------------------------------------------------------------------------------- v1 store
+
+
+def _write_v1(store, entries):
+ store.write_text(
+ json.dumps({"version": 1, "runtime": mp.runtime_stamp(), "entries": entries}),
+ encoding = "utf-8",
+ )
+ mp._cache.update(path = None, mtime = None, data = None)
+
+
+def test_pre_workflow_store_peak_no_longer_sizes_txt2img(qwen_pipe, store):
+ """Before workflow keys every workflow recorded max-wins under the load key: an old edit peak there must not
+ reserve for text-to-image (nor block the phase split) once a txt2img render records its own peak."""
+ plan = _qwen_flat_plan()
+ fresh = _refine(qwen_pipe, plan)
+ base = mp.model_key(
+ KEY["family"], KEY["quant"], KEY["speed_mode"], 1, "NVIDIA GeForce RTX 3090"
+ )
+ bucket = str(1024 * 1024)
+ edit = mp.model_key(
+ KEY["family"], KEY["quant"] + "|wf:edit", KEY["speed_mode"], 1, "NVIDIA GeForce RTX 3090"
+ )
+ other = mp.model_key(
+ "flux.1-dev", KEY["quant"], KEY["speed_mode"], 1, "NVIDIA GeForce RTX 3090"
+ )
+ phased = {name: {bucket: value} for name, value in _phases(900, 1864).items()}
+ _write_v1(
+ store,
+ {
+ base: {"peaks": {bucket: 13000}}, # an edit render's peak, pre-PR
+ edit: {"peaks": {bucket: 9000}},
+ # the per-workflow format: the inflated whole-render value falls back to its phases
+ other: {"peaks": {bucket: 13000}, "phases": phased},
+ },
+ )
+ assert mp.lookup(**KEY) is None
+ assert mp.lookup(**dict(KEY, quant = KEY["quant"] + "|wf:edit")) == 9000
+ assert mp.lookup(family = "flux.1-dev", quant = KEY["quant"], speed_mode = "default") == 6051
+ assert (
+ mp.lookup(
+ family = "flux.1-dev",
+ quant = KEY["quant"],
+ speed_mode = "default",
+ phase = mp.PHASE_UNPADDED,
+ )
+ == 1864
+ )
+ legacy = _refine(qwen_pipe, plan)
+ assert legacy.estimates.get("measured_learned_peak_mib") is None
+ assert (
+ legacy.estimates["measured_runtime_headroom_mib"]
+ == fresh.estimates["measured_runtime_headroom_mib"]
+ )
+ assert legacy.resident_text_encoder_mib == fresh.resident_text_encoder_mib == 8917
+ # a fresh txt2img sample is learned, the split works again, and the store is rewritten as version 2
+ _record(**B200)
+ assert mp.lookup(**KEY) == B200["peak"]
+ assert dm.learned_phase_split(KEY["family"], KEY["quant"], KEY["speed_mode"]) is not None
+ later = _refine(qwen_pipe, plan)
+ assert later.resident_text_encoder_mib == 8917
+ assert later.estimates["decode_lend_text_encoder_mib"] == 8917 - 4621
+ data = json.loads(store.read_text(encoding = "utf-8"))
+ assert data["version"] == mp._VERSION and set(data["entries"]) == {edit, other, base}