Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: learned peaks per render phase, so later loads keep text encoders resident (#14979)
Daniel Han93fc9c730f61
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex ea7cbab442d..27d2fe9fa68 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -222,9 +222,13 @@ from .diffusion_memory import (with_label_part,applied_plan,release_resident_groups,+ release_for_decode,resident_group_mib,hook_resident_denoiser,measured_request_extra_mib,+ DECODE_LEND_ATTR,+ decode_lend_from_plan,+ with_workflow,settled_snapshot_device_memory,snapshot_device_memory,_torchao_stream_pinnable,@@ -2531,6 +2535,9 @@ def _drop_measured_reserve_for_layer_split(pipe: Any, plan: Any) -> bool:return Falsetry:pipe._unsloth_measured_reserve = None+ setattr(+ pipe, DECODE_LEND_ATTR, None+ ) # the lent encoder residency was priced on the same one-card planexcept Exception: # noqa: BLE001return Falsereturn True@@ -2939,17 +2946,30 @@ def _record_peak_probe(height: Optional[int],batch_size: int,target: Any = None,+ workflow: Optional[str] = None,) -> None:- """Persist a completed render's peak above its starting allocation under the load's peak identity."""+ """Persist a completed render's peak above its starting allocation under the load's peak identity, keyed for+ ``workflow`` (``with_workflow``: text-to-image keeps the load's key), with its pre-decode / decode phases."""if probe is None or not width or not height:returnidentity = getattr(getattr(state, "pipe", None), "_unsloth_peak_identity", None)try:- from .diffusion_measured_peaks import device_name, record+ from .diffusion_measured_peaks import (+ PHASE_DECODE,+ PHASE_DECODE_UNPADDED,+ PHASE_PRE_DECODE,+ PHASE_UNPADDED,+ device_name,+ record,+ )peak = probe.finish()if peak is None or not isinstance(identity, tuple) or len(identity) != 3:return+ if getattr(probe, "decode_released", False):+ # the decode streamed lent groups out, so its peak is net of their bytes: a sample would under-size the+ # bucket (the placement that lent them already held a sample covering this decode)+ returnif not _peak_identity_matches(state, identity):# the configuration moved during the render (a compile fallback, a toggled cache): no key names itreturn@@ -2972,7 +2992,19 @@ def _record_peak_probe(index = getattr(probe.device, "index", None)graphs = cuda_graph.live_graph_count(index)generation = cuda_graph.capture_generation(index)- peak += int(getattr(probe, "window_pad", 0) or 0)+ pad = int(getattr(probe, "window_pad", 0) or 0)+ peak += pad+ split = getattr(probe, "phase_peaks", None) or {}+ phases = {}+ if PHASE_PRE_DECODE in split and PHASE_DECODE in split:+ pre, dec = int(split[PHASE_PRE_DECODE]), int(split[PHASE_DECODE])+ phases = {+ PHASE_PRE_DECODE: pre + pad,+ PHASE_DECODE: dec + pad,+ PHASE_DECODE_UNPADDED: dec,+ PHASE_UNPADDED: max(pre, dec),+ }+ quant = with_workflow(quant, _peak_workflow(getattr(state, "family", None), workflow))held: list = []raised = record(family = family,@@ -2985,6 +3017,7 @@ def _record_peak_probe(device_name = device_name(probe.device),quadratic = _quadratic_attention(target, engaged, state.pipe),held = held,+ phases = phases,)if held:# the store holds this sample, so it covers the graphs the render recorded; unpersisted, they stay dirty@@ -9993,6 +10026,7 @@ class DiffusionBackend:plan.estimates.get("measured_static_seed", True),plan.estimates.get("measured_learned_cfg_off", False),)+ setattr(pipe, DECODE_LEND_ATTR, decode_lend_from_plan(plan))from .diffusion_qwenimage21_vision import configure_vision_attention@@ -13231,6 +13265,7 @@ class DiffusionBackend:condition_pixels = guard_condition_pixels,cfg = guard_cfg,unpriced_references = guard_unpriced_refs,+ workflow = _peak_workflow(fam, workflow),)if extra_mib > 0:releasable_mib = resident_group_mib(state.pipe)@@ -13471,6 +13506,7 @@ class DiffusionBackend:except Exception: # noqa: BLE001 - an unknown guidance mode records nothingstate.pipe._unsloth_peak_cfg = Nonepeak_probe = _start_peak_probe(state)+ decode_restores: list = []try:previewer = _start_previewer()pending = list(chunks)@@ -13548,6 +13584,22 @@ class DiffusionBackend:"""The decoder was entered: a HOST position, possibly far ahead of the GPU. Mark theboundary in the stream and let the poller flip once the GPU reaches it; with no eventto wait on (CPU, MPS) there is no queue to be ahead of, so flip now."""+ if peak_probe is not None:+ peak_probe.mark_decode()+ # encoder residency lent to the decode streams again only if free memory is short now+ restore = release_for_decode(+ state.pipe,+ width = guard_width,+ height = guard_height,+ batch_size = len(chunk),+ device = getattr(guard_target, "torch_device", None),+ logger = logger,+ )+ if restore is not None:+ decode_restores.append(restore)+ if peak_probe is not None:+ # the decode ran with fewer weights than the probe's baseline: its peak understates+ peak_probe.decode_released = Truemarked = Falsefor _ in range(_BOUNDARY_MARK_ATTEMPTS):with _hold_off_cuda_graph_capture() as clear:@@ -13614,6 +13666,14 @@ class DiffusionBackend:)continuefinally:+ while decode_restores:+ try:+ decode_restores.pop()()+ except Exception as restore_exc: # noqa: BLE001 - the groups keep streaming+ logger.warning(+ "diffusion.memory: re-pinning groups after the decode failed (%s)",+ restore_exc,+ )# A guarded block that fell back to eager (compile failed at its first forward) no longer# runs compiled: report it on every exit, cancel and error included, so status, LoRA gating# and the compile-cache shape registry stop treating it as compiled.@@ -13662,6 +13722,7 @@ class DiffusionBackend:height = guard_height,batch_size = max(chunk_shapes or [1]),target = guard_target,+ workflow = workflow,)if static_skip:logger.debug("diffusion.step_skip: %s", static_skip_stats(state.pipe))@@ -14207,6 +14268,17 @@ def _hires_status(fam: DiffusionFamily) -> dict[str, Any]:}+def _peak_workflow(fam: Any, workflow: Optional[str]) -> Optional[str]:+ """``workflow`` for the learned-peak key, None when it is the family's only workflow (an edit-only family): those+ renders are what its loads place from, so they stay on the load key."""+ try:+ if fam is not None and _family_workflows(fam) == [str(workflow or "").strip().lower()]:+ return None+ except Exception: # noqa: BLE001+ pass+ return workflow++def _family_workflows(fam: DiffusionFamily) -> list[str]:"""The workflow ids the diffusers engine can run for ``fam`` (drives UI gating)."""# Instruction-editing families have no txt2img mode, so expose only "edit".diff --git a/studio/backend/core/inference/diffusion_measured_peaks.py b/studio/backend/core/inference/diffusion_measured_peaks.pyindex 1b631288676..86008d6945d 100644--- a/studio/backend/core/inference/diffusion_measured_peaks.py+++ b/studio/backend/core/inference/diffusion_measured_peaks.py@@ -11,6 +11,10 @@ measurement above the seed means this machine needs more) and supplies one whereseed, because one render does not exercise every phase (a prompt-cache hit skips the encoder, a smaller requestdecodes less), and the seed was measured on the worst phase.+Phases: ``pre_decode`` / ``decode`` on the whole-render scale (prefetch window pad included), ``decode_unpadded`` /+``unpadded`` without it (only the VAE runs at decode; resident weights hold no window). Pre-phase stores place from the+whole-render peak. ``UNSLOTH_DIFFUSION_PHASE_PEAKS=0`` ignores the phases.+``UNSLOTH_DIFFUSION_LEARNED_PEAKS=0`` turns lookups and recording off; ``UNSLOTH_DIFFUSION_LEARNED_PEAKS_FILE``names the JSON file (default ``<studio cache>/diffusion_measured_peaks.json``)."""@@ -29,8 +33,17 @@ from typing import Any, OptionalLEARNED_PEAKS_ENV = "UNSLOTH_DIFFUSION_LEARNED_PEAKS"LEARNED_PEAKS_FILE_ENV = "UNSLOTH_DIFFUSION_LEARNED_PEAKS_FILE"+PHASE_PEAKS_ENV = "UNSLOTH_DIFFUSION_PHASE_PEAKS"+PHASE_PRE_DECODE = "pre_decode"+PHASE_DECODE = "decode"+PHASE_DECODE_UNPADDED = "decode_unpadded"+PHASE_UNPADDED = "unpadded"+PHASES = (PHASE_PRE_DECODE, PHASE_DECODE, PHASE_DECODE_UNPADDED, PHASE_UNPADDED)_FILE_NAME = "diffusion_measured_peaks.json"-_VERSION = 1+# 2: non-txt2img workflows record under their own ``wf:`` key. Version 1 kept every workflow's max-wins peak on the+# load key (an edit's larger peak inflated txt2img forever), so ``_migrate_v1`` keeps only what the phase format wrote.+_VERSION = 2+_WORKFLOW_KEY_PART = "|wf:"_MAX_ENTRIES = 512_MEGAPIXEL = 1024 * 1024# Smallest bucket: a quarter megapixel. Below it the activation is dominated by fixed costs.@@ -49,6 +62,15 @@ def enabled() -> bool:)+def phases_enabled() -> bool:+ return enabled() and (os.environ.get(PHASE_PEAKS_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )++def store_path() -> Optional[Path]:explicit = (os.environ.get(LEARNED_PEAKS_FILE_ENV) or "").strip()if explicit:@@ -137,6 +159,34 @@ def _empty() -> dict:return {"version": _VERSION, "runtime": runtime_stamp(), "entries": {}}+def _migrate_v1(entries: dict) -> dict:+ """A version 1 store's entries on the version 2 scale. ``wf:`` keys and the phase tables only ever came from the+ per-workflow format and stay as they are. A load key's whole-render peaks may hold another workflow's sample, so+ a bucket keeps only the larger of its pre-decode / decode phases (each a whole-render peak of a phase-recorded+ render, which ran under the per-workflow key); a bucket or entry with no phase is dropped and relearned, the+ load placing from its seed or the flat estimate meanwhile, as on a fresh store."""+ kept: dict = {}+ for key, entry in entries.items():+ if not isinstance(entry, dict):+ continue+ if _WORKFLOW_KEY_PART in str(key):+ kept[key] = entry+ continue+ table = entry.get("phases") if isinstance(entry.get("phases"), dict) else {}+ peaks: dict = {}+ for name in (PHASE_PRE_DECODE, PHASE_DECODE):+ for bucket, value in (table.get(name) or {}).items():+ try:+ value = int(value)+ except (TypeError, ValueError):+ continue+ if value > int(peaks.get(bucket) or 0):+ peaks[str(bucket)] = value+ if peaks:+ kept[key] = dict(entry, peaks = peaks)+ return kept++def _read(path: Path, fresh: bool = False) -> dict:try:mtime = path.stat().st_mtime_ns@@ -151,6 +201,13 @@ def _read(path: Path, fresh: bool = False) -> dict:return _cache["data"]try:data = json.loads(path.read_text(encoding = "utf-8"))+ if (+ isinstance(data, dict)+ and data.get("version") == 1+ and data.get("runtime") == runtime_stamp()+ and isinstance(data.get("entries"), dict)+ ):+ data = dict(data, version = _VERSION, entries = _migrate_v1(data["entries"]))if (not isinstance(data, dict)or data.get("version") != _VERSION@@ -212,11 +269,13 @@ def record(device_name: Optional[str] = None,quadratic: bool = False,held: Optional[list] = None,+ phases: Optional[dict] = None,) -> bool:"""Keep the max peak seen for this key and bucket, scaled to the bucket's ceiling so a smaller render in the bucketstill answers a full-size one (squared with ``quadratic``: SDPA math attention grows with pixels squared, and thekey's entry keeps scaling that way). Returns True when the stored value rose. ``held`` (a list) gets True appended- when the store holds at least this peak afterwards (raised, or already covered): False when nothing was persisted."""+ when the store holds at least this peak afterwards (raised, or already covered): False when nothing was persisted.+ ``phases`` ({name: MiB}, capped at ``peak_mib``) are kept max-wins and never affect the return value."""if (not enabled()or peak_mib is None@@ -235,12 +294,13 @@ def record(bucket_pixels = resolution_bucket(width, height, batch_size)bucket = str(bucket_pixels)pixels = max(1, int(width or 1024)) * max(1, int(height or 1024)) * max(1, int(batch_size or 1))- peak_mib = int(- math.ceil(- int(peak_mib)- * (bucket_pixels / float(max(pixels, bucket_pixels // 2))) ** (2 if quadratic else 1)- )- )+ factor = (bucket_pixels / float(max(pixels, bucket_pixels // 2))) ** (2 if quadratic else 1)+ peak_mib = int(math.ceil(int(peak_mib) * factor))+ phase_mib = {+ name: min(peak_mib, int(math.ceil(int(value) * factor)))+ for name, value in (phases or {}).items()+ if name in PHASES and value is not None and int(value) > 0+ }try:with _lock, _file_lock(path):data = _read(path, fresh = True)@@ -249,7 +309,17 @@ def record(entries[key] = entry # most recently raised last: the cap evicts from the frontpeaks = entry.setdefault("peaks", {})old = int(peaks.get(bucket) or 0)+ phase_raised = False+ if phase_mib:+ table = entry.setdefault("phases", {})+ for name, value in phase_mib.items():+ by_bucket = table.setdefault(name, {})+ if int(value) > int(by_bucket.get(bucket) or 0):+ by_bucket[bucket] = int(value)+ phase_raised = Trueif int(peak_mib) <= old:+ if phase_raised:+ _write(path, data)if held is not None:held.append(True)return False@@ -279,9 +349,11 @@ def lookup(batch_size: int = 1,frames: int = 1,device: Optional[str] = None,+ phase: Optional[str] = None,) -> Optional[int]:"""Learned peak MiB (no margin) for a request of this size, or None when nothing was recorded.- ``device`` (GPU name) defaults to the current CUDA device, the one the planner snapshots.+ ``device`` (GPU name) defaults to the current CUDA device, the one the planner snapshots. ``phase``+ (one of ``PHASES``) reads that phase's peak instead of the whole render's: None when the entry has none.The smallest recorded bucket at or above the request's bucket answers as is (an overestimate); with only smallerbuckets recorded, the largest one is scaled linearly by pixel count."""@@ -294,6 +366,7 @@ def lookup(batch_size = batch_size,frames = frames,device = device,+ phase = phase,)return None if found is None else found[0]@@ -317,6 +390,7 @@ def lookup_bucket(batch_size: int = 1,frames: int = 1,device: Optional[str] = None,+ phase: Optional[str] = None,) -> Optional[tuple[int, int]]:"""``lookup`` plus the pixels the peak covers and whether it scales squared: (peak MiB, bucket pixels,quadratic), or None. A ``graphs:on`` label (a load placed before it knows whether placement keeps the whole-step@@ -335,6 +409,7 @@ def lookup_bucket(batch_size = batch_size,frames = frames,device = device,+ phase = phase,)for mode in ("step", "blocks")]@@ -350,6 +425,8 @@ def lookup_bucket()if not enabled() or not family or not identified(quant):return None+ if phase is not None and (phase not in PHASES or not phases_enabled()):+ return Nonepath = store_path()if path is None:return None@@ -357,9 +434,12 @@ def lookup_bucket(with _lock:key = model_key(family, quant, speed_mode, frames, device or current_device_name())entry = (_read(path).get("entries") or {}).get(key)- peaks = {- int(k): int(v) for k, v in ((entry or {}).get("peaks") or {}).items() if int(v) > 0- }+ table = (+ (entry or {}).get("peaks")+ if phase is None+ else ((entry or {}).get("phases") or {}).get(phase)+ )+ peaks = {int(k): int(v) for k, v in (table or {}).items() if int(v) > 0}except Exception: # noqa: BLE001return Noneif not peaks:@@ -489,6 +569,25 @@ class PeakProbe:self.device = devexcept Exception: # noqa: BLE001 - measurement onlyself.device = None+ self.pre_decode = None+ self.decodes = 0+ self.phase_peaks: dict = {}++ def mark_decode(self) -> None:+ """Read the pre-decode peak without a reset (other measurements share the counters). A second decode (split+ batch) interleaves phases, so no split is reported."""+ if self.device is None or self.base is None:+ return+ self.decodes += 1+ if self.decodes > 1:+ self.pre_decode = None+ return+ try:+ import torch+ peak = int(torch.cuda.max_memory_allocated(self.device)) - self.base+ except Exception: # noqa: BLE001+ return+ self.pre_decode = -(-peak // (1024 * 1024)) if peak > 0 else 0def finish(self) -> Optional[int]:if self.device is None or self.base is None:@@ -508,7 +607,14 @@ class PeakProbe:# closed only after the read: a render arming in between would otherwise reset the peak unseenif not close_window(self.device, getattr(self, "window", None)) or peak is None:return None- return -(-peak // (1024 * 1024)) if peak > 0 else None+ mib = -(-peak // (1024 * 1024)) if peak > 0 else None+ if mib is not None and self.decodes == 1 and self.pre_decode is not None:+ # the decode's own peak is unknown when it stayed under the earlier phases: the whole render bounds it+ self.phase_peaks = {+ PHASE_PRE_DECODE: max(1, min(int(self.pre_decode), mib)),+ PHASE_DECODE: mib,+ }+ return mibdef _range_bytes(dev: Any) -> int:diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 5133f7fa090..782a9aba80e 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -19,6 +19,7 @@ from __future__ import annotationsimport contextlibimport contextvarsimport functools+import mathimport osimport sysimport threading@@ -1026,6 +1027,23 @@ def _metal_working_set_mib() -> tuple[Optional[int], int]:return (working_set, in_use) if working_set > 0 else (None, 0)+def _graph_held_cache_bytes(device: Optional[int] = None) -> int:+ """Bytes inside ``memory_reserved() - memory_allocated()`` that ordinary allocations cannot reuse: the CUDA graph+ pools (per-block pools whole, the whole-step graphs' shared pool's unallocated part)."""+ held = 0+ try:+ from .diffusion_block_graph import pool_bytes as _block_graph_pool_bytes+ held += int(_block_graph_pool_bytes(device))+ except Exception: # noqa: BLE001+ pass+ try:+ from .diffusion_cuda_graph import live_pool_free_bytes+ held += int(live_pool_free_bytes())+ except Exception: # noqa: BLE001 -- no graph layer: nothing held+ pass+ return held++def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:"""``snapshot_device_memory`` with the caching allocator's RECLAIMABLE bytes credited back as free,without flushing it.@@ -1062,21 +1080,10 @@ def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:except Exception: # noqa: BLE001 -- no allocator reading: the plain snapshot still standsreturn snapshottry:- # Per-block graph pools are reserved, mostly unallocated between replays, and never reusable by anything else.- from .diffusion_block_graph import pool_bytes as _block_graph_pool_bytes- try:- device = torch.cuda.current_device()- except Exception: # noqa: BLE001 - no current CUDA device: count every pool- device = None- reclaimable -= _block_graph_pool_bytes(device)- except Exception: # noqa: BLE001- pass- try:- # The whole-step graphs' shared pool: only its unallocated part sits inside the difference above.- from .diffusion_cuda_graph import live_pool_free_bytes- reclaimable -= live_pool_free_bytes()- except Exception: # noqa: BLE001 -- no graph layer: nothing held- pass+ device = torch.cuda.current_device()+ except Exception: # noqa: BLE001 - no current CUDA device: count every pool+ device = None+ reclaimable -= _graph_held_cache_bytes(device)if reclaimable <= 0:return snapshotfree = int(snapshot.free_mib) + reclaimable // (1024 * 1024)@@ -3065,8 +3072,10 @@ def measured_image_runtime_mib(phase_seeds: bool = False,static_seed: bool = True,dense_tier_seeds: bool = False,+ phase: Optional[str] = None,) -> Optional[int]:"""Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate).+ ``phase`` (``diffusion_measured_peaks.PHASES``) reads that phase's learned peak instead of the whole render's.``dense_transformer_mib`` selects the dense eager table; ``compute_bytes`` scales it (fp32 Qwen-Image: 8480 vs 4248).Seeds: the peak tables above, then the per-phase calibration (``_MEASURED_IMAGE_ACTIVATION_MIB``, the same@@ -3104,7 +3113,13 @@ def measured_image_runtime_mib(if quant is not None:from .diffusion_measured_peaks import lookup as learned_peaklearned = learned_peak(- family = family, quant = quant, speed_mode = speed_mode, width = w, height = h, batch_size = batch+ family = family,+ quant = quant,+ speed_mode = speed_mode,+ width = w,+ height = h,+ batch_size = batch,+ phase = phase,)if seed is None and learned is None:return None@@ -3179,6 +3194,43 @@ def with_label_part(label: str, name: str, value: str) -> str:)+# Non-txt2img workflows record under ``wf:<workflow>`` so an edit's larger peak never inflates the next txt2img load.+WORKFLOW_PART = "wf"+TXT2IMG_WORKFLOW = "txt2img"+++def with_workflow(label: str, workflow: Optional[str]) -> str:+ """``label`` keyed for ``workflow``: unchanged for text-to-image (and an unknown workflow), else ``|wf:<name>``."""+ name = str(workflow or "").strip().lower()+ if not name or name == TXT2IMG_WORKFLOW:+ return str(label)+ return f"{label}|{WORKFLOW_PART}:{name}"+++def learned_phase_split(+ family: Optional[str], quant: Optional[str], speed_mode: Optional[str]+) -> Optional[dict]:+ """Learned phases (+ ``pixels`` / ``quadratic``) at the default bucket, or None unless every phase shares the+ whole-render bucket and the larger padded phase reaches the whole-render peak."""+ from .diffusion_measured_peaks import PHASE_DECODE, PHASE_PRE_DECODE, PHASES, lookup_bucket++ if quant is None:+ return None+ total = lookup_bucket(family = family, quant = quant, speed_mode = speed_mode)+ if total is None:+ return None+ found = {}+ for phase in PHASES:+ value = lookup_bucket(family = family, quant = quant, speed_mode = speed_mode, phase = phase)+ if value is None or value[1] != total[1]:+ return None+ found[phase] = int(value[0])+ if max(found[PHASE_PRE_DECODE], found[PHASE_DECODE]) < int(total[0]):+ return None+ found.update(pixels = int(total[1]), quadratic = bool(total[2]))+ return found++_VAE_SAVERS = (("use_tiling", "enable_vae_tiling", "enable_tiling"),("use_slicing", "enable_vae_slicing", "enable_slicing"),@@ -3642,6 +3694,33 @@ def refine_plan_from_loaded_weights(# The flat plan's hooks stay installed and whole groups are kept resident within the room, so an oversized# request can stream them again (release_resident_groups) and run exactly as the flat plan would.stream_te = bool(getattr(plan, "stream_text_encoders", False))+ # The whole-render peak includes the stream prefetch window, which an all-resident placement never allocates:+ # all-resident is checked against the unpadded peak, else encoders against the padded pre-decode phase. Extra+ # residency is lent to the decode; release_for_decode re-streams it only if free memory is short then.+ pre_headroom = resident_headroom = None+ split = None+ if stream_te and policy == OFFLOAD_GROUP:+ split = learned_phase_split(family, read_quant, speed_mode)+ if split is not None:+ from .diffusion_measured_peaks import PHASE_PRE_DECODE, PHASE_UNPADDED++ def _phase_headroom(phase: str) -> Optional[int]:+ value = measured_image_runtime_mib(+ family,+ speed_mode,+ dense_transformer_mib = dense_mib,+ compute_bytes = compute_bytes or 2,+ quant = read_quant,+ phase_seeds = bool(getattr(pipe, "_unsloth_phase_seeds", False)),+ dense_tier_seeds = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False)),+ static_seed = static_seed,+ phase = phase,+ )+ return value if value is not None and value < headroom else None++ pre_headroom = _phase_headroom(PHASE_PRE_DECODE)+ if not bool(getattr(plan, "stream_transformer", True)):+ resident_headroom = _phase_headroom(PHASE_UNPADDED)room = budget - floorif policy == OFFLOAD_GROUP and not stream_te:room -= encoders # resident companions@@ -3674,6 +3753,30 @@ def refine_plan_from_loaded_weights(te_room = (min(max(0, room - dit_room), encoders) if stream_te and policy == OFFLOAD_GROUP else 0)+ lend = 0+ phase_headroom = None+ if split is not None and not whole_dit and not wide_dit:+ phase_te_room = te_room+ if (+ resident_headroom is not None+ and room + (headroom - resident_headroom) - dit_room >= encoders+ ):+ phase_headroom, phase_te_room = resident_headroom, encoders+ elif pre_headroom is not None:+ phase_headroom = pre_headroom+ phase_te_room = min(max(0, room + (headroom - pre_headroom) - dit_room), encoders)+ lend = max(0, phase_te_room - te_room)+ if lend > 0:+ from .diffusion_measured_peaks import PHASE_DECODE_UNPADDED+ te_room = phase_te_room+ estimates.update(+ measured_phase_headroom_mib = phase_headroom,+ measured_phase_all_resident = phase_headroom == resident_headroom,+ measured_decode_peak_mib = split[PHASE_DECODE_UNPADDED],+ measured_decode_peak_pixels = split["pixels"],+ measured_decode_peak_quadratic = split["quadratic"],+ decode_lend_text_encoder_mib = lend,+ )if dit_room <= 0 and te_room <= 0:return planif whole_dit or wide_dit:@@ -3696,7 +3799,18 @@ def refine_plan_from_loaded_weights(else f"{int(dit_room)} MiB of the {dit} MiB transformer")+ f" and {int(te_room)} MiB of the {encoders} MiB "- f"encoders stay resident (measured peak {headroom} MiB); the rest streams"+ f"encoders stay resident (measured peak {headroom} MiB"+ + (+ (+ f"; {phase_headroom} MiB without the streaming window, every weight resident"+ if phase_headroom == resident_headroom+ else f"; {phase_headroom} MiB before the VAE decode"+ )+ + f", the decode may stream {lend} MiB of them again"+ if lend > 0+ else ""+ )+ + "); the rest streams"+ (f" (whole transformer within free memory less a {_resident_dit_slack_mib(memory)} MiB slack;"f" {encode_room} MiB of it while the encoders run)"@@ -4079,6 +4193,96 @@ def release_resident_groups(return restore if released else None+DECODE_LEND_ATTR = "_unsloth_decode_lend"+# Kept free beside the decode's learned peak x margin when deciding whether lent encoder groups must stream again.+_DECODE_SLACK_MIB = 256+++def decode_lend_from_plan(plan: Any) -> Optional[tuple[int, int, int, bool]]:+ """``(lent MiB, decode peak MiB, bucket pixels, quadratic)`` when ``plan`` lent encoder residency to the decode."""+ estimates = getattr(plan, "estimates", None) or {}+ lend = estimates.get("decode_lend_text_encoder_mib")+ if not lend or not getattr(plan, "resident_text_encoder_mib", None):+ return None+ return (+ int(lend),+ int(estimates.get("measured_decode_peak_mib") or 0),+ int(+ estimates.get("measured_decode_peak_pixels")+ or DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT+ ),+ bool(estimates.get("measured_decode_peak_quadratic")),+ )+++def decode_need_mib(+ lend: Any,+ *,+ width: Optional[int],+ height: Optional[int],+ batch_size: int = 1,+) -> int:+ """MiB the decode of a ``width x height x batch`` request needs above the allocation at its start (margin+ included), scaled from the learned decode peak the way the store scales (``diffusion_measured_peaks``)."""+ from .diffusion_measured_peaks import resolution_bucket++ _, decode_mib, covered, quadratic = lend+ w = max(64, int(width or DEFAULT_IMAGE_WIDTH))+ h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))+ batch = max(1, int(batch_size or 1))+ covered = max(1, int(covered))+ scale = 1.0+ if quadratic:+ if w * h > covered:+ scale = (resolution_bucket(w, h, 1) / float(covered)) ** 2+ scale *= batch+ elif w * h * batch > covered:+ scale = resolution_bucket(w, h, batch) / float(covered)+ return int(math.ceil(float(decode_mib) * scale * _MEASURED_PEAK_MARGIN))+++def release_for_decode(+ pipe: Any,+ *,+ width: Optional[int],+ height: Optional[int],+ batch_size: int = 1,+ device: Any = None,+ logger: Any = None,+) -> Optional[Callable[[], None]]:+ """If free memory (driver + the allocator cache outside CUDA graph pools) is short of the learned decode peak x margin, stream the shortfall of+ lent groups again (text encoders first) and return the restore; else None. Mirrors ComfyUI's+ ``load_models_gpu(memory_required=...)``. The decode is untouched, so output is bit-identical."""+ lend = getattr(pipe, DECODE_LEND_ATTR, None)+ if not lend:+ return None+ try:+ import torch++ dev = torch.device(device if device is not None else "cuda")+ if dev.type != "cuda":+ return None+ if dev.index is None:+ dev = torch.device("cuda", torch.cuda.current_device())+ need = decode_need_mib(lend, width = width, height = height, batch_size = batch_size)+ free = int(torch.cuda.mem_get_info(dev)[0])+ # only the cache an ordinary decode allocation can reuse: a CUDA graph pool's free blocks are not+ cached = max(+ 0,+ int(torch.cuda.memory_reserved(dev))+ - int(torch.cuda.memory_allocated(dev))+ - _graph_held_cache_bytes(dev.index),+ )+ short = need + _DECODE_SLACK_MIB - ((free + cached) >> 20)+ except Exception as exc: # noqa: BLE001 - unreadable: the decode's OOM retry still frees the card+ if logger is not None:+ logger.debug("diffusion.memory: decode residency check skipped (%s)", exc)+ return None+ if short <= 0:+ return None+ return release_resident_groups(pipe, short, logger, reason = "the VAE decode", device = dev)++def resident_group_mib(pipe: Any) -> int:"""MiB an oversized request could release: every resident offload group of the pipeline's components."""total = 0@@ -4382,8 +4586,10 @@ def measured_request_extra_mib(condition_pixels: int = 0,cfg: Optional[bool] = None,unpriced_references: bool = False,+ workflow: Optional[str] = None,) -> int:- """MiB a request needs beyond what the measured placement reserved (0 when it fits or none was made). ``cfg``:+ """MiB a request needs beyond what the measured placement reserved (0 when it fits or none was made). ``workflow``:+ a workflow other than text-to-image also reads the peak recorded under its own key (``with_workflow``). ``cfg``:whether the request runs classifier-free guidance (None: unknown, treated as running it). ``unpriced_references``:the request carries reference images whose latents join the denoiser's sequence but are not in``condition_pixels`` (FLUX.2 Klein's primary reference): a placement made from a dense default / max tier seed,@@ -4467,12 +4673,32 @@ def measured_request_extra_mib(need = max(need, stored)if need is None and isinstance(identity, tuple) and len(identity) == 3:need = measured_image_runtime_mib(family, speed_mode, quant = identity[1], **sized)- if need is None:- return 0cond = max(0, int(condition_pixels or 0)) * max(1, int(batch_size or 1))- need += int(+ allowance = int(8192 * cond / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT) * _MEASURED_PEAK_MARGIN)+ if need is not None:+ need += allowance+ if isinstance(identity, tuple) and len(identity) == 3:+ keyed = with_workflow(identity[1], workflow)+ if keyed != identity[1]:+ own = None+ if (+ cfg is False+ and getattr(pipe, "_unsloth_peak_expects_cfg", False)+ and cfg_off_peaks_enabled()+ ):+ # a guidance-off render on a CFG-capable family records ``|cfg:off`` before the workflow part+ off = with_workflow(f"{without_cfg_off(identity[1])}|{CFG_OFF_PART}", workflow)+ own = measured_image_runtime_mib(family, speed_mode, quant = off, **sized)+ if own is None:+ own = measured_image_runtime_mib(family, speed_mode, quant = keyed, **sized)+ if own is not None:+ # the learned workflow peak is the other complete estimate: it was measured with the condition images+ # resident, so the allowance is not charged on top of it+ need = own if need is None else max(need, own)+ if need is None:+ return 0return max(0, int(need) - int(headroom))diff --git a/studio/backend/tests/test_diffusion_phase_peaks.py b/studio/backend/tests/test_diffusion_phase_peaks.pynew file mode 100644index 00000000000..cfedc564ee5--- /dev/null+++ b/studio/backend/tests/test_diffusion_phase_peaks.py@@ -0,0 +1,657 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Learned peaks per render phase (``diffusion_measured_peaks``) and the encoder residency the planner lends to the+VAE decode (``diffusion_memory.refine_plan_from_loaded_weights`` / ``release_for_decode``), plus the per-workflow key.++The Qwen-Image-2.1 case uses what an auto load logged on an RTX 3090: int8 DiT 6922 MiB resident, 8917 MiB of text+encoders, a 21013 MiB safe budget. The first load (empty store) placed from the 2304 MiB seed and kept every encoder+resident; after one 1024x1024 render the store held ~5.9 GB and every later load kept only part of the encoders+resident, so every new prompt streamed the rest. The phases are the ones a 24 GiB-capped B200 render recorded: 1864 MiB+real peak (the VAE decode; encode + steps ~900) plus a 4187 MiB streamed-window pad (the encoders' largest leaf and the+transformer's largest block, times the prefetch depth + 1), 6051 MiB on the whole-render scale. CPU-only.+"""++from __future__ import annotations++import json+import sys+import types++import pytest++import core.inference.diffusion_measured_peaks as mp+import core.inference.diffusion_memory as dm++QWEN_LOADED = {+ "transformer": (6922, "dit"),+ "text_encoder": (8917, "text_encoder"),+ "vae": (254, "other"),+}+KEY = dict(family = "qwen-image-2.1", quant = "torchao:Int8Tensor", speed_mode = "default")+++def _memory(budget_mib: int, total_mib: int) -> "dm.DeviceMemory":+ reserve = dm._reserve_mib("discrete_vram", total_mib)+ return dm.DeviceMemory("cuda", "cuda", "discrete_vram", budget_mib + reserve, total_mib)+++def _qwen_flat_plan(budget_mib: int = 21013, total_mib: int = 24576):+ plan = dm.plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = _memory(budget_mib, total_mib),+ runtime_headroom_mib = dm.estimate_image_runtime_mib(+ width = None, height = None, family = "qwen-image-2.1"+ ),+ requested_mode = None,+ model_dense_mib = 6922 + 8917 + 254,+ companion_dense_mib = 8917 + 254,+ text_encoder_dense_mib = 8917,+ )+ if plan.offload_policy == dm.OFFLOAD_MODEL:+ plan = dm.torchao_streaming_plan(plan)+ return plan+++def store(tmp_path, monkeypatch):+ path = tmp_path / "peaks.json"+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(path))+ for env in (+ mp.LEARNED_PEAKS_ENV,+ mp.PHASE_PEAKS_ENV,+ dm.MEASURED_ACTIVATION_ENV,+ dm.PARTIAL_RESIDENT_ENV,+ ):+ monkeypatch.delenv(env, raising = False)+ mp._cache.update(path = None, mtime = None, data = None)+ monkeypatch.setattr(mp, "current_device_name", lambda: "NVIDIA GeForce RTX 3090")+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA GeForce RTX 3090")+ return path+++def qwen_pipe(monkeypatch, store):+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(QWEN_LOADED))+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ monkeypatch.setattr(dm, "denoiser_quant_label", lambda pipe: "torchao:Int8Tensor")+ return types.SimpleNamespace(_unsloth_phase_seeds = True)+++def _refine(pipe, plan):+ return dm.refine_plan_from_loaded_weights(+ pipe, plan, family = "qwen-image-2.1", speed_mode = "default"+ )+++PAD = 4187+++def _phases(+ pre,+ dec,+ pad = PAD,+):+ """What ``diffusion._record_peak_probe`` stores for real pre-decode / decode peaks ``pre`` / ``dec``."""+ return {+ mp.PHASE_PRE_DECODE: pre + pad,+ mp.PHASE_DECODE: dec + pad,+ mp.PHASE_DECODE_UNPADDED: dec,+ mp.PHASE_UNPADDED: max(pre, dec),+ }+++B200 = dict(peak = 1864 + PAD, phases = _phases(900, 1864))+++def _record(+ peak,+ phases = None,+ **extra,+):+ kw = dict(KEY, width = 1024, height = 1024)+ kw.update(extra)+ return mp.record(**kw, peak_mib = peak, phases = phases)+++# ---------------------------------------------------------------------------------------------------- planner+++def test_decode_phase_peak_keeps_the_encoders_resident_on_24gb(qwen_pipe):+ plan = _qwen_flat_plan()+ assert plan.offload_policy == dm.OFFLOAD_GROUP and not plan.stream_transformer+ assert plan.stream_text_encoders+ first = _refine(qwen_pipe, plan)+ assert first.resident_text_encoder_mib == 8917 # the seeded first load+ _record(**B200)+ later = _refine(qwen_pipe, plan)+ assert (+ later.estimates["measured_runtime_headroom_mib"] == 7168+ ) # the whole-render reserve is unchanged+ assert later.resident_text_encoder_mib == 8917 # main: 4621+ assert later.estimates["measured_phase_headroom_mib"] == 2304 # max(seed 1849, 1864) x 1.15+ assert later.estimates["measured_phase_all_resident"]+ assert later.estimates["decode_lend_text_encoder_mib"] == 8917 - 4621+ assert "every weight resident" in later.reasons[-1]+ assert dm.decode_lend_from_plan(later) == (4296, 1864, 1024 * 1024, False)+++def test_whole_render_peaks_alone_place_as_before(qwen_pipe):+ """A store written before phases existed (or a render that recorded none) keeps today's placement."""+ plan = _qwen_flat_plan()+ _record(B200["peak"])+ later = _refine(qwen_pipe, plan)+ assert later.resident_text_encoder_mib == 4621+ assert "decode_lend_text_encoder_mib" not in later.estimates+ assert dm.decode_lend_from_plan(later) is None+++def test_a_peak_no_phase_recorded_render_explains_keeps_the_whole_render_reserve(qwen_pipe):+ plan = _qwen_flat_plan()+ _record(6800) # an older render, phase unknown, above what the phase-recorded one saw+ _record(**B200)+ assert _refine(qwen_pipe, plan).resident_text_encoder_mib < 8917+ assert dm.learned_phase_split(KEY["family"], KEY["quant"], KEY["speed_mode"]) is None+++def test_encoders_that_cannot_all_stay_resident_size_against_the_pre_decode_phase(qwen_pipe):+ """16 GiB: the transformer streams, so the unpadded check does not apply; the encoders size against encode ++ steps (padded), and the decode borrows the difference."""+ plan = _qwen_flat_plan(budget_mib = 13132, total_mib = 16384)+ _record(B200["peak"])+ before = _refine(qwen_pipe, plan)+ _record(**B200)+ after = _refine(qwen_pipe, plan)+ assert not before.resident_text_encoder_mib+ assert after.resident_text_encoder_mib == 7168 - 5888+ assert after.resident_transformer_mib == before.resident_transformer_mib+ assert not after.estimates["measured_phase_all_resident"]+ assert after.estimates["measured_phase_headroom_mib"] == 5888+++def test_pre_decode_phase_at_the_whole_render_peak_lends_nothing(qwen_pipe):+ plan = _qwen_flat_plan(budget_mib = 13132, total_mib = 16384)+ _record(B200["peak"], _phases(1864, 1864))+ assert not _refine(qwen_pipe, plan).resident_text_encoder_mib+++def test_phase_kill_switch_restores_the_whole_render_placement(qwen_pipe, monkeypatch):+ plan = _qwen_flat_plan()+ _record(**B200)+ monkeypatch.setenv(mp.PHASE_PEAKS_ENV, "0")+ assert _refine(qwen_pipe, plan).resident_text_encoder_mib == 4621++[email protected]("budget,total", [(9300, 12288), (13132, 16384), (21013, 24576)])+def test_12_16_and_24gb_never_keep_less_resident(qwen_pipe, budget, total):+ plan = _qwen_flat_plan(budget_mib = budget, total_mib = total)+ _record(B200["peak"])+ before = _refine(qwen_pipe, plan)+ _record(**B200)+ after = _refine(qwen_pipe, plan)+ assert (after.resident_text_encoder_mib or 0) >= (before.resident_text_encoder_mib or 0)+ assert after.resident_transformer_mib == before.resident_transformer_mib+ assert after.offload_policy == before.offload_policy+ assert after.stream_transformer == before.stream_transformer+++# ---------------------------------------------------------------------------------------------------- store+++def test_phases_are_stored_beside_the_whole_render_peak(store):+ assert _record(5900, {mp.PHASE_PRE_DECODE: 2000, mp.PHASE_DECODE: 5900})+ assert mp.lookup(**KEY, phase = mp.PHASE_UNPADDED) is None # not given+ assert mp.lookup(**KEY) == 5900+ assert mp.lookup(**KEY, phase = mp.PHASE_PRE_DECODE) == 2000+ assert mp.lookup(**KEY, phase = mp.PHASE_DECODE) == 5900+ # a lower whole-render peak still raises a phase (max wins per phase), the whole-render value stays+ assert not _record(5000, {mp.PHASE_PRE_DECODE: 2400, mp.PHASE_DECODE: 5000})+ assert mp.lookup(**KEY) == 5900+ assert mp.lookup(**KEY, phase = mp.PHASE_PRE_DECODE) == 2400+ entry = json.loads(store.read_text(encoding = "utf-8"))["entries"]+ (only,) = entry.values()+ assert only["phases"][mp.PHASE_DECODE] == {str(1024 * 1024): 5900}+ # a phase never exceeds the render's own peak, unknown names are dropped+ _record(100, {mp.PHASE_PRE_DECODE: 9000, "bogus": 1}, width = 512, height = 512)+ assert mp.lookup(**KEY, width = 512, height = 512, phase = mp.PHASE_PRE_DECODE) == 100+++def test_store_without_phases_answers_none_per_phase(store):+ _record(5900)+ assert mp.lookup(**KEY) == 5900+ assert mp.lookup(**KEY, phase = mp.PHASE_DECODE) is None+ assert dm.learned_phase_split(KEY["family"], KEY["quant"], KEY["speed_mode"]) is None+ mp._cache.update(path = None, mtime = None, data = None)+ store.unlink()+ _record(**B200)+ split = dm.learned_phase_split(KEY["family"], KEY["quant"], KEY["speed_mode"])+ assert split[mp.PHASE_DECODE] == 6051 and split[mp.PHASE_UNPADDED] == 1864+++class _FakeCudaTorch:+ def __init__(self):+ self.allocated = 0+ self.peak = 0+ torch_self = self++ class _Dev:+ def __init__(self, *args):+ self.type = "cuda"+ self.index = 0++ def __str__(self):+ return "cuda:0"++ class cuda: # noqa: N801+ is_available = staticmethod(lambda: True)+ current_device = staticmethod(lambda: 0)+ synchronize = staticmethod(lambda dev = None: None)+ memory_allocated = staticmethod(lambda dev = None: torch_self.allocated)+ max_memory_allocated = staticmethod(lambda dev = None: torch_self.peak)++ @staticmethod+ def reset_peak_memory_stats(dev = None):+ torch_self.peak = torch_self.allocated++ self.device = _Dev+ self.cuda = cuda++ def alloc(self, mib):+ self.allocated += mib << 20+ self.peak = max(self.peak, self.allocated)++ def free(self, mib):+ self.allocated -= mib << 20+++def test_probe_splits_the_render_at_the_decode_without_resetting(monkeypatch):+ fake = _FakeCudaTorch()+ monkeypatch.setitem(sys.modules, "torch", fake)+ fake.alloc(15000) # resident weights before the render+ probe = mp.PeakProbe("cuda:0")+ fake.alloc(2000) # encode+ fake.free(2000)+ fake.alloc(1500) # denoise+ fake.free(1500)+ probe.mark_decode()+ peak_before = fake.peak+ fake.alloc(5900) # untiled decode+ fake.free(5900)+ assert fake.peak >= peak_before # nothing reset the shared counters+ assert probe.finish() == 5900+ assert probe.phase_peaks == {mp.PHASE_PRE_DECODE: 2000, mp.PHASE_DECODE: 5900}+++def test_probe_with_two_decodes_reports_no_split(monkeypatch):+ fake = _FakeCudaTorch()+ monkeypatch.setitem(sys.modules, "torch", fake)+ probe = mp.PeakProbe("cuda:0")+ probe.mark_decode()+ fake.alloc(800)+ fake.free(800)+ probe.mark_decode() # a split batch: chunk 2 encodes after chunk 1 decoded+ fake.alloc(3000)+ assert probe.finish() == 3000+ assert probe.phase_peaks == {}+++def test_render_records_phases_and_keys_non_txt2img_workflows(store, monkeypatch):+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)+ # recorded without a device target: the SDPA verdict counts as known+ monkeypatch.setattr(diffusion, "_attention_scaling_known", lambda target, engaged = None: True)+ pipe = types.SimpleNamespace(_unsloth_peak_identity = (KEY["family"], KEY["quant"], "default"))+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default")++ class _Probe:+ device = "cuda:0"+ failed_captures = 0+ window_pad = 64++ def __init__(self, peak, phases):+ self._peak = peak+ self.phase_peaks = phases++ def finish(self):+ return self._peak++ diffusion._record_peak_probe(+ state,+ _Probe(5900, {mp.PHASE_PRE_DECODE: 2000, mp.PHASE_DECODE: 5900}),+ width = 1024,+ height = 1024,+ batch_size = 1,+ workflow = "txt2img",+ )+ assert mp.lookup(**KEY) == 5964+ assert mp.lookup(**KEY, phase = mp.PHASE_PRE_DECODE) == 2064+ assert mp.lookup(**KEY, phase = mp.PHASE_DECODE) == 5964+ # nothing streams during the decode, and a load keeping every weight resident holds no window: no pad+ assert mp.lookup(**KEY, phase = mp.PHASE_DECODE_UNPADDED) == 5900+ assert mp.lookup(**KEY, phase = mp.PHASE_UNPADDED) == 5900+ diffusion._record_peak_probe(+ state, _Probe(7800, {}), width = 1024, height = 1024, batch_size = 1, workflow = "edit"+ )+ assert mp.lookup(**KEY) == 5964 # an edit no longer raises the text-to-image key+ assert mp.lookup(**dict(KEY, quant = KEY["quant"] + "|wf:edit")) == 7864+++def test_with_workflow_keeps_txt2img_on_the_load_key():+ assert dm.with_workflow("a|b", None) == "a|b"+ assert dm.with_workflow("a|b", "txt2img") == "a|b"+ assert dm.with_workflow("a|b", "edit") == "a|b|wf:edit"+++def test_edit_request_is_sized_from_its_own_key(qwen_pipe):+ plan = _qwen_flat_plan()+ _record(**B200)+ later = _refine(qwen_pipe, plan)+ qwen_pipe._unsloth_measured_reserve = (+ later.estimates["measured_runtime_headroom_mib"],+ "qwen-image-2.1",+ "default",+ None,+ None,+ later.estimates["measured_learned_peak_mib"],+ later.estimates["measured_learned_peak_pixels"],+ False,+ True,+ False,+ )+ kw = dict(width = 1024, height = 1024, batch_size = 1, cfg = False)+ assert dm.measured_request_extra_mib(qwen_pipe, **kw, workflow = "txt2img") == 0+ assert (+ dm.measured_request_extra_mib(qwen_pipe, **kw, workflow = "edit") == 0+ ) # nothing learned for edits yet+ _record(7800, quant = KEY["quant"] + "|wf:edit")+ # 7800 x 1.15 rounded to 256 = 9216, past the 7168 the load reserved+ assert dm.measured_request_extra_mib(qwen_pipe, **kw, workflow = "edit") == 9216 - 7168+ assert dm.measured_request_extra_mib(qwen_pipe, **kw, workflow = "txt2img") == 0+++# ---------------------------------------------------------------------------------------------------- decode guard+++class _GuardTorch:+ def __init__(+ self,+ free_mib,+ reserved_mib = 0,+ allocated_mib = 0,+ ):+ outer = self+ self.free = free_mib+ self.reserved = reserved_mib+ self.allocated = allocated_mib++ class device: # noqa: N801+ def __init__(+ self,+ spec,+ index = None,+ ):+ spec = str(spec)+ self.type = spec.split(":")[0]+ self.index = (+ index+ if index is not None+ else (int(spec.split(":")[1]) if ":" in spec else None)+ )++ class cuda: # noqa: N801+ current_device = staticmethod(lambda: 0)+ mem_get_info = staticmethod(lambda dev = None: (outer.free << 20, 24576 << 20))+ memory_reserved = staticmethod(lambda dev = None: outer.reserved << 20)+ memory_allocated = staticmethod(lambda dev = None: outer.allocated << 20)++ self.device = device+ self.cuda = cuda+++def test_decode_streams_lent_encoders_only_when_free_memory_is_short(monkeypatch):+ calls = []+ monkeypatch.setattr(+ dm,+ "release_resident_groups",+ lambda pipe, need, logger = None, reason = "", device = None, **kw: calls.append(need)+ or (lambda: None),+ )+ pipe = types.SimpleNamespace()+ pipe._unsloth_decode_lend = (4040, 5900, 1024 * 1024, False)+ need = dm.decode_need_mib(pipe._unsloth_decode_lend, width = 1024, height = 1024)+ assert need == 6785 # 5900 x 1.15+ monkeypatch.setitem(sys.modules, "torch", _GuardTorch(free_mib = 7500))+ assert dm.release_for_decode(pipe, width = 1024, height = 1024, device = "cuda:0") is None+ assert calls == []+ # cached allocator blocks count as free+ monkeypatch.setitem(+ sys.modules, "torch", _GuardTorch(free_mib = 6000, reserved_mib = 3000, allocated_mib = 1000)+ )+ assert dm.release_for_decode(pipe, width = 1024, height = 1024, device = "cuda:0") is None+ monkeypatch.setitem(sys.modules, "torch", _GuardTorch(free_mib = 5000))+ assert dm.release_for_decode(pipe, width = 1024, height = 1024, device = "cuda:0") is not None+ assert calls == [6785 + dm._DECODE_SLACK_MIB - 5000]+ # a larger canvas scales the learned decode the way the store does+ assert dm.decode_need_mib(pipe._unsloth_decode_lend, width = 2048, height = 2048) == 6785 * 4+ # no lend, no check+ assert dm.release_for_decode(types.SimpleNamespace(), width = 1024, height = 1024) is None+++def test_edit_only_family_records_under_the_load_key(store, monkeypatch):+ # an edit-only family runs nothing but edits: its renders keep teaching the key its next load places from+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)+ monkeypatch.setattr(diffusion, "_attention_scaling_known", lambda target, engaged = None: True)+ pipe = types.SimpleNamespace(_unsloth_peak_identity = (KEY["family"], KEY["quant"], "default"))+ family = types.SimpleNamespace(edit = True)+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default", family = family)++ class _Probe:+ device = "cuda:0"+ failed_captures = 0+ window_pad = 0+ phase_peaks = {}++ def finish(self):+ return 7800++ diffusion._record_peak_probe(+ state, _Probe(), width = 1024, height = 1024, batch_size = 1, workflow = "edit"+ )+ assert mp.lookup(**KEY) == 7800+ assert mp.lookup(**dict(KEY, quant = KEY["quant"] + "|wf:edit")) is None+ assert diffusion._peak_workflow(family, "edit") is None+ assert diffusion._peak_workflow(types.SimpleNamespace(unified_edit = True), "edit") == "edit"+++def test_cfg_off_edit_request_reads_its_cfg_off_workflow_key(qwen_pipe):+ plan = _qwen_flat_plan()+ _record(**B200)+ later = _refine(qwen_pipe, plan)+ qwen_pipe._unsloth_peak_expects_cfg = True+ qwen_pipe._unsloth_peak_identity = (KEY["family"], KEY["quant"], "default")+ qwen_pipe._unsloth_measured_reserve = (+ later.estimates["measured_runtime_headroom_mib"],+ "qwen-image-2.1",+ "default",+ None,+ None,+ later.estimates["measured_learned_peak_mib"],+ later.estimates["measured_learned_peak_pixels"],+ False,+ True,+ False,+ )+ kw = dict(width = 1024, height = 1024, batch_size = 1, workflow = "edit")+ # what the recorder writes for a guidance-off edit on a CFG-capable family+ _record(7800, quant = KEY["quant"] + "|cfg:off|wf:edit")+ assert dm.measured_request_extra_mib(qwen_pipe, **kw, cfg = False) == 9216 - 7168+ # a CFG request is not covered by the guidance-off sample+ assert dm.measured_request_extra_mib(qwen_pipe, **kw, cfg = True) == 0+++def _seeded_edit_pipe(pipe):+ """A phase-seeded load that reserved 7168 MiB from a 6051 MiB learned whole-render peak."""+ pipe._unsloth_peak_expects_cfg = True+ pipe._unsloth_peak_identity = (KEY["family"], KEY["quant"], "default")+ pipe._unsloth_measured_reserve = (+ 7168,+ "qwen-image-2.1",+ "default",+ None,+ None,+ 6051,+ 1024 * 1024,+ False,+ True,+ False,+ )+ return pipe+++def test_edit_workflow_peak_does_not_charge_the_condition_images_twice(qwen_pipe):+ pipe = _seeded_edit_pipe(qwen_pipe)+ kw = dict(+ width = 1024,+ height = 1024,+ batch_size = 1,+ cfg = False,+ workflow = "edit",+ condition_pixels = int(1024 * 1024 * 0.32),+ )+ # 7168 seed reserve + 3014 condition allowance - 7168 headroom: what main charges, store or no store+ assert dm.measured_request_extra_mib(pipe, **kw) == 3014+ # a learned edit peak (7800 x 1.15 -> 9216) already holds the condition images; it is an alternative to the seed+ # estimate plus allowance (10182), not a base the allowance is added on top of (that gave 5062)+ _record(7800, quant = KEY["quant"] + "|cfg:off|wf:edit")+ assert dm.measured_request_extra_mib(pipe, **kw) == 3014+++def test_edit_workflow_peak_above_seed_plus_allowance_is_followed(qwen_pipe):+ pipe = _seeded_edit_pipe(qwen_pipe)+ kw = dict(width = 1024, height = 1024, batch_size = 1, cfg = False, workflow = "edit")+ cond = int(1024 * 1024 * 0.32)+ # 12000 x 1.15 rounded to 256 = 13824, past 7168 + 3014+ _record(12000, quant = KEY["quant"] + "|cfg:off|wf:edit")+ assert dm.measured_request_extra_mib(pipe, **kw, condition_pixels = cond) == 13824 - 7168+ assert dm.measured_request_extra_mib(pipe, **kw, condition_pixels = 0) == 13824 - 7168+++def test_decode_headroom_does_not_credit_cuda_graph_pool_cache(monkeypatch):+ # a graph pool's free blocks sit inside reserved - allocated yet an ordinary VAE allocation cannot reuse them+ import core.inference.diffusion_cuda_graph as cg++ calls = []+ monkeypatch.setattr(+ dm,+ "release_resident_groups",+ lambda pipe, need, logger = None, reason = "", device = None, **kw: calls.append(need)+ or (lambda: None),+ )+ pipe = types.SimpleNamespace(_unsloth_decode_lend = (4040, 5900, 1024 * 1024, False))+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 2000 << 20)+ monkeypatch.setitem(+ sys.modules, "torch", _GuardTorch(free_mib = 6000, reserved_mib = 3000, allocated_mib = 1000)+ )+ assert dm.release_for_decode(pipe, width = 1024, height = 1024, device = "cuda:0") is not None+ assert calls == [6785 + dm._DECODE_SLACK_MIB - 6000]+++def test_render_whose_decode_released_lent_groups_records_nothing(store, monkeypatch):+ # the decode ran net of the streamed groups' bytes: its peak (and whole-render peak) would under-size the bucket+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)+ monkeypatch.setattr(diffusion, "_attention_scaling_known", lambda target, engaged = None: True)+ pipe = types.SimpleNamespace(_unsloth_peak_identity = (KEY["family"], KEY["quant"], "default"))+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default")+ finished = []++ class _Probe:+ device = "cuda:0"+ failed_captures = 0+ window_pad = 0+ decode_released = True+ phase_peaks = {mp.PHASE_PRE_DECODE: 2000, mp.PHASE_DECODE: 2000}++ def finish(self):+ finished.append(1)+ return 2000++ diffusion._record_peak_probe(+ state, _Probe(), width = 1024, height = 1024, batch_size = 1, workflow = "txt2img"+ )+ assert finished == [1] # the probe's window still closes+ assert mp.lookup(**KEY) is None+ assert mp.lookup(**KEY, phase = mp.PHASE_DECODE_UNPADDED) is None+++# ---------------------------------------------------------------------------------------------------- v1 store+++def _write_v1(store, entries):+ store.write_text(+ json.dumps({"version": 1, "runtime": mp.runtime_stamp(), "entries": entries}),+ encoding = "utf-8",+ )+ mp._cache.update(path = None, mtime = None, data = None)+++def test_pre_workflow_store_peak_no_longer_sizes_txt2img(qwen_pipe, store):+ """Before workflow keys every workflow recorded max-wins under the load key: an old edit peak there must not+ reserve for text-to-image (nor block the phase split) once a txt2img render records its own peak."""+ plan = _qwen_flat_plan()+ fresh = _refine(qwen_pipe, plan)+ base = mp.model_key(+ KEY["family"], KEY["quant"], KEY["speed_mode"], 1, "NVIDIA GeForce RTX 3090"+ )+ bucket = str(1024 * 1024)+ edit = mp.model_key(+ KEY["family"], KEY["quant"] + "|wf:edit", KEY["speed_mode"], 1, "NVIDIA GeForce RTX 3090"+ )+ other = mp.model_key(+ "flux.1-dev", KEY["quant"], KEY["speed_mode"], 1, "NVIDIA GeForce RTX 3090"+ )+ phased = {name: {bucket: value} for name, value in _phases(900, 1864).items()}+ _write_v1(+ store,+ {+ base: {"peaks": {bucket: 13000}}, # an edit render's peak, pre-PR+ edit: {"peaks": {bucket: 9000}},+ # the per-workflow format: the inflated whole-render value falls back to its phases+ other: {"peaks": {bucket: 13000}, "phases": phased},+ },+ )+ assert mp.lookup(**KEY) is None+ assert mp.lookup(**dict(KEY, quant = KEY["quant"] + "|wf:edit")) == 9000+ assert mp.lookup(family = "flux.1-dev", quant = KEY["quant"], speed_mode = "default") == 6051+ assert (+ mp.lookup(+ family = "flux.1-dev",+ quant = KEY["quant"],+ speed_mode = "default",+ phase = mp.PHASE_UNPADDED,+ )+ == 1864+ )+ legacy = _refine(qwen_pipe, plan)+ assert legacy.estimates.get("measured_learned_peak_mib") is None+ assert (+ legacy.estimates["measured_runtime_headroom_mib"]+ == fresh.estimates["measured_runtime_headroom_mib"]+ )+ assert legacy.resident_text_encoder_mib == fresh.resident_text_encoder_mib == 8917+ # a fresh txt2img sample is learned, the split works again, and the store is rewritten as version 2+ _record(**B200)+ assert mp.lookup(**KEY) == B200["peak"]+ assert dm.learned_phase_split(KEY["family"], KEY["quant"], KEY["speed_mode"]) is not None+ later = _refine(qwen_pipe, plan)+ assert later.resident_text_encoder_mib == 8917+ assert later.estimates["decode_lend_text_encoder_mib"] == 8917 - 4621+ data = json.loads(store.read_text(encoding = "utf-8"))+ assert data["version"] == mp._VERSION and set(data["entries"]) == {edit, other, base}