Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: learned activation peaks, OOM retry in group offload setup, pinned GGUF host copies, tied-weight groups (#13724)

Daniel Han1f74450757b6
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 5cf395b322f..da47d3fe037 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -160,6 +160,12 @@ from .diffusion_memory import (
refine_balanced_plan_for_components,
refine_memory_plan_for_components,
refine_plan_from_loaded_weights,
+ learned_peak_label,
+ live_vae_saver_tag,
+ planned_vae_saver_tag,
+ stream_window_pad_mib,
+ with_label_part,
+ applied_plan,
release_resident_groups,
resident_group_mib,
hook_resident_denoiser,
@@ -1962,6 +1968,288 @@ def _quadratic_attention(
return True
+def _attention_scaling_known(target: Any, engaged_backend: Optional[str] = None) -> bool:
+ """Whether ``_quadratic_attention``'s answer is evidence: a non-native backend engaged, or the SDPA probe saw math
+ alone or a sub-quadratic kernel run. An unanswered probe (or no device target) is unknown."""
+ if engaged_backend is not None and str(engaged_backend) != "native":
+ return True
+ if target is None:
+ return False
+ try:
+ return bool(sdpa_math_only(target) or sdpa_subquadratic_confirmed(target))
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _start_peak_probe(state: Any) -> Any:
+ """Arm the learned-peak measurement for one render (CUDA loads that carry a peak identity), else None."""
+ pipe = getattr(state, "pipe", None)
+ identity = getattr(pipe, "_unsloth_peak_identity", None)
+ if identity is None:
+ return None
+ try:
+ from .diffusion_measured_peaks import PeakProbe, enabled
+
+ index = _peak_device_index(state)
+ graphs = cuda_graph.live_graph_count(index)
+ generation = cuda_graph.capture_generation(index)
+ accounted = getattr(pipe, "_unsloth_peak_graphs", None)
+ if accounted is None or graphs < accounted:
+ # the load's own graphs (every render starts above them), or fewer after a reset freed some
+ first = accounted is None
+ pipe._unsloth_peak_graphs = accounted = graphs
+ if first or graphs == 0:
+ # nothing alive on the card: no unrecorded capture can remain. A drop to a non-zero count may be
+ # another pipeline's reset, so a capture this one made unrecorded keeps the generation dirty
+ pipe._unsloth_peak_graph_gen = generation
+ if getattr(pipe, "_unsloth_peak_graph_gen", None) is None:
+ pipe._unsloth_peak_graph_gen = generation
+ if graphs > accounted or generation > pipe._unsloth_peak_graph_gen:
+ # an unrecorded render (cache-served, refused, failed) left graphs allocated, or replaced one in a full
+ # cache (same count): a later peak above that start omits memory a fresh process allocates on its first
+ # render
+ return None
+ if _live_peak_identity(state) != tuple(identity):
+ # this render runs a configuration the key does not name (a speed tier or step cache that toggled, a VAE
+ # saver the guard engaged for it, another attention backend): its peak belongs to no key
+ return None
+ if not enabled():
+ return None
+ window_pad = stream_window_pad_mib(pipe)
+ if window_pad is None:
+ return (
+ None # the streamed baseline is unknown: the peak cannot be put on the common scale
+ )
+ probe = PeakProbe(getattr(state, "device", None))
+ if probe.device is None:
+ return None
+ probe.window_pad = window_pad
+ probe.failed_captures = cuda_graph.failed_capture_count()
+ probe.conditioning_hits = _conditioning_cache_hits(pipe)
+ return probe
+ except Exception: # noqa: BLE001 - measurement only
+ return None
+
+
+def _peak_device_index(state: Any) -> Optional[int]:
+ """CUDA ordinal the pipeline renders on (``state.device`` may be un-indexed: the thread's current card)."""
+ try:
+ import torch
+ dev = torch.device(getattr(state, "device", None) or "cuda")
+ return dev.index if dev.index is not None else torch.cuda.current_device()
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _live_peak_identity(state: Any) -> tuple:
+ """The load's learned-peak identity (the canonical configuration captured at load) with every part that can change
+ after the load read from the live state: speed tier, step cache, VAE savers, attention backend. A render records
+ only when this equals the identity."""
+ pipe = getattr(state, "pipe", None)
+ family, label, _ = tuple(getattr(pipe, "_unsloth_peak_identity", ("", "", None)))[:3]
+ label = with_label_part(
+ str(label), "cache", str(getattr(state, "transformer_cache", None) or "off")
+ )
+ label = with_label_part(label, "vae", live_vae_saver_tag(pipe))
+ label = with_label_part(label, "attn", getattr(state, "attention_backend", None) or "native")
+ if "|graphs:" in label:
+ label = with_label_part(label, "graphs", _live_graph_mode(state))
+ speed = getattr(state, "speed_mode", None)
+ if "compile_fallback_eager" in tuple(getattr(state, "speed_optims", ()) or ()):
+ speed = None # the compiled tier fell back to eager: an execution path no key names
+ try:
+ from .diffusion_bg_compile import eager_forced
+ if eager_forced():
+ speed = None # a background compile is still pending: this render runs eager under a compiled tier
+ except Exception: # noqa: BLE001
+ speed = None
+ # a family that can run classifier-free guidance records only renders that do (the doubled denoiser batch)
+ family_obj = getattr(state, "family", None)
+ expects_cfg = bool(getattr(family_obj, "uses_negative_prompt", False))
+ if bool(getattr(pipe, "_unsloth_peak_cfg", expects_cfg)) != expects_cfg:
+ speed = None
+ if getattr(pipe, "_unsloth_loras", None):
+ speed = None # LoRA adapters add rank-dependent activations no key names
+ return (family, label, speed)
+
+
+def _live_graph_mode(state: Any) -> str:
+ """``step`` / ``blocks`` (``_engaged_graph_mode``) while the load's CUDA graphs are armed and not dropped
+ (``live_status``), else ``off``."""
+ try:
+ _, optims = cuda_graph.live_status(
+ getattr(state, "resolved", None),
+ getattr(state, "speed_optims", ()),
+ getattr(state, "cuda_graphs", ()),
+ )
+ if "cuda_graph" not in optims:
+ return "off"
+ return _engaged_graph_mode(getattr(state, "pipe", None), {"cuda_graph": True})
+ except Exception: # noqa: BLE001
+ return "unknown"
+
+
+def _peak_te_label(te_quant: Any, outcome: Any) -> str:
+ """The text encoders' precision as the learned-peak key names it. A partial cast (some encoders quantised, others
+ left dense) is no build the key can name: ``unknown``, so it neither reads nor records a learned peak."""
+ if getattr(outcome, "partial", False):
+ return "unknown"
+ return str(te_quant or "dense")
+
+
+def _engaged_graph_mode(pipe: Any, applied: Any) -> str:
+ """The graph implementation placement armed: whole-step and per-block graphs hold different statics, slot rings
+ and pools, so their peaks never share a key."""
+ if not (applied or {}).get("cuda_graph"):
+ return "off"
+ return "blocks" if getattr(pipe, "_unsloth_cuda_graph_mode", None) == "blocks" else "step"
+
+
+def _local_weights_digest(path: Any) -> Optional[str]:
+ """Size and mtime of every weight file under a local pipeline folder (``checkpoint_fingerprint`` reads only its
+ manifests); None for a repo id or an unreadable folder."""
+ try:
+ import hashlib
+
+ root = Path(str(path)).expanduser()
+ if not root.is_dir():
+ return None
+ digest = hashlib.sha256()
+ for f in sorted(root.rglob("*")):
+ if f.is_file() and f.suffix in (
+ ".safetensors",
+ ".bin",
+ ".gguf",
+ ".pt",
+ ".pth",
+ ".ckpt",
+ ):
+ st = f.stat()
+ digest.update(f"{f.relative_to(root)}:{st.st_size}:{st.st_mtime_ns}\0".encode())
+ return digest.hexdigest()[:16]
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _peak_model_identity(
+ repo_id: Any, gguf_filename: Optional[str], kind: Optional[str], base: Any
+) -> str:
+ """The exact build for the learned-peak key: the denoiser source and its weights' revision / fingerprint, plus the
+ companion base and its revision for a GGUF / single-file load. ``unknown`` (no learned peak used or recorded)
+ when a revision cannot be read."""
+ from .diffusion_component_sizes import checkpoint_fingerprint
+
+ source = f"{repo_id}/{gguf_filename}" if gguf_filename else str(repo_id)
+ revision = checkpoint_fingerprint(repo_id, gguf_filename)
+ if not revision:
+ return "unknown"
+ if str(revision).startswith("local:"):
+ weights = _local_weights_digest(repo_id)
+ if not weights:
+ return "unknown"
+ revision = f"{revision}:{weights}"
+ identity = f"{source}@{revision}"
+ if (kind or "pipeline") != "pipeline" and base and str(base) != str(repo_id):
+ base_revision = checkpoint_fingerprint(base)
+ if not base_revision:
+ return "unknown"
+ if str(base_revision).startswith("local:"):
+ weights = _local_weights_digest(base)
+ if not weights:
+ return "unknown"
+ base_revision = f"{base_revision}:{weights}"
+ identity += f"+{base}@{base_revision}"
+ return identity
+
+
+def _conditioning_cache_hits(pipe: Any) -> int:
+ """Encodes served from the in-memory prompt cache or the persistent conditioning cache so far: a render with a
+ hit skipped the text-encoder phase, so its peak covers only part of what a new prompt needs."""
+ hits = 0
+ try:
+ cache = prompt_cache.cache_for(pipe)
+ if cache is not None:
+ hits += int(cache.stats.get("hits", 0))
+ stats = getattr(pipe, "_unsloth_cond_cache_stats", None)
+ if isinstance(stats, dict):
+ hits += int(stats.get("hits", 0))
+ except Exception: # noqa: BLE001 - measurement only
+ pass
+ return hits
+
+
+def _record_peak_probe(
+ state: Any,
+ probe: Any,
+ *,
+ width: Optional[int],
+ height: Optional[int],
+ batch_size: int,
+ target: Any = None,
+) -> None:
+ """Persist a completed render's peak above its starting allocation under the load's peak identity."""
+ if probe is None or not width or not height:
+ return
+ identity = getattr(getattr(state, "pipe", None), "_unsloth_peak_identity", None)
+ try:
+ from .diffusion_measured_peaks import device_name, record
+
+ peak = probe.finish()
+ if peak is None or not isinstance(identity, tuple) or len(identity) != 3:
+ return
+ if _live_peak_identity(state) != tuple(identity):
+ # the configuration moved during the render (a compile fallback, a toggled cache): no key names it
+ return
+ if cuda_graph.failed_capture_count() != getattr(probe, "failed_captures", 0):
+ # a capture failed (an absorbed OOM): the peak is the failed attempt's, not the render's need
+ return
+ if _conditioning_cache_hits(state.pipe) != getattr(probe, "conditioning_hits", 0):
+ # a cached prompt skipped the text encoder: the peak misses that phase, and an unseeded family places
+ # its next load from this value alone
+ return
+ engaged = getattr(state, "attention_backend", None)
+ if not _attention_scaling_known(target, engaged):
+ # the SDPA probe could not tell math from a fused kernel: a quadratic peak stored as linear would
+ # under-size every larger request
+ return
+ family, quant, speed_mode = identity
+ pipe = state.pipe
+ index = getattr(probe.device, "index", None)
+ graphs = cuda_graph.live_graph_count(index)
+ generation = cuda_graph.capture_generation(index)
+ peak += int(getattr(probe, "window_pad", 0) or 0)
+ held: list = []
+ raised = record(
+ family = family,
+ quant = quant,
+ speed_mode = speed_mode,
+ width = width,
+ height = height,
+ batch_size = batch_size,
+ peak_mib = peak,
+ device_name = device_name(probe.device),
+ quadratic = _quadratic_attention(target, engaged, state.pipe),
+ held = held,
+ )
+ if held:
+ # the store holds this sample, so it covers the graphs the render recorded; unpersisted, they stay dirty
+ pipe._unsloth_peak_graphs = graphs
+ pipe._unsloth_peak_graph_gen = generation
+ if raised:
+ logger.info(
+ "diffusion.memory: learned activation peak %d MiB recorded for %s / %s / %s at %dx%d x%d",
+ peak,
+ family,
+ quant,
+ speed_mode,
+ int(width),
+ int(height),
+ int(batch_size),
+ )
+ except Exception as exc: # noqa: BLE001 - recording must never fail a finished render
+ logger.debug("diffusion.memory: learned activation peak not recorded (%s)", exc)
+
+
def _activation_guard_batch(chunks: Sequence[Sequence[Any]]) -> int:
"""The batch size the generate-time activation guard budgets for.
@@ -8206,6 +8494,45 @@ class DiffusionBackend:
"; ".join(refined_plan.reasons),
)
plan = refined_plan
+ peak_model = _peak_model_identity(repo_id, gguf_filename, kind, base)
+ if getattr(pipe, "_unsloth_loras", None):
+ peak_model = (
+ "unknown" # adapters baked in at load: a build the key cannot name
+ )
+ try:
+ # the phase-table seed holds only where it was measured
+ pipe._unsloth_phase_seeds = _calibrated_activation(fam, target) is not None
+ except Exception: # noqa: BLE001
+ pass
+ peak_vae = planned_vae_saver_tag(pipe, plan)
+ peak_te = _peak_te_label(te_quant, te_outcome)
+ # an auto step cache toggles per request: only uncached renders (the full transformer) record
+ peak_cache = "off" if cache_may_toggle else str(cache_engaged or "off")
+ # graph statics and pools sit above a graph-free peak; a graph armed after an offloaded placement
+ # counts as on
+ peak_graphs = (
+ "on"
+ if speed_applied.get("cuda_graph")
+ or getattr(pipe, "_unsloth_cuda_graph_after_placement", False)
+ else "off"
+ )
+ try:
+ # the learned-peak key: recorded at every completed render, read by later loads
+ pipe._unsloth_peak_identity = (
+ fam.name,
+ learned_peak_label(
+ pipe,
+ attention_engaged or "native",
+ peak_model,
+ peak_vae,
+ peak_te,
+ peak_cache,
+ peak_graphs,
+ ),
+ effective_speed,
+ )
+ except Exception: # noqa: BLE001 - recording is optional
+ pass
# not under the legacy cpu_offload flag alone: that request asked for offload
if not cpu_offload or normalize_memory_mode(memory_mode) is not None:
plan = refine_plan_from_loaded_weights(
@@ -8214,6 +8541,12 @@ class DiffusionBackend:
family = fam.name,
speed_mode = effective_speed,
logger = logger,
+ attention = attention_engaged or "native",
+ model = peak_model,
+ vae = peak_vae,
+ te = peak_te,
+ cache = peak_cache,
+ graphs = peak_graphs,
)
headroom = plan.estimates.get("measured_runtime_headroom_mib")
if headroom and (
@@ -8225,6 +8558,9 @@ class DiffusionBackend:
effective_speed,
plan.estimates.get("measured_dense_transformer_mib"),
plan.estimates.get("measured_compute_bytes"),
+ plan.estimates.get("measured_learned_peak_mib"),
+ plan.estimates.get("measured_learned_peak_pixels"),
+ plan.estimates.get("measured_learned_peak_quadratic"),
)
from .diffusion_qwenimage21_vision import configure_vision_attention
@@ -8271,6 +8607,16 @@ class DiffusionBackend:
placement_device = target.torch_device,
logger = logger,
)
+ # a transient OOM during setup may have re-planned to lower residency
+ plan = applied_plan(pipe, plan)
+ identity = getattr(pipe, "_unsloth_peak_identity", None)
+ if isinstance(identity, tuple) and len(identity) == 3:
+ # a saver the plan asked for may not have engaged: key the renders by what did
+ live_vae = with_label_part(identity[1], "vae", live_vae_saver_tag(pipe))
+ if live_vae != identity[1]:
+ pipe._unsloth_peak_identity = (identity[0], live_vae, identity[2])
+ # placed from the planned saver's peak: requests size from the flat estimate instead
+ pipe._unsloth_peak_static = True
# streams the whole-resident denoiser back to the flat room while the encoders run
install_encode_release(pipe, plan, logger)
# the speed layer saw only the plan; placement may have pinned every denoiser group since
@@ -8294,6 +8640,20 @@ class DiffusionBackend:
speed_mode = effective_speed,
logger = logger,
)
+ identity = getattr(pipe, "_unsloth_peak_identity", None)
+ if (
+ isinstance(identity, tuple)
+ and len(identity) == 3
+ and "graphs:on" in str(identity[1]).split("|")
+ ):
+ # placed from the larger of both graph modes' peaks: renders record under the one that armed
+ pipe._unsloth_peak_identity = (
+ identity[0],
+ with_label_part(
+ identity[1], "graphs", _engaged_graph_mode(pipe, speed_applied)
+ ),
+ identity[2],
+ )
# Per-control provenance for status. cpu_offload=False is the unset default, so only True is
# explicit.
@@ -10354,6 +10714,21 @@ class DiffusionBackend:
target = target,
)
object.__setattr__(state, "attention_backend", attention_engaged)
+ identity = getattr(state.pipe, "_unsloth_peak_identity", None)
+ if isinstance(identity, tuple) and len(identity) == 3:
+ # later renders record under the backend that now runs
+ try:
+ # the profile engages the default tier too (state.speed_mode below)
+ state.pipe._unsloth_peak_identity = (
+ identity[0],
+ with_label_part(identity[1], "attn", attention_engaged or "native"),
+ SPEED_DEFAULT,
+ )
+ except Exception: # noqa: BLE001 - recording is optional
+ pass
+ # the reserve was placed for the load's tier: until a reload re-places it, requests size from the flat
+ # estimate
+ state.pipe._unsloth_peak_static = True
# Keep the badge in step with the state it describes: the top-level field moved, so leaving `resolved` on the
# load-time value would make the panel report an attention backend that is no longer the one running.
if isinstance(state.resolved, dict) and "attention_backend" in state.resolved:
@@ -10932,6 +11307,8 @@ class DiffusionBackend:
# cap; the OOM backoff below halves a failed chunk. Layered: one input per forward (the pipeline's
# per-prompt layer stride is only right for one).
chunks = [[job] for job in jobs] if layer_count else chunk_jobs(jobs, batch_size)
+ guard_width = guard_height = guard_target = None
+ guard_batch = 1
try:
# The dimensions the forward runs at, not the sliders: img2img / inpaint / upscale / edit take
@@ -11169,6 +11546,11 @@ class DiffusionBackend:
chunk_shapes: list[int] = []
graphs_before = fresh_compile_count()
compile_cache.note_use(state.compile_cache_ctx)
+ try:
+ state.pipe._unsloth_peak_cfg = _negative_prompt_engaged(state.family, guidance)
+ except Exception: # noqa: BLE001 - an unknown guidance mode records nothing
+ state.pipe._unsloth_peak_cfg = None
+ peak_probe = _start_peak_probe(state)
try:
previewer = _start_previewer()
pending = list(chunks)
@@ -11290,6 +11672,7 @@ class DiffusionBackend:
empty_cache()
first_half, second_half = split_chunk(chunk)
pending[:0] = [first_half, second_half]
+ peak_probe = None # the failed attempt's peak is the card's limit, not this request's
gen.total_steps += steps # one extra chunk to run
logger.warning(
"diffusion.generate: batch of %d hit OOM; retrying as %d + %d",
@@ -11331,6 +11714,14 @@ class DiffusionBackend:
raise
if previewer is not None:
previewer.finish()
+ _record_peak_probe(
+ state,
+ peak_probe,
+ width = guard_width,
+ height = guard_height,
+ batch_size = max(chunk_shapes or [1]),
+ target = guard_target,
+ )
if static_skip:
logger.debug("diffusion.step_skip: %s", static_skip_stats(state.pipe))
# Keep progress ACTIVE through the post-denoise work: the route persists the image after this returns,
diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.py
index 371961f3e23..bea93715035 100644
--- a/studio/backend/core/inference/diffusion_block_graph.py
+++ b/studio/backend/core/inference/diffusion_block_graph.py
@@ -371,6 +371,18 @@ def _meta(t: Any) -> tuple:
return (tuple(t.shape), tuple(t.stride()), t.dtype, t.device, bool(t.is_inference()))
+_LIVE_BLOCKS: "weakref.WeakSet" = weakref.WeakSet()
+
+
+def live_block_graph_count(device_index: Optional[int]) -> int:
+ """Per-block graphs alive on CUDA device ``device_index`` (``diffusion_cuda_graph.live_graph_count``)."""
+ return sum(
+ len(b.cache)
+ for b in tuple(_LIVE_BLOCKS)
+ if getattr(b, "_unsloth_capture_device", None) == device_index
+ )
+
+
class BlockGraph:
"""Replaces one block's compute callable (below its offload hooks) with a recording keyed by inputs and weight
placement. Any refusal or failure runs ``compute`` itself, so the block is never worse than ungraphed."""
@@ -594,6 +606,11 @@ class BlockGraph:
self.stats["recaptures"] += 1
self.cache[full] = entry
self.stats["captures"] += 1
+ from .diffusion_cuda_graph import capture_device, note_capture
+
+ self._unsloth_capture_device = capture_device()
+ note_capture(self._unsloth_capture_device)
+ _LIVE_BLOCKS.add(self)
while len(self.cache) > self.max_graphs:
_, evicted = self.cache.popitem(last = False)
self._forget(evicted)
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index c71733fc17e..3b0238b5e45 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -300,6 +300,50 @@ def captures_exhausted() -> bool:
return len(_FAILED_GRAPHS) >= MAX_FAILED_CAPTURES
+def capture_device() -> Optional[int]:
+ """The CUDA ordinal a capture records on (the current device), or None when unreadable."""
+ try:
+ return int(_torch().cuda.current_device())
+ except Exception: # noqa: BLE001
+ return None
+
+
+def live_graph_count(device_index: Optional[int]) -> int:
+ """Graphs alive on CUDA device ``device_index`` (whole-step and per-block). Their statics and pool stay allocated
+ after the render that recorded them, so a later render's start allocation includes them; a reset frees them."""
+ total = sum(
+ len(w.cache)
+ for w in tuple(_LIVE_WRAPPERS)
+ if getattr(w, "_unsloth_capture_device", None) == device_index
+ )
+ from .diffusion_block_graph import live_block_graph_count
+
+ return total + live_block_graph_count(device_index)
+
+
+_CAPTURES: dict = {}
+# image and video render threads capture concurrently: an unserialized read-modify-write could lose an increment
+_CAPTURES_LOCK = threading.Lock()
+
+
+def note_capture(device_index: Optional[int]) -> None:
+ """Count a recorded graph (whole-step or per-block) on ``device_index``; see ``capture_generation``."""
+ with _CAPTURES_LOCK:
+ _CAPTURES[device_index] = _CAPTURES.get(device_index, 0) + 1
+
+
+def capture_generation(device_index: Optional[int]) -> int:
+ """Graphs ever recorded on ``device_index``. A full cache evicts before it captures, so a replacement leaves
+ ``live_graph_count`` unchanged; this count still moves."""
+ return _CAPTURES.get(device_index, 0)
+
+
+def failed_capture_count() -> int:
+ """Captures that failed in this process (an absorbed OOM among them): a render that saw one grow peaked at the
+ failed attempt, not at what it needed."""
+ return len(_FAILED_GRAPHS)
+
+
def refuse_if_exhausted(logger: Any = None) -> None:
"""Raise (the caller's failed-capture path then runs eager) once the failure budget is spent; log that once."""
global _EXHAUSTED_LOGGED
@@ -1130,6 +1174,8 @@ class GraphedForward:
return self._eager(eager_args, eager_kwargs)
entry = captured
self.cache[key] = entry
+ self._unsloth_capture_device = capture_device()
+ note_capture(self._unsloth_capture_device)
if self.logger is not None:
self.logger.debug(
"diffusion.cuda_graph: captured %s graph %d/%d over %d input tensor(s)",
diff --git a/studio/backend/core/inference/diffusion_measured_peaks.py b/studio/backend/core/inference/diffusion_measured_peaks.py
new file mode 100644
index 00000000000..20486e6232b
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_measured_peaks.py
@@ -0,0 +1,523 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Learned activation peaks: the real CUDA peak above the resident weights, recorded per
+(family, denoiser quant, speed tier, resolution bucket, frames, GPU name) at a successful render and persisted under
+the Studio cache dir, so later loads of the same model place it from a measurement instead of the flat
+8192 MiB/MP estimate.
+
+The static tables in ``diffusion_memory`` stay the seed: a learned value only RAISES a seeded peak (a local
+measurement above the seed means this machine needs more) and supplies one where no seed exists. It never lowers a
+seed, because one render does not exercise every phase (a prompt-cache hit skips the encoder, a smaller request
+decodes less), and the seed was measured on the worst phase.
+
+``UNSLOTH_DIFFUSION_LEARNED_PEAKS=0`` turns lookups and recording off; ``UNSLOTH_DIFFUSION_LEARNED_PEAKS_FILE``
+names the JSON file (default ``<studio cache>/diffusion_measured_peaks.json``).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import math
+import os
+import tempfile
+import threading
+import weakref
+from pathlib import Path
+from typing import Any, Optional
+
+LEARNED_PEAKS_ENV = "UNSLOTH_DIFFUSION_LEARNED_PEAKS"
+LEARNED_PEAKS_FILE_ENV = "UNSLOTH_DIFFUSION_LEARNED_PEAKS_FILE"
+_FILE_NAME = "diffusion_measured_peaks.json"
+_VERSION = 1
+_MAX_ENTRIES = 512
+_MEGAPIXEL = 1024 * 1024
+# Smallest bucket: a quarter megapixel. Below it the activation is dominated by fixed costs.
+_MIN_BUCKET_PIXELS = _MEGAPIXEL // 4
+
+_lock = threading.Lock()
+_cache: dict[str, Any] = {"path": None, "mtime": None, "data": None}
+
+
+def enabled() -> bool:
+ return (os.environ.get(LEARNED_PEAKS_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def store_path() -> Optional[Path]:
+ explicit = (os.environ.get(LEARNED_PEAKS_FILE_ENV) or "").strip()
+ if explicit:
+ return Path(explicit)
+ try:
+ from utils.paths.storage_roots import cache_root
+ except Exception: # noqa: BLE001 - imported without studio/backend on sys.path: nowhere safe to write
+ return None
+ try:
+ return cache_root() / _FILE_NAME
+ except Exception: # noqa: BLE001
+ return None
+
+
+def resolution_bucket(
+ width: Optional[int],
+ height: Optional[int],
+ batch_size: int = 1,
+) -> int:
+ """Pixels of the power-of-two megapixel bucket covering ``width x height x batch`` (at least 1/4 MP)."""
+ pixels = max(1, int(width or 1024)) * max(1, int(height or 1024)) * max(1, int(batch_size or 1))
+ if pixels <= _MIN_BUCKET_PIXELS:
+ return _MIN_BUCKET_PIXELS
+ return int(_MIN_BUCKET_PIXELS * 2 ** math.ceil(math.log2(pixels / _MIN_BUCKET_PIXELS)))
+
+
+def model_key(
+ family: Optional[str],
+ quant: Optional[str],
+ speed_mode: Optional[str],
+ frames: int = 1,
+ device: Optional[str] = None,
+) -> str:
+ """``device``: the GPU's name. Kernels and workspaces differ by card, so a heterogeneous host keeps one set of
+ peaks per card model instead of placing one GPU from another's measurement."""
+ return "|".join(
+ (
+ str(family or "unknown").lower(),
+ str(quant or "unknown"),
+ str(speed_mode or "unknown"),
+ f"f{max(1, int(frames or 1))}",
+ str(device or "unknown"),
+ )
+ )
+
+
+_runtime: list = []
+_KERNEL_DISTS = (
+ "triton",
+ "torchao",
+ "flash-attn",
+ "flash_attn_3",
+ "xformers",
+ "sageattention",
+ "kernels",
+)
+
+
+def runtime_stamp() -> str:
+ """The runtimes a peak depends on (kernel choice, workspaces): a store written under others is ignored."""
+ if not _runtime:
+ from importlib.metadata import version
+
+ parts = []
+ for dist in ("torch", "diffusers"):
+ try:
+ parts.append(f"{dist}={version(dist)}")
+ except Exception: # noqa: BLE001
+ parts.append(f"{dist}=?")
+ # attention and quantization kernels (their workspaces are part of a peak); an absent one is left out
+ for dist in _KERNEL_DISTS:
+ try:
+ parts.append(f"{dist}={version(dist)}")
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ import torch
+ parts.append(f"cuda={torch.version.cuda or torch.version.hip or '?'}")
+ except Exception: # noqa: BLE001
+ parts.append("cuda=?")
+ _runtime.append("|".join(parts))
+ return _runtime[0]
+
+
+def _empty() -> dict:
+ return {"version": _VERSION, "runtime": runtime_stamp(), "entries": {}}
+
+
+def _read(path: Path, fresh: bool = False) -> dict:
+ try:
+ mtime = path.stat().st_mtime_ns
+ except OSError:
+ return _empty()
+ if (
+ not fresh
+ and _cache["path"] == str(path)
+ and _cache["mtime"] == mtime
+ and _cache["data"] is not None
+ ):
+ return _cache["data"]
+ try:
+ data = json.loads(path.read_text())
+ if (
+ not isinstance(data, dict)
+ or data.get("version") != _VERSION
+ or data.get("runtime") != runtime_stamp()
+ or not isinstance(data.get("entries"), dict)
+ ):
+ data = _empty()
+ except Exception: # noqa: BLE001 - a corrupt file is an empty store, never a failed load
+ data = _empty()
+ _cache.update(path = str(path), mtime = mtime, data = data)
+ return data
+
+
+def _file_lock(path: Path):
+ """Exclusive lock on ``<store>.lock`` across processes sharing the cache root (two Studio servers, a worker), so a
+ read / merge / replace cannot overwrite another process's higher peak. Advisory; held only for the merge."""
+ path.parent.mkdir(parents = True, exist_ok = True)
+ handle = open(str(path) + ".lock", "a+b")
+ try:
+ try:
+ import fcntl
+ fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
+ except ImportError: # Windows
+ import msvcrt
+ handle.seek(0)
+ msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
+ yield
+ finally:
+ handle.close() # closing releases either lock
+
+
+def _write(path: Path, data: dict) -> None:
+ path.parent.mkdir(parents = True, exist_ok = True)
+ fd, tmp = tempfile.mkstemp(prefix = ".peaks.", dir = str(path.parent))
+ try:
+ with os.fdopen(fd, "w") as handle:
+ json.dump(data, handle, indent = 1) # insertion order is the eviction order
+ os.replace(tmp, path)
+ except BaseException:
+ try:
+ os.unlink(tmp)
+ except OSError:
+ pass
+ raise
+ _cache.update(path = None, mtime = None, data = None)
+
+
+def record(
+ *,
+ family: Optional[str],
+ quant: Optional[str],
+ speed_mode: Optional[str],
+ width: Optional[int],
+ height: Optional[int],
+ batch_size: int = 1,
+ frames: int = 1,
+ peak_mib: int,
+ device_name: Optional[str] = None,
+ quadratic: bool = False,
+ held: Optional[list] = None,
+) -> bool:
+ """Keep the max peak seen for this key and bucket, scaled to the bucket's ceiling so a smaller render in the bucket
+ still answers a full-size one (squared with ``quadratic``: SDPA math attention grows with pixels squared, and the
+ key's entry keeps scaling that way). Returns True when the stored value rose. ``held`` (a list) gets True appended
+ when the store holds at least this peak afterwards (raised, or already covered): False when nothing was persisted."""
+ if (
+ not enabled()
+ or peak_mib is None
+ or int(peak_mib) <= 0
+ or not family
+ or not identified(quant)
+ ):
+ return False
+ if quadratic and int(batch_size or 1) > 1:
+ return False # a math-attention entry holds single images: it scales squared in pixels, linearly in batch
+ path = store_path()
+ if path is None:
+ return False
+ device_name = device_name or current_device_name()
+ key = model_key(family, quant, speed_mode, frames, device_name)
+ bucket_pixels = resolution_bucket(width, height, batch_size)
+ bucket = str(bucket_pixels)
+ pixels = max(1, int(width or 1024)) * max(1, int(height or 1024)) * max(1, int(batch_size or 1))
+ peak_mib = int(
+ math.ceil(
+ int(peak_mib)
+ * (bucket_pixels / float(max(pixels, bucket_pixels // 2))) ** (2 if quadratic else 1)
+ )
+ )
+ try:
+ with _lock, _file_lock(path):
+ data = _read(path, fresh = True)
+ entries = data.setdefault("entries", {})
+ entry = entries.pop(key, None) or {"peaks": {}}
+ entries[key] = entry # most recently raised last: the cap evicts from the front
+ peaks = entry.setdefault("peaks", {})
+ old = int(peaks.get(bucket) or 0)
+ if int(peak_mib) <= old:
+ if held is not None:
+ held.append(True)
+ return False
+ peaks[bucket] = int(peak_mib)
+ if quadratic:
+ entry["quadratic"] = True
+ if device_name:
+ entry["device"] = str(device_name) # also in the key; kept for readers of the file
+ if len(entries) > _MAX_ENTRIES:
+ for stale in list(entries)[: len(entries) - _MAX_ENTRIES]:
+ entries.pop(stale, None)
+ _write(path, data)
+ if held is not None:
+ held.append(True)
+ return True
+ except Exception: # noqa: BLE001 - persistence is best-effort
+ return False
+
+
+def lookup(
+ *,
+ family: Optional[str],
+ quant: Optional[str],
+ speed_mode: Optional[str],
+ width: Optional[int] = None,
+ height: Optional[int] = None,
+ batch_size: int = 1,
+ frames: int = 1,
+ device: Optional[str] = None,
+) -> Optional[int]:
+ """Learned peak MiB (no margin) for a request of this size, or None when nothing was recorded.
+ ``device`` (GPU name) defaults to the current CUDA device, the one the planner snapshots.
+
+ The smallest recorded bucket at or above the request's bucket answers as is (an overestimate); with only smaller
+ buckets recorded, the largest one is scaled linearly by pixel count."""
+ found = lookup_bucket(
+ family = family,
+ quant = quant,
+ speed_mode = speed_mode,
+ width = width,
+ height = height,
+ batch_size = batch_size,
+ frames = frames,
+ device = device,
+ )
+ return None if found is None else found[0]
+
+
+def identified(quant: Optional[str]) -> bool:
+ """False when the label or a part of it could not be identified (empty, a bare ``unknown`` format,
+ ``name:unknown``, a ``#unknown`` quant variant): such a load neither reads nor records a learned peak and keeps the static estimate."""
+ parts = str(quant or "").split("|")
+ return bool(parts[0]) and not any(
+ part == "unknown" or part.endswith((":unknown", "#unknown")) for part in parts
+ )
+
+
+def lookup_bucket(
+ *,
+ family: Optional[str],
+ quant: Optional[str],
+ speed_mode: Optional[str],
+ width: Optional[int] = None,
+ height: Optional[int] = None,
+ batch_size: int = 1,
+ frames: int = 1,
+ device: Optional[str] = None,
+) -> Optional[tuple[int, int]]:
+ """``lookup`` plus the pixels the peak covers and whether it scales squared: (peak MiB, bucket pixels,
+ quadratic), or None. A ``graphs:on`` label (a load placed before it knows whether placement keeps the whole-step
+ graph or records per block) reads an envelope over the ``graphs:step`` and ``graphs:blocks`` entries."""
+ if "graphs:on" in str(quant or "").split("|"):
+ found = [
+ lookup_bucket(
+ family = family,
+ quant = "|".join(
+ f"graphs:{mode}" if part == "graphs:on" else part
+ for part in str(quant).split("|")
+ ),
+ speed_mode = speed_mode,
+ width = width,
+ height = height,
+ batch_size = batch_size,
+ frames = frames,
+ device = device,
+ )
+ for mode in ("step", "blocks")
+ ]
+ found = [f for f in found if f is not None]
+ if not found:
+ return None
+ # one envelope over both modes (the placement can arm either): the larger peak over the smaller covered
+ # bucket, scaled squared if either mode is, sizes every later request at least as large as either mode would
+ return (
+ max(f[0] for f in found),
+ min(f[1] for f in found),
+ any(f[2] for f in found),
+ )
+ if not enabled() or not family or not identified(quant):
+ return None
+ path = store_path()
+ if path is None:
+ return None
+ try:
+ with _lock:
+ key = model_key(family, quant, speed_mode, frames, device or current_device_name())
+ entry = (_read(path).get("entries") or {}).get(key)
+ peaks = {
+ int(k): int(v) for k, v in ((entry or {}).get("peaks") or {}).items() if int(v) > 0
+ }
+ except Exception: # noqa: BLE001
+ return None
+ if not peaks:
+ return None
+ quadratic = bool((entry or {}).get("quadratic"))
+ batch = max(1, int(batch_size or 1))
+ # a math-attention entry holds single images (``record``): its buckets are per image, the batch scales linearly
+ want = resolution_bucket(width, height, 1 if quadratic else batch)
+ times = batch if quadratic else 1
+ above = [b for b in peaks if b >= want]
+ if above:
+ return peaks[min(above)] * times, min(above), quadratic
+ largest = max(peaks)
+ power = 2 if quadratic else 1
+ return (
+ int(math.ceil(peaks[largest] * (want / float(largest)) ** power)) * times,
+ want,
+ quadratic,
+ )
+
+
+def record_work(
+ key: Any,
+ work: int,
+ peak_mib: int,
+ held: Optional[list] = None,
+) -> bool:
+ """Persist a peak under an opaque calibration key (the video path's work = pixels x frames table); max wins.
+ ``held`` as in ``record``."""
+ if not enabled() or key is None or peak_mib is None or int(peak_mib) <= 0 or int(work) <= 0:
+ return False
+ path = store_path()
+ if path is None:
+ return False
+ try:
+ with _lock, _file_lock(path):
+ data = _read(path, fresh = True)
+ table = data.setdefault("work", {})
+ peaks = table.pop(repr(key), None) or {}
+ table[repr(key)] = peaks
+ old = int(peaks.get(str(int(work))) or 0)
+ if int(peak_mib) <= old:
+ if held is not None:
+ held.append(True)
+ return False
+ peaks[str(int(work))] = int(peak_mib)
+ if len(table) > _MAX_ENTRIES:
+ for stale in list(table)[: len(table) - _MAX_ENTRIES]:
+ table.pop(stale, None)
+ _write(path, data)
+ if held is not None:
+ held.append(True)
+ return True
+ except Exception: # noqa: BLE001 - persistence is best-effort
+ return False
+
+
+def work_peaks(key: Any) -> dict[int, int]:
+ """Persisted {work: peak MiB} for an opaque calibration key ({} when none or disabled)."""
+ if not enabled() or key is None:
+ return {}
+ path = store_path()
+ if path is None:
+ return {}
+ try:
+ with _lock:
+ peaks = (_read(path).get("work") or {}).get(repr(key)) or {}
+ return {int(w): int(v) for w, v in peaks.items() if int(v) > 0}
+ except Exception: # noqa: BLE001
+ return {}
+
+
+class _Window:
+ overlapped = False
+
+
+_windows_lock = threading.Lock()
+_windows: dict[str, "weakref.WeakSet[_Window]"] = {}
+
+
+def open_window(device: Any) -> _Window:
+ """Register a peak measurement on ``device``. Resetting the peak is allocator-wide, so a second render arming on
+ the same card (image and video run on separate threads) spoils both: each one's window reads as overlapped. An
+ abandoned window drops out once nothing references it."""
+ window = _Window()
+ with _windows_lock:
+ live = _windows.setdefault(str(device), weakref.WeakSet())
+ if len(live):
+ window.overlapped = True
+ for other in live:
+ other.overlapped = True
+ live.add(window)
+ return window
+
+
+def close_window(device: Any, window: Optional[_Window]) -> bool:
+ """End a measurement; True when no other measurement on the card overlapped it."""
+ if window is None:
+ return True
+ with _windows_lock:
+ live = _windows.get(str(device))
+ if live is not None:
+ live.discard(window)
+ return not window.overlapped
+
+
+class PeakProbe:
+ """CUDA peak above the allocation at ``start``; ``finish`` returns MiB or None (non-CUDA / unreadable, or another
+ measurement on the card reset the peak meanwhile)."""
+
+ def __init__(self, device: Any) -> None:
+ self.device = None
+ self.base = None
+ try:
+ import torch
+
+ dev = torch.device(device)
+ if dev.type != "cuda" or not torch.cuda.is_available():
+ return
+ if dev.index is None:
+ dev = torch.device("cuda", torch.cuda.current_device())
+ self.window = open_window(dev)
+ torch.cuda.synchronize(dev)
+ torch.cuda.reset_peak_memory_stats(dev)
+ self.base = int(torch.cuda.memory_allocated(dev))
+ self.device = dev
+ except Exception: # noqa: BLE001 - measurement only
+ self.device = None
+
+ def finish(self) -> Optional[int]:
+ if self.device is None or self.base is None:
+ return None
+ try:
+ import torch
+ torch.cuda.synchronize(self.device)
+ peak = int(torch.cuda.max_memory_allocated(self.device)) - self.base
+ except Exception: # noqa: BLE001
+ peak = None
+ # closed only after the read: a render arming in between would otherwise reset the peak unseen
+ if not close_window(self.device, getattr(self, "window", None)) or peak is None:
+ return None
+ return -(-peak // (1024 * 1024)) if peak > 0 else None
+
+
+def current_device_name() -> Optional[str]:
+ """Name of the current CUDA device (None without CUDA); the planner's snapshot reads this device."""
+ try:
+ import torch
+ if not torch.cuda.is_available():
+ return None
+ except Exception: # noqa: BLE001
+ return None
+ return device_name("cuda")
+
+
+def device_name(device: Any) -> Optional[str]:
+ try:
+ import torch
+ return str(torch.cuda.get_device_name(torch.device(device)))
+ except Exception: # noqa: BLE001
+ return None
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 37f53f03b86..28de185be7a 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -2284,37 +2284,325 @@ def measured_image_runtime_mib(
batch_size: int = 1,
dense_transformer_mib: Optional[int] = None,
compute_bytes: int = 2,
+ quant: Optional[str] = None,
+ phase_seeds: bool = False,
) -> Optional[int]:
"""Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate).
- ``dense_transformer_mib`` selects the dense eager table; ``compute_bytes`` scales it (fp32 Qwen-Image: 8480 vs 4248)."""
+ ``dense_transformer_mib`` selects the dense eager table; ``compute_bytes`` scales it (fp32 Qwen-Image: 8480 vs 4248).
+
+ Seeds: the peak tables above, then the per-phase calibration (``_MEASURED_IMAGE_ACTIVATION_MIB``, the same
+ quantity per phase) for a quantised family the peak table lacks, only with ``phase_seeds`` (the target is where
+ those were measured: NVIDIA with a sub-quadratic attention kernel, ``_calibrated_activation``). ``quant`` (``denoiser_quant_label``) adds the
+ peak this machine recorded for (family, quant, speed tier, resolution bucket) at an earlier render
+ (diffusion_measured_peaks): it raises a seed and stands in for a missing one, never lowers a seed."""
if _env_off(MEASURED_ACTIVATION_ENV):
return None
+ seed: Optional[int] = None
if dense_transformer_mib is not None:
if _env_off(MEASURED_ACTIVATION_DENSE_ENV):
return None
entry = _MEASURED_DENSE_EAGER_PEAK_MIB.get(str(family or "").lower())
- if entry is None or str(speed_mode or "") not in _MEASURED_DENSE_SPEED_MODES:
- return None
- peak, max_dit = entry
- widen = max(2, int(compute_bytes or 2))
- peak, max_dit = peak * widen // 2, max_dit * widen // 2
- if int(dense_transformer_mib) <= 0 or int(dense_transformer_mib) > max_dit:
- return None
- else:
- peak = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())
- if peak is None or str(speed_mode or "") not in _MEASURED_PEAK_SPEED_MODES:
- return None
+ if entry is not None and str(speed_mode or "") in _MEASURED_DENSE_SPEED_MODES:
+ peak, max_dit = entry
+ widen = max(2, int(compute_bytes or 2))
+ peak, max_dit = peak * widen // 2, max_dit * widen // 2
+ if 0 < int(dense_transformer_mib) <= max_dit:
+ seed = peak
+ elif str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES:
+ seed = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())
+ if seed is None and phase_seeds:
+ phases = _MEASURED_IMAGE_ACTIVATION_MIB.get(str(family or "").lower())
+ if phases is not None:
+ # worst of encoder / denoise / untiled decode at 1024x1024 for this tier
+ seed = max(phases[1 if speed_mode == "max" else 0][:3])
w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
- scale = max(
- 1.0,
- (w * h * max(1, int(batch_size or 1))) / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT),
- )
- need = peak * scale * _MEASURED_PEAK_MARGIN
+ batch = max(1, int(batch_size or 1))
+ learned = None
+ if quant is not None:
+ from .diffusion_measured_peaks import lookup as learned_peak
+ learned = learned_peak(
+ family = family, quant = quant, speed_mode = speed_mode, width = w, height = h, batch_size = batch
+ )
+ if seed is None and learned is None:
+ return None
+ scale = max(1.0, (w * h * batch) / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT))
+ need = max(float(seed or 0) * scale, float(learned or 0)) * _MEASURED_PEAK_MARGIN
step = _MEASURED_PEAK_ROUND_MIB
return int(-(-need // step) * step)
+def learned_peak_label(
+ pipe: Any,
+ attention: Optional[str] = None,
+ model: Optional[str] = None,
+ vae: Optional[str] = None,
+ te: Optional[str] = None,
+ cache: Optional[str] = None,
+ graphs: Optional[str] = None,
+) -> str:
+ """``denoiser_quant_label`` plus the loaded model (variants of one family differ in size), the text encoders'
+ precision (a dense streamed encoder peaks above a quantised one), the VAE savers (an untiled decode peaks far
+ above a tiled one) and the engaged attention backend (backends with different workspaces): none shares a learned
+ peak. The VAE savers and the backend can change after the load; ``with_label_part`` rewrites them."""
+ label = denoiser_quant_label(pipe)
+ for name, value in (
+ ("model", model),
+ ("te", te),
+ ("cache", cache),
+ ("vae", vae),
+ ("attn", attention),
+ ("graphs", graphs),
+ ):
+ if value is not None:
+ label = f"{label}|{name}:{value}"
+ return label
+
+
+def with_label_part(label: str, name: str, value: str) -> str:
+ """``label`` with its ``name:`` part replaced by ``value`` (unchanged when it has no such part)."""
+ return "|".join(
+ f"{name}:{value}" if part.startswith(f"{name}:") else part for part in str(label).split("|")
+ )
+
+
+_VAE_SAVERS = (
+ ("use_tiling", "enable_vae_tiling", "enable_tiling"),
+ ("use_slicing", "enable_vae_slicing", "enable_slicing"),
+)
+
+
+def _vae_saver_can(pipe: Any, pipe_method: str, vae_method: str) -> bool:
+ return callable(getattr(pipe, pipe_method, None)) or callable(
+ getattr(getattr(pipe, "vae", None), vae_method, None)
+ )
+
+
+def _vae_savers_observable(pipe: Any) -> bool:
+ """False when a saver can be engaged but the VAE keeps no ``use_*`` flag to read it back: its state is unknown."""
+ vae = getattr(pipe, "vae", None)
+ return all(
+ hasattr(vae, flag) or not _vae_saver_can(pipe, pm, vm) for flag, pm, vm in _VAE_SAVERS
+ )
+
+
+def planned_vae_saver_tag(pipe: Any, plan: Any) -> str:
+ """The VAE savers ``plan`` asks for that this pipeline can engage (what ``apply_memory_plan`` will turn on);
+ ``unknown`` (no learned peak used or recorded) when the VAE cannot report them."""
+ if not _vae_savers_observable(pipe):
+ return "unknown"
+ return vae_saver_tag(
+ getattr(plan, "vae_tiling", False) and _vae_saver_can(pipe, *_VAE_SAVERS[0][1:]),
+ getattr(plan, "vae_slicing", False) and _vae_saver_can(pipe, *_VAE_SAVERS[1][1:]),
+ )
+
+
+def live_vae_saver_tag(pipe: Any) -> str:
+ """The VAE savers in force right now (``unknown`` as in ``planned_vae_saver_tag``)."""
+ if not _vae_savers_observable(pipe):
+ return "unknown"
+ vae = getattr(pipe, "vae", None)
+ return vae_saver_tag(getattr(vae, "use_tiling", False), getattr(vae, "use_slicing", False))
+
+
+def stream_window_pad_mib(pipe: Any) -> Optional[int]:
+ """MiB a learned peak measured with no denoiser group streaming adds to reach the streamed baseline (the prefetch
+ window a streamed placement holds above its start allocation), so every placement reads one conservative scale.
+ None when that window cannot be sized, or the denoiser is neither group-streamed nor on the card (whole-module /
+ sequential offload onloads it whole inside the render): the peak is then not recorded. Resident text encoders add
+ their leaf-streaming window the same way (``_encoder_stream_window_mib``)."""
+ encoders = _encoder_stream_window_mib(pipe)
+ if encoders is None:
+ return None
+ streaming = partial = False
+ for name, module in (getattr(pipe, "components", {}) or {}).items():
+ if str(name) not in ("transformer", "transformer_2", "unconditional_transformer"):
+ continue
+ state = _streaming_state(module)
+ if state is None:
+ return None
+ if state:
+ streaming = True
+ partial = partial or any(
+ getattr(g, "_unsloth_resident", False) for g in _offload_groups(module)
+ )
+ if streaming and not partial:
+ return encoders
+ window = _stream_window_mib(pipe)
+ if window < 0:
+ return None
+ if streaming:
+ # some groups resident: the measured window is the largest STREAMED group's, a fully streamed load holds the
+ # largest of all; pad the difference
+ streamed = _streamed_group_window_mib(pipe)
+ if streamed < 0:
+ return None
+ window = max(0, window - streamed)
+ return window + encoders
+
+
+def _streamed_group_window_mib(pipe: Any) -> int:
+ """Device MiB the prefetcher holds for the denoiser groups that stream now: ``depth + 1`` of the largest, or -1."""
+ try:
+ from .diffusion_offload_prefetch import prefetch_depth
+
+ largest = 0
+ for name, module in (getattr(pipe, "components", {}) or {}).items():
+ if str(name) not in ("transformer", "transformer_2", "unconditional_transformer"):
+ continue
+ for group in _offload_groups(module):
+ if getattr(group, "_unsloth_resident", False):
+ continue
+ largest = max(largest, _group_nbytes(group))
+ if largest <= 0:
+ return -1
+ return -(-largest * (prefetch_depth() + 1) // (1024 * 1024))
+ except Exception: # noqa: BLE001
+ return -1
+
+
+def _group_nbytes(group: Any) -> int:
+ """Storage bytes of one offload group's modules, parameters and buffers (shared tensors once)."""
+ seen: set[int] = set()
+ nbytes = 0
+ for member in getattr(group, "modules", None) or ():
+ for t in (*member.parameters(), *member.buffers()):
+ if id(t) not in seen:
+ seen.add(id(t))
+ nbytes += sum(_storage_nbytes(t))
+ for t in (
+ *(getattr(group, "parameters", None) or ()),
+ *(getattr(group, "buffers", None) or ()),
+ ):
+ if id(t) not in seen:
+ seen.add(id(t))
+ nbytes += sum(_storage_nbytes(t))
+ return nbytes
+
+
+def _streaming_state(module: Any) -> Optional[bool]:
+ """True when ``module`` streams offload groups, False when it sits on the card, None when neither can be told
+ (whole-module / sequential offload, weights off the card without hooks)."""
+ try:
+ groups = _offload_groups(module) or ()
+ if any(not getattr(g, "_unsloth_resident", False) for g in groups):
+ return True
+ if not groups:
+ if getattr(module, "_hf_hook", None) is not None:
+ return None
+ first = next(iter(module.parameters()), None)
+ if first is not None and getattr(first.device, "type", None) != "cuda":
+ return None
+ return False
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _encoder_stream_window_mib(pipe: Any) -> Optional[int]:
+ """MiB a load that streams the text encoders (leaf-level groups) holds in flight that a render with them resident
+ did not: ``depth + 1`` of the largest leaf of each resident encoder, less the streamed leaves' window of a
+ partially resident one. None when an encoder's placement is unknown."""
+ try:
+ import torch
+
+ from .diffusion_offload_prefetch import prefetch_depth
+
+ total = 0
+ for name, module in (getattr(pipe, "components", {}) or {}).items():
+ if not str(name).startswith("text_encoder") or not isinstance(module, torch.nn.Module):
+ continue
+ state = _streaming_state(module)
+ if state is None:
+ return None
+ groups = list(_offload_groups(module) or ()) if state else []
+ if state and not any(getattr(g, "_unsloth_resident", False) for g in groups):
+ continue # fully streamed during the render: its whole window is in the peak
+ largest = 0
+ for sub in module.modules():
+ seen: set[int] = set()
+ nbytes = 0
+ for t in (*sub.parameters(recurse = False), *sub.buffers(recurse = False)):
+ if id(t) not in seen:
+ seen.add(id(t))
+ nbytes += sum(_storage_nbytes(t))
+ largest = max(largest, nbytes)
+ window = -(-largest * (prefetch_depth() + 1) // (1024 * 1024))
+ if state:
+ # some leaves resident: the peak holds the largest STREAMED leaf's window, a fully streamed encoder
+ # the largest of all; pad the difference
+ streamed = max(
+ (
+ _group_nbytes(g)
+ for g in groups
+ if not getattr(g, "_unsloth_resident", False)
+ ),
+ default = 0,
+ )
+ window = max(0, window - -(-streamed * (prefetch_depth() + 1) // (1024 * 1024)))
+ total += window
+ return total
+ except Exception: # noqa: BLE001
+ return None
+
+
+def vae_saver_tag(tiling: Any, slicing: Any) -> str:
+ """The VAE savers in force, as the learned-peak key names them."""
+ return "+".join(name for name, on in (("tiled", tiling), ("sliced", slicing)) if on) or "full"
+
+
+def denoiser_quant_label(pipe: Any) -> str:
+ """Stable label of the denoiser weights' format for the learned-peak key: torchao subclasses, GGUF, int8 / uint8
+ packed, or the dense dtype."""
+ try:
+ import torch
+
+ dits = [
+ getattr(pipe, name, None)
+ for name in ("transformer", "transformer_2", "unconditional_transformer", "unet")
+ ]
+ dits = [d for d in dits if isinstance(d, torch.nn.Module)]
+ if not dits:
+ return "none"
+ classes: set[str] = set()
+ for dit in dits:
+ classes |= _torchao_weight_classes(dit)
+ if classes:
+ return "torchao:" + "+".join(sorted(classes)) + _quant_variant_tag(dits)
+ dtypes: set[str] = set()
+ for dit in dits:
+ for param in dit.parameters():
+ if type(param).__name__ == "GGUFParameter":
+ return "gguf"
+ if param.dtype in (torch.int8, torch.uint8):
+ dtypes.add("int8")
+ elif param.is_floating_point():
+ dtypes.add(str(param.dtype).replace("torch.", ""))
+ if "int8" in dtypes:
+ return "packed:int8" + _quant_variant_tag(dits)
+ return "dense:" + "+".join(sorted(dtypes)) if dtypes else "unknown"
+ except Exception: # noqa: BLE001
+ return "unknown"
+
+
+def _quant_variant_tag(dits: list) -> str:
+ """``#`` plus a short digest of the denoisers' quant graph variant (``graph_variant``, the compile cache's key):
+ plain vs ConvRot, W8A8 vs weight-only, hosted vs runtime scales share a weight class but not their kernels or
+ workspaces. ``#unknown`` when it cannot be read (the label then neither reads nor records a peak)."""
+ try:
+ import hashlib
+ import json
+
+ from .diffusion_compile_cache import graph_variant
+
+ variants = [graph_variant(dit) for dit in dits]
+ if any(isinstance(v.get("weights"), str) for v in variants):
+ return "#unknown"
+ blob = json.dumps(variants, sort_keys = True, default = str)
+ return "#" + hashlib.sha256(blob.encode()).hexdigest()[:12]
+ except Exception: # noqa: BLE001
+ return "#unknown"
+
+
def _loaded_component_mib(pipe: Any) -> Optional[dict[str, tuple[int, str]]]:
"""name -> (MiB, role in 'dit' / 'text_encoder' / 'other') from storage bytes (a torchao int8 weight counts its
int8 data + scales); None when unreadable."""
@@ -2441,6 +2729,12 @@ def refine_plan_from_loaded_weights(
family: Optional[str],
speed_mode: Optional[str],
logger: Any = None,
+ attention: Optional[str] = None,
+ model: Optional[str] = None,
+ vae: Optional[str] = None,
+ te: Optional[str] = None,
+ cache: Optional[str] = None,
+ graphs: Optional[str] = None,
) -> MemoryPlan:
"""Re-place a streamed ``auto`` load from its LOADED weights and the family's measured activation peak.
@@ -2473,16 +2767,34 @@ def refine_plan_from_loaded_weights(
if dense_mib is not None and (compute_bytes is None or _pipe_denoisers_hold_packed(pipe)):
# measured on fp16 / fp32 cards only; bf16 cards can compile later
return plan
+ quant = learned_peak_label(pipe, attention, model, vae, te, cache, graphs)
+ try:
+ pipe._unsloth_peak_identity = (family, quant, speed_mode)
+ except Exception: # noqa: BLE001 - a pipe refusing attributes only skips recording
+ pass
headroom = measured_image_runtime_mib(
- family, speed_mode, dense_transformer_mib = dense_mib, compute_bytes = compute_bytes or 2
+ family,
+ speed_mode,
+ dense_transformer_mib = dense_mib,
+ compute_bytes = compute_bytes or 2,
+ quant = quant,
+ phase_seeds = bool(getattr(pipe, "_unsloth_phase_seeds", False)),
)
if headroom is None:
return plan
+ from .diffusion_measured_peaks import lookup_bucket
+
+ learned, learned_pixels, learned_quadratic = lookup_bucket(
+ family = family, quant = quant, speed_mode = speed_mode
+ ) or (None, None, False)
overhead = int(plan.estimates.get("base_overhead_mib") or DEFAULT_BASE_OVERHEAD_MIB)
floor = headroom + overhead + other
estimates = dict(plan.estimates)
estimates.update(
measured_runtime_headroom_mib = headroom,
+ measured_learned_peak_mib = learned,
+ measured_learned_peak_pixels = learned_pixels,
+ measured_learned_peak_quadratic = learned_quadratic,
measured_dense_transformer_mib = dense_mib,
measured_compute_bytes = compute_bytes,
loaded_transformer_mib = dit,
@@ -2523,7 +2835,9 @@ def refine_plan_from_loaded_weights(
encode_room = int(dit_room)
dit_room, room, wide_dit = streamed_room, streamed_room, True
estimates["stream_window_mib"] = window
- te_room = max(0, room - dit_room) if stream_te and policy == OFFLOAD_GROUP else 0
+ te_room = (
+ min(max(0, room - dit_room), encoders) if stream_te and policy == OFFLOAD_GROUP else 0
+ )
if dit_room <= 0 and te_room <= 0:
return plan
if whole_dit or wide_dit:
@@ -2540,7 +2854,12 @@ def refine_plan_from_loaded_weights(
},
reasons = plan.reasons
+ (
- f"{int(dit_room)} MiB of the {dit} MiB transformer and {int(te_room)} MiB of the {encoders} MiB "
+ (
+ f"the {dit} MiB transformer is resident (not streamed by the plan)"
+ if not bool(getattr(plan, "stream_transformer", True))
+ else f"{int(dit_room)} MiB of the {dit} MiB transformer"
+ )
+ + f" and {int(te_room)} MiB of the {encoders} MiB "
f"encoders stay resident (measured peak {headroom} MiB); the rest streams"
+ (
f" (whole transformer within free memory less a {_resident_dit_slack_mib(memory)} MiB slack;"
@@ -2653,35 +2972,57 @@ def _keep_groups_resident(
module._unsloth_resident_device = onload
except AttributeError:
pass
+ # Tied weights (an LM head sharing the embedding, as in Qwen3 text encoders) sit in two groups. A resident
+ # group next to a streamed partner breaks: the partner's offload points the shared tensor back at its host
+ # copy and the resident group's no-op onload never brings it back. Such groups go resident together or not.
+ tied = _tied_group_closures(ordered, _tensors)
+ done: set = set()
for group in ordered:
if getattr(group, "offload_to_disk_path", None):
break
- tensors = _tensors(group)
+ if id(group) in done:
+ continue
+ closure = tied.get(id(group)) or [group]
+ done.update(id(g) for g in closure)
+ unique: dict = {}
+ for member in closure:
+ for t in _tensors(member):
+ unique.setdefault(id(t), t)
+ tensors = list(unique.values())
need = sum(sum(_storage_nbytes(t)) for t in tensors)
- if getattr(group, "_unsloth_resident", False):
+ if all(getattr(g, "_unsloth_resident", False) for g in closure):
left -= need
continue
- if need > left or (only is not None and id(group) not in only):
+ if (
+ need > left
+ or any(getattr(g, "offload_to_disk_path", None) for g in closure)
+ or (only is not None and any(id(g) not in only for g in closure))
+ ):
# too large (a later, smaller group may still fit), or another release's group
continue
- cpu = getattr(group, "cpu_param_dict", None) or {}
- for t in tensors:
- if _placed_on(t, onload.type, is_torchao):
+ for member in closure:
+ cpu = getattr(member, "cpu_param_dict", None) or {}
+ for t in _tensors(member):
+ if _placed_on(t, onload.type, is_torchao):
+ continue
+ moved = cpu.get(t, t).to(onload)
+ if is_torchao(t):
+ go._swap_torchao_tensor(t, moved)
+ else:
+ t.data = moved
+ for i, member in enumerate(closure):
+ if getattr(member, "_unsloth_resident", False):
continue
- moved = cpu.get(t, t).to(onload)
- if is_torchao(t):
- go._swap_torchao_tensor(t, moved)
- else:
- t.data = moved
- # host copies stay: release_resident_groups streams the group again for an oversized request
- group._unsloth_streamed_hooks = (
- group.__dict__.get("onload_"),
- group.__dict__.get("offload_"),
- )
- group.onload_ = _resident_onload(getattr(group, "stream", None))
- group.offload_ = noop
- group._unsloth_resident = True
- group._unsloth_resident_bytes = need
+ # host copies stay: release_resident_groups streams the group again for an oversized request
+ member._unsloth_streamed_hooks = (
+ member.__dict__.get("onload_"),
+ member.__dict__.get("offload_"),
+ )
+ member.onload_ = _resident_onload(getattr(member, "stream", None))
+ member.offload_ = noop
+ member._unsloth_resident = True
+ member._unsloth_resident_bytes = need if i == 0 else 0
+ member._unsloth_tied_groups = [g for g in closure if g is not member]
left -= need
kept += need
state["streamed"] = _streamed_group_count(ordered)
@@ -2704,6 +3045,31 @@ def _keep_groups_resident(
return 0
+def _tied_group_closures(groups: list, tensors_of: Callable[[Any], list]) -> dict:
+ """id(group) -> every group that shares a tensor with it, transitively (only for groups that share any)."""
+ owners: dict[int, list] = {}
+ for group in groups:
+ for t in tensors_of(group):
+ owners.setdefault(id(t), []).append(group)
+ closures: dict = {}
+ for group in groups:
+ if id(group) in closures:
+ continue
+ stack, seen = [group], {id(group): group}
+ while stack:
+ current = stack.pop()
+ for t in tensors_of(current):
+ for other in owners.get(id(t), ()):
+ if id(other) not in seen:
+ seen[id(other)] = other
+ stack.append(other)
+ if len(seen) > 1:
+ members = [g for g in groups if id(g) in seen]
+ for g in members:
+ closures[id(g)] = members
+ return closures
+
+
def _bump_placement_epoch() -> None:
try:
from .diffusion_offload_prefetch import bump_placement_epoch
@@ -2768,9 +3134,13 @@ def release_resident_groups(
state = getattr(module, "_unsloth_stream_state", None)
if isinstance(state, dict):
state["streamed"] = 1
- _release_group(group)
- streamed.add(id(group))
- left -= int(getattr(group, "_unsloth_resident_bytes", 0))
+ # tied partners stream together (a resident partner would keep reading the host copy)
+ for member in [group, *(getattr(group, "_unsloth_tied_groups", None) or [])]:
+ if not getattr(member, "_unsloth_resident", False):
+ continue
+ _release_group(member)
+ streamed.add(id(member))
+ left -= int(getattr(member, "_unsloth_resident_bytes", 0))
if module not in released:
released.append(module)
if not released:
@@ -2941,7 +3311,32 @@ def _return_module_tensors(module: Any, onload: Any) -> None:
go._swap_torchao_tensor(t, moved)
else:
t.data = t.data.to(onload)
- torch.cuda.synchronize(onload)
+ if onload.type == "cuda":
+ torch.cuda.synchronize(onload)
+
+
+def _unwind_group_offload(
+ pipe: Any,
+ snapshots: list,
+ logger: Any = None,
+) -> bool:
+ """Undo a partial ``_apply_group_offload``: group hooks, the prefetchers' torch hooks and attributes, deferred
+ pins. The weights stay where the hooks left them (host copies or the card); the caller's retry moves them."""
+ try:
+ from .diffusion_offload_prefetch import PREFETCHER_ATTR
+ from .diffusion_small_host import ENCODER_PREFETCH_ATTR
+
+ for module, before in snapshots:
+ _remove_group_offload_hooks(module)
+ _drop_module_forward_hooks_since(module, before)
+ for attr in (PREFETCHER_ATTR, ENCODER_PREFETCH_ATTR, "_unsloth_stream_state"):
+ module.__dict__.pop(attr, None)
+ _drop_deferred_pinning(pipe, module)
+ return bool(snapshots)
+ except Exception as exc: # noqa: BLE001 - the original failure propagates
+ if logger is not None:
+ logger.warning("diffusion.memory: could not unwind partial group offload (%s)", exc)
+ return False
def _is_text_encoder_module(pipe: Any, module: Any) -> bool:
@@ -3045,15 +3440,65 @@ def measured_request_extra_mib(
headroom, family, speed_mode = reserve[:3]
dense_mib = reserve[3] if len(reserve) > 3 else None
compute_bytes = reserve[4] if len(reserve) > 4 and reserve[4] else 2
- need = measured_image_runtime_mib(
- family,
- speed_mode,
+ identity = getattr(pipe, "_unsloth_peak_identity", None)
+ sized = dict(
width = width,
height = height,
batch_size = batch_size,
dense_transformer_mib = dense_mib,
compute_bytes = compute_bytes,
)
+ # A seeded family keeps the seed and the learned peak the load placed from: a peak this session's renders recorded
+ # after the load only places the NEXT load, it must not stream resident groups for a request the same size as one
+ # that just completed. An unseeded family was placed from its learned peak, which is all there is to check against.
+ sized["phase_seeds"] = bool(getattr(pipe, "_unsloth_phase_seeds", False))
+ need = measured_image_runtime_mib(family, speed_mode, **sized)
+ learned = reserve[5] if len(reserve) > 5 else None
+ if (learned or need is None) and (
+ getattr(pipe, "_unsloth_peak_static", False) or getattr(pipe, "_unsloth_loras", None)
+ ):
+ # the learned peak no longer describes this pipeline (runtime LoRA adapters, a planned VAE saver that did not
+ # engage): requests size from the flat estimate
+ flat = estimate_image_runtime_mib(
+ width = width,
+ height = height,
+ batch_size = batch_size,
+ family = family,
+ condition_pixels = condition_pixels,
+ )
+ return max(0, int(flat) - int(headroom))
+ if learned:
+ # the learned peak the load reserved, scaled here (not reread from the store, which another worker may evict
+ # or a failed write may leave empty while this pipeline stays loaded)
+ from .diffusion_measured_peaks import resolution_bucket
+
+ w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
+ h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
+ batch = max(1, int(batch_size or 1))
+ covered = (
+ reserve[6]
+ if len(reserve) > 6 and reserve[6]
+ else DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT
+ )
+ # the learned store's own scaling: a request past the covered bucket takes its bucket's ceiling, squared over
+ # the image's pixels for a math-attention entry (linear over the batch)
+ scale = 1.0
+ if len(reserve) > 7 and reserve[7]:
+ if w * h > int(covered):
+ scale = (resolution_bucket(w, h, 1) / float(covered)) ** 2
+ scale *= batch
+ elif w * h * batch > int(covered):
+ scale = resolution_bucket(w, h, batch) / float(covered)
+ step = _MEASURED_PEAK_ROUND_MIB
+ scaled = int(-(-(float(learned) * scale * _MEASURED_PEAK_MARGIN) // step) * step)
+ seeded = need is not None
+ need = scaled if need is None else max(need, scaled)
+ if not seeded and isinstance(identity, tuple) and len(identity) == 3:
+ stored = measured_image_runtime_mib(family, speed_mode, quant = identity[1], **sized)
+ if stored is not None:
+ need = max(need, stored)
+ if need is None and isinstance(identity, tuple) and len(identity) == 3:
+ need = measured_image_runtime_mib(family, speed_mode, quant = identity[1], **sized)
if need is None:
return 0
cond = max(0, int(condition_pixels or 0)) * max(1, int(batch_size or 1))
@@ -3085,6 +3530,10 @@ def apply_memory_plan(
the modules would page onto the very card the selection existed to avoid while generation ran
on another. ``device`` stays bare for anything reading it as a policy string."""
placement = placement_device or device
+ try:
+ pipe._unsloth_applied_plan = None
+ except Exception: # noqa: BLE001
+ pass
tiling_engaged = False
if plan.vae_tiling:
tiling_engaged = _enable_vae_saver(pipe, "enable_vae_tiling", "enable_tiling", logger)
@@ -3117,7 +3566,18 @@ def apply_memory_plan(
resident_te_mib = getattr(plan, "resident_text_encoder_mib", None)
if resident_te_mib and group_kwargs["stream_text_encoders"]:
group_kwargs["resident_text_encoder_mib"] = int(resident_te_mib)
- if not _apply_group_offload(pipe, placement, logger, **group_kwargs):
+ _set_group_offload_failure(pipe, None)
+ applied = _apply_group_offload(pipe, placement, logger, **group_kwargs)
+ failure = getattr(pipe, _GROUP_OFFLOAD_FAILURE_ATTR, None)
+ if not applied and failure is not None and _is_oom(failure):
+ lowered = _retry_group_offload_lower(pipe, plan, placement, logger, failure)
+ if lowered is not None:
+ applied = True
+ try:
+ pipe._unsloth_applied_plan = lowered
+ except Exception: # noqa: BLE001
+ pass
+ if not applied:
if "stream_transformer" in group_kwargs and _pipe_denoisers_hold_torchao(pipe):
raise RuntimeError(
"the text encoder could not be streamed beside the resident quantised "
@@ -3161,6 +3621,83 @@ def apply_memory_plan(
return policy, tiling_engaged
+_GROUP_OFFLOAD_FAILURE_ATTR = "_unsloth_group_offload_failure"
+
+
+def _set_group_offload_failure(pipe: Any, exc: Optional[BaseException]) -> None:
+ """What made ``_apply_group_offload`` return False (it absorbs the exception so the caller can fall back)."""
+ try:
+ setattr(pipe, _GROUP_OFFLOAD_FAILURE_ATTR, exc)
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _is_oom(exc: BaseException) -> bool:
+ from .diffusion_batched import is_oom_error
+ return is_oom_error(exc)
+
+
+def applied_plan(pipe: Any, plan: Any) -> Any:
+ """The plan ``apply_memory_plan`` actually placed: ``plan``, or the lower-residency re-plan a transient setup OOM
+ forced (callers gate graphs, fused GEMMs and the encode release on residency)."""
+ return getattr(pipe, "_unsloth_applied_plan", None) or plan
+
+
+def _retry_group_offload_lower(
+ pipe: Any, plan: MemoryPlan, placement: str, logger: Any, exc: BaseException
+) -> Optional[MemoryPlan]:
+ """One retry after an OOM while setting up group offload (another process can move VRAM between the plan's
+ snapshot and placement): return every denoiser and text encoder to the host, empty the cache, and stream the
+ denoisers and encoders with nothing kept resident. Returns the re-plan, or None (the caller refuses as before)."""
+ try:
+ import gc
+
+ import torch
+
+ for name, comp in (getattr(pipe, "components", {}) or {}).items():
+ name = str(name)
+ if not isinstance(comp, torch.nn.Module):
+ continue
+ if name in (
+ "transformer",
+ "transformer_2",
+ "unconditional_transformer",
+ ) or name.startswith("text_encoder"):
+ _return_module_tensors(comp, torch.device("cpu"))
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ if logger is not None:
+ logger.warning(
+ "diffusion.memory: group offload setup hit OOM (%s); emptied the cache and re-planning once with "
+ "the transformer and text encoders streamed",
+ str(exc).splitlines()[0][:200] if str(exc) else type(exc).__name__,
+ )
+ except Exception as retry_exc: # noqa: BLE001 - the caller's refusal stands
+ if logger is not None:
+ logger.warning(
+ "diffusion.memory: lower-residency group offload retry failed (%s)", retry_exc
+ )
+ return None
+ # outside the catch: a raise here left hooks on part of the pipe, which the caller's whole-module fallback rejects
+ if not _apply_group_offload(pipe, placement, logger, stream_text_encoders = True):
+ return None
+ estimates = dict(plan.estimates or {})
+ estimates.update(resident_transformer_mib = 0, resident_text_encoder_mib = 0, setup_oom_retry = True)
+ return replace(
+ plan,
+ stream_text_encoders = True,
+ stream_transformer = True,
+ resident_transformer_mib = None,
+ resident_text_encoder_mib = None,
+ estimates = estimates,
+ reasons = tuple(plan.reasons)
+ + (
+ "group offload setup hit OOM; re-planned once with the transformer and text encoders streamed",
+ ),
+ )
+
+
def _enable_vae_saver(pipe: Any, pipe_method: str, vae_method: str, logger: Any) -> bool:
"""Turn on a VAE memory saver, trying the pipeline shortcut first then the VAE submodule
(some pipelines, e.g. Z-Image, only expose it on ``pipe.vae``). Returns whether it engaged."""
@@ -3399,6 +3936,13 @@ def denoisers_pinned_resident(pipe: Any) -> bool:
return pinned
+def _has_offload_groups(module: Any) -> bool:
+ try:
+ return bool(_offload_groups(module))
+ except Exception: # noqa: BLE001 - an unreadable module counts as unhooked
+ return False
+
+
def _offload_groups(module: Any) -> list:
"""The diffusers offload groups hooked under ``module``, in registration (block) order."""
try:
@@ -3584,10 +4128,14 @@ class _GroupPinner:
def _unpinned(self, group: Any) -> list:
import torch
+
+ # GGUF host copies are GGUFParameter (packed bytes + quant type): pin_memory keeps the subclass, which the
+ # per-onload fallback already relies on. Leaving them unpinned kept every CUDA graph off the GGUF stream.
+ gguf = _gguf_parameter_class()
return [
(tensor, src)
for tensor, src in list(group.cpu_param_dict.items())
- if type(src) is torch.Tensor
+ if (type(src) is torch.Tensor or (gguf is not None and type(src) is gguf))
and src.device.type == "cpu"
and src.numel() > 0
and not src.is_pinned()
@@ -3627,7 +4175,16 @@ class _GroupPinner:
)
for group in self.groups:
copies[id(group)] = [
- (tensor, src, pool.submit(registered_host_copy, src))
+ (
+ tensor,
+ src,
+ pool.submit(
+ registered_host_copy
+ if type(src) is torch.Tensor
+ else src.pin_memory,
+ *((src,) if type(src) is torch.Tensor else ()),
+ ),
+ )
for tensor, src in self._unpinned(group)
]
for group in self.groups:
@@ -4084,6 +4641,9 @@ def _apply_group_offload(
if not stream_transformer and not stream_text_encoders:
return False
installed = 0
+ # every component's torch forward hooks before this call: an OOM after hooks exist unwinds to this state
+ hook_snapshots: list = []
+ grouped_before: set = set()
try:
import inspect
@@ -4092,6 +4652,12 @@ def _apply_group_offload(
install_group_offload_buffer_restore()
install_group_offload_hooks_eager()
+ hook_snapshots = [
+ (comp, _module_forward_hook_ids(comp))
+ for comp in (getattr(pipe, "components", {}) or {}).values()
+ if isinstance(comp, torch.nn.Module)
+ ]
+ grouped_before = {id(comp) for comp, _ in hook_snapshots if _has_offload_groups(comp)}
# A dual-DiT pipeline (Ideogram 4) carries a second denoiser as large as the first, so stream every DiT and keep
# only smaller companions resident.
@@ -4266,6 +4832,16 @@ def _apply_group_offload(
module.to(onload)
return True
except Exception as exc: # noqa: BLE001 - fall back to whole-module offload
+ # apply_group_offloading can raise after hooking part of a module, before ``installed`` counts it
+ if not installed and any(
+ id(comp) not in grouped_before and _has_offload_groups(comp)
+ for comp, _ in hook_snapshots
+ ):
+ installed = 1
+ if installed and _is_oom(exc) and _unwind_group_offload(pipe, hook_snapshots, logger):
+ # An OOM after some hooks exist (a later DiT, an encoder moved resident): every hook is gone again, so
+ # the caller's one lower-residency retry can run instead of the load failing.
+ installed = 0
if installed:
# An earlier streamed module already has hooks but a later one failed: the pipe is in a PARTIAL
# group-offload state enable_model_cpu_offload rejects, so propagate the real failure instead of a
@@ -4278,6 +4854,7 @@ def _apply_group_offload(
exc,
)
raise
+ _set_group_offload_failure(pipe, exc)
if logger is not None:
logger.warning(
"diffusion.memory: group offload failed (%s); falling back to "
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index d762072ec29..9671e50a6ae 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -124,6 +124,7 @@ from .diffusion_memory import (
reclaim_host_memory,
reclaim_offload_host_memory,
refine_memory_plan_for_components,
+ applied_plan,
finish_background_pins,
release_pinned_host_memory,
request_background_pins,
@@ -2675,19 +2676,16 @@ def _calibration_revision(state: Any) -> Optional[str]:
cached = getattr(pipe, "_unsloth_calibration_revision", None)
if cached is not None:
return cached or None
- from .diffusion_component_sizes import checkpoint_fingerprint
-
- revision = checkpoint_fingerprint(
- getattr(state, "repo_id", None), getattr(state, "gguf_filename", None)
+ from .diffusion import _peak_model_identity
+
+ # the image peak identity's rule: snapshot commit, or a local folder's manifests plus its weight files' size and
+ # mtime, plus the companion base for a GGUF / single-file load; "unknown" persists nothing
+ revision = _peak_model_identity(
+ getattr(state, "repo_id", None),
+ getattr(state, "gguf_filename", None),
+ getattr(state, "kind", None),
+ getattr(state, "base_repo", None),
)
- if getattr(state, "kind", "pipeline") != "pipeline":
- # a GGUF / single-file load takes its companions from a base repo: its id and exact weights are part of it
- base = getattr(state, "base_repo", None)
- revision = (
- f"{revision or ''}|base={base or ''}@{checkpoint_fingerprint(base) or ''}"
- if base
- else revision
- )
try:
pipe._unsloth_calibration_revision = revision or ""
except Exception: # noqa: BLE001
@@ -7995,6 +7993,8 @@ class VideoBackend:
placement_device = target.torch_device,
logger = logger,
)
+ # a transient OOM during setup may have re-planned to lower residency
+ plan = applied_plan(pipe, plan)
ltx25_upsampler = getattr(pipe, "_unsloth_ltx25_upsampler", None)
if ltx25_upsampler is not None and offload_policy == OFFLOAD_NONE:
# ~1 GB; resident with the rest, else render() moves it in for the upsample only.
@@ -10836,7 +10836,11 @@ class VideoBackend:
_log_failed_generation(request_shape, exc)
raise
finally:
- i2v_swap.close()
+ try:
+ from . import video_stream_residency
+ video_stream_residency.discard_request(getattr(state, "pipe", None))
+ finally:
+ i2v_swap.close()
if bg_token is not None:
bg_compile._FORCE_EAGER.reset(bg_token)
with self._lock:
diff --git a/studio/backend/core/inference/video_stream_residency.py b/studio/backend/core/inference/video_stream_residency.py
index f754b8989cb..e8353c255cc 100644
--- a/studio/backend/core/inference/video_stream_residency.py
+++ b/studio/backend/core/inference/video_stream_residency.py
@@ -187,10 +187,39 @@ def _covering(peaks: dict, work: int) -> Optional[int]:
return max(covering) if covering else None
+def _store_key(key: Any) -> Any:
+ """``key`` as persisted: the device in it is a process-local ordinal (``cuda:0``), so the GPU's name rides along
+ and workers with different ``CUDA_VISIBLE_DEVICES`` never share one card's peaks with another."""
+ if not isinstance(key, tuple) or len(key) != 5:
+ return key
+ if dict(key[4]).get("revision", "?") in ("", "None", "unknown"):
+ return None # the exact weights are unknown (an unreadable local build): nothing persists for them
+ try:
+ from .diffusion_measured_peaks import device_name
+ name = device_name(key[3]) if key[3] else None
+ except Exception: # noqa: BLE001
+ name = None
+ return (key, name)
+
+
+def _key_peaks(key: Any) -> dict:
+ """This process's peaks for ``key``, merged with the persisted store's (other processes' measurements) at every
+ read; the store's own mtime cache keeps that a stat when nothing changed."""
+ peaks = _PEAKS.setdefault(key, {})
+ try:
+ from .diffusion_measured_peaks import work_peaks
+ for work, extra in work_peaks(_store_key(key)).items():
+ if extra > peaks.get(work, 0):
+ peaks[work] = extra
+ except Exception: # noqa: BLE001 - no store: this process's peaks only
+ pass
+ return peaks
+
+
def measured_extra_mib(key: Any, work: int) -> Optional[int]:
if key is None or not measured_enabled():
return None
- return _covering(_PEAKS.get(key) or {}, int(work))
+ return _covering(_key_peaks(key), int(work))
def _measured_extra_mib(module: Any, work: int) -> Optional[int]:
@@ -199,7 +228,7 @@ def _measured_extra_mib(module: Any, work: int) -> Optional[int]:
peaks it recorded itself; the module's own mode-blind table is the fallback for an untagged one."""
key = getattr(module, "_unsloth_video_calibration_key", None)
if key is not None:
- return _covering(_PEAKS.get(key) or {}, work)
+ return _covering(_key_peaks(key), work)
return _covering(getattr(module, "_unsloth_video_peaks", None) or {}, work)
@@ -227,14 +256,107 @@ def arm_request(
module._unsloth_video_calibration_key = key
if not rearm:
return True
+ dev = _open_window(module, dev)
torch.cuda.reset_peak_memory_stats(dev)
work = int(width) * int(height) * int(frames)
module._unsloth_video_pending = (work, int(torch.cuda.memory_allocated(dev)), dev)
+ module._unsloth_video_failed_captures = _failed_captures()
return True
except Exception: # noqa: BLE001 -- an unarmed request keeps the estimate
return False
+def _open_window(module: Any, dev: Any) -> Any:
+ """Register this request's peak measurement on its card (``diffusion_measured_peaks.open_window``), replacing a
+ stale one; returns the indexed device."""
+ import torch
+
+ from .diffusion_measured_peaks import close_window, open_window
+
+ if dev.index is None:
+ dev = torch.device("cuda", torch.cuda.current_device())
+ stale = getattr(module, "_unsloth_video_window", None)
+ if stale is not None:
+ close_window(stale[0], stale[1])
+ module._unsloth_video_window = (dev, open_window(dev))
+ graphs = _live_graphs(dev)
+ generation = _capture_generation(dev)
+ accounted = getattr(module, "_unsloth_video_graphs", None)
+ if accounted is None or graphs < accounted:
+ # the load's own graphs (every request starts above them), or fewer after a reset freed some
+ first = accounted is None
+ module._unsloth_video_graphs = accounted = graphs
+ if first or graphs <= 0:
+ # nothing alive on the card; a drop to a non-zero count may be another pipeline's reset, so a capture
+ # this one made unrecorded keeps the generation dirty
+ module._unsloth_video_graph_gen = generation
+ if getattr(module, "_unsloth_video_graph_gen", None) is None:
+ module._unsloth_video_graph_gen = generation
+ # graphs a discarded request recorded stay allocated, also one that replaced an evicted graph (same count): a
+ # peak above that start would omit them
+ module._unsloth_video_graphs_clean = (
+ graphs == accounted and generation <= module._unsloth_video_graph_gen
+ )
+ return dev
+
+
+def _capture_generation(dev: Any) -> int:
+ try:
+ from .diffusion_cuda_graph import capture_generation
+ return capture_generation(getattr(dev, "index", None))
+ except Exception: # noqa: BLE001
+ return -1
+
+
+def _live_graphs(dev: Any) -> int:
+ try:
+ from .diffusion_cuda_graph import live_graph_count
+ return live_graph_count(getattr(dev, "index", None))
+ except Exception: # noqa: BLE001
+ return -1
+
+
+def discard_request(pipe: Any) -> None:
+ """Drop a request's armed measurement without recording it (cancel, failure, refusal): its window would otherwise
+ mark every later measurement on the card as overlapped. A no-op after ``record_request_peak``."""
+ module = getattr(pipe, "transformer", None)
+ if module is None:
+ return
+ try:
+ module._unsloth_video_pending = None
+ window = getattr(module, "_unsloth_video_window", None)
+ module._unsloth_video_window = None
+ if window is not None:
+ from .diffusion_measured_peaks import close_window
+ close_window(window[0], window[1])
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _failed_captures() -> int:
+ try:
+ from .diffusion_cuda_graph import failed_capture_count
+ return failed_capture_count()
+ except Exception: # noqa: BLE001
+ return 0
+
+
+def _keyed_graphs_live(pipe: Any, key: Any) -> bool:
+ """False when ``key`` names CUDA graphs that ``live_status`` reports dropped or never engaged for this load."""
+ try:
+ optims = dict(key[4]).get("speed_optims", "")
+ if "cuda_graph" not in str(optims):
+ return True
+ from .diffusion_cuda_graph import live_status
+
+ _, live = live_status(
+ None, ("cuda_graph",), tuple(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())
+ )
+ return "cuda_graph" in live
+ except Exception: # noqa: BLE001 - unreadable: no key names this render
+ return False
+
+
def record_request_peak(pipe: Any, *, logger: Any = None) -> Optional[int]:
"""Record the request's peak allocation above its start; call only after decode and export."""
module = getattr(pipe, "transformer", None)
@@ -247,7 +369,15 @@ def record_request_peak(pipe: Any, *, logger: Any = None) -> Optional[int]:
module._unsloth_video_pending = None
work, start, dev = pending
extra = (int(torch.cuda.max_memory_allocated(dev)) - int(start)) >> 20
- if extra <= 0:
+ window = getattr(module, "_unsloth_video_window", None)
+ module._unsloth_video_window = None
+ if window is not None:
+ from .diffusion_measured_peaks import close_window
+
+ # closed after the read: a render arming in between would otherwise reset the peak unseen
+ if not close_window(window[0], window[1]):
+ return None # another render on the card reset the peak meanwhile
+ if extra <= 0 or not getattr(module, "_unsloth_video_graphs_clean", True):
return None
peaks = getattr(module, "_unsloth_video_peaks", None)
if not isinstance(peaks, dict):
@@ -255,9 +385,42 @@ def record_request_peak(pipe: Any, *, logger: Any = None) -> Optional[int]:
module._unsloth_video_peaks = peaks
peaks[work] = max(extra, peaks.get(work, 0))
key = getattr(module, "_unsloth_video_calibration_key", None)
- if key is not None:
- shared = _PEAKS.setdefault(key, {})
- shared[work] = max(extra, shared.get(work, 0))
+ # a failed capture absorbed an OOM: this peak is that attempt's, keep it out of the keyed and persisted peaks
+ # a load keyed with CUDA graphs whose graphs no longer run (a poisoned wrapper after a failed capture runs
+ # eager without counting a new failure) records an eager peak a graph-capturing load would undershoot
+ if (
+ key is not None
+ and _failed_captures() == getattr(module, "_unsloth_video_failed_captures", 0)
+ and _keyed_graphs_live(pipe, key)
+ ):
+ graphs, generation = _live_graphs(dev), _capture_generation(dev)
+ stored_key = _store_key(key)
+ try:
+ from .diffusion_memory import stream_window_pad_mib
+
+ # on the streamed baseline, as image peaks: a later request of the key may stream groups this request
+ # kept resident; an unknown placement enters neither the keyed table nor the store
+ pad = stream_window_pad_mib(pipe)
+ except Exception: # noqa: BLE001
+ pad = None
+ held: list = []
+ if pad is not None:
+ shared = _key_peaks(key)
+ shared[work] = max(extra + pad, shared.get(work, 0))
+ # an unpersistable key keeps this process's table only, where max wins: nothing lower can be stored
+ # later
+ if stored_key is None:
+ held.append(True)
+ else:
+ try:
+ from .diffusion_measured_peaks import record_work
+ record_work(stored_key, work, extra + pad, held = held)
+ except Exception: # noqa: BLE001 - persistence is best-effort
+ pass
+ if held:
+ # the store holds this sample: it covers the graphs the request recorded; unpersisted, they stay dirty
+ module._unsloth_video_graphs = graphs
+ module._unsloth_video_graph_gen = generation
_COUNTS["recorded"] += 1
if logger is not None:
logger.info("video.dit_resident: request peak %d MiB above its start recorded", extra)
@@ -332,8 +495,10 @@ def fit_for_request(
_keep_groups_resident(module, room, dev, logger)
promoted = max(0, resident_mib(module) - after_release)
kept = resident_mib(module)
+ dev = _open_window(module, dev)
torch.cuda.reset_peak_memory_stats(dev)
module._unsloth_video_pending = (work, int(torch.cuda.memory_allocated(dev)), dev)
+ module._unsloth_video_failed_captures = _failed_captures()
try:
# release_resident_groups' restore reads it back; keep it at this request's room
module._unsloth_resident_room = room if room > 0 else None
diff --git a/studio/backend/tests/test_diffusion_learned_peaks.py b/studio/backend/tests/test_diffusion_learned_peaks.py
new file mode 100644
index 00000000000..8e27bc2d792
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_learned_peaks.py
@@ -0,0 +1,2162 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Learned activation peaks (``diffusion_measured_peaks``), the per-phase seed for families the peak table lacks,
+and the one lower-residency retry after a transient OOM during group-offload setup (``diffusion_memory``).
+
+The FLUX.2-klein-4B case uses the estimates a real auto load logged on a 16 GB card (int8 DiT, bf16 text encoder):
+``safe_device_budget_mib 13132``, group offload with the text encoders streamed, and the whole DiT streamed although
+the load peaked at 4.6 GiB. CPU-only; loaded sizes and the torchao check are stubbed.
+"""
+
+from __future__ import annotations
+
+import json
+import types
+
+import pytest
+
+import core.inference.diffusion_measured_peaks as mp
+import core.inference.diffusion_memory as dm
+import core.inference.video_stream_residency as vsr
+
+KLEIN_LOADED = {
+ "transformer": (4091, "dit"),
+ "text_encoder": (7629, "text_encoder"),
+ "vae": (160, "other"),
+}
+KLEIN_FLAT = dict(model_dense_mib = 11911, companion_dense_mib = 7820, text_encoder_dense_mib = 7629)
+
+
+def _memory(budget_mib: int, total_mib: int) -> "dm.DeviceMemory":
+ reserve = dm._reserve_mib("discrete_vram", total_mib)
+ return dm.DeviceMemory("cuda", "cuda", "discrete_vram", budget_mib + reserve, total_mib)
+
+
+def _klein_flat_plan(budget_mib: int = 13132, total_mib: int = 16380):
+ plan = dm.plan_diffusion_memory(
+ target = types.SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = _memory(budget_mib, total_mib),
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(
+ width = None, height = None, family = "flux.2-klein"
+ ),
+ requested_mode = None,
+ **KLEIN_FLAT,
+ )
+ if plan.offload_policy == dm.OFFLOAD_MODEL:
+ plan = dm.torchao_streaming_plan(plan)
+ return plan
+
+
+def store(tmp_path, monkeypatch):
+ path = tmp_path / "peaks.json"
+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(path))
+ monkeypatch.delenv(mp.LEARNED_PEAKS_ENV, raising = False)
+ monkeypatch.delenv(dm.MEASURED_ACTIVATION_ENV, raising = False)
+ monkeypatch.delenv(dm.PARTIAL_RESIDENT_ENV, raising = False)
+ mp._cache.update(path = None, mtime = None, data = None)
+ monkeypatch.setattr(
+ mp, "current_device_name", lambda: "NVIDIA L4", raising = False
+ ) # the planner's card
+ return path
+
+
+def klein_pipe(monkeypatch, store):
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(KLEIN_LOADED))
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ monkeypatch.setattr(dm, "denoiser_quant_label", lambda pipe: "torchao:Int8Tensor")
+ return types.SimpleNamespace(_unsloth_phase_seeds = True)
+
+
+def _refine(
+ pipe,
+ plan,
+ family = "flux.2-klein",
+ speed = "default",
+):
+ return dm.refine_plan_from_loaded_weights(pipe, plan, family = family, speed_mode = speed)
+
+
+# ---------------------------------------------------------------------------------------------------- seeds
+
+
+def test_family_missing_from_peak_table_is_seeded_from_phase_calibration(store):
+ assert "flux.2-klein" not in dm._MEASURED_IMAGE_PEAK_MIB
+ # worst of encoder 1516 / denoise 1160 / untiled decode 2645 x 1.15, rounded up to 256 MiB
+ assert dm.measured_image_runtime_mib("flux.2-klein", "default", phase_seeds = True) == 3072
+ assert dm.measured_image_runtime_mib("flux.2-klein", "max", phase_seeds = True) == 3328
+ # eager tiers and unseeded families still keep the flat estimate
+ assert dm.measured_image_runtime_mib("flux.2-klein", "off") is None
+ assert dm.measured_image_runtime_mib("hidream", "default") is None
+ # the peak table still wins where it has the family (Qwen-Image-2.1 decode is tiled: 1849, not 7648)
+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "default") == 2304
+
+
+def test_klein_16gb_keeps_whole_dit_resident(klein_pipe):
+ plan = _klein_flat_plan()
+ assert (
+ plan.offload_policy == dm.OFFLOAD_GROUP
+ and plan.stream_text_encoders
+ and plan.stream_transformer
+ )
+ assert plan.resident_transformer_mib is None
+ new = _refine(klein_pipe, plan)
+ assert new is not plan
+ assert new.resident_transformer_mib == 4091 # every DiT group resident
+ assert new.estimates["measured_runtime_headroom_mib"] == 3072
+ assert klein_pipe._unsloth_peak_identity == ("flux.2-klein", "torchao:Int8Tensor", "default")
+
+
+# ---------------------------------------------------------------------------------------------------- store
+
+
+def test_bucket_is_power_of_two_megapixels():
+ assert mp.resolution_bucket(256, 256) == mp._MIN_BUCKET_PIXELS
+ assert mp.resolution_bucket(1024, 1024) == 1024 * 1024
+ assert mp.resolution_bucket(1280, 720) == 1024 * 1024
+ assert mp.resolution_bucket(1024, 1024, 2) == 2 * 1024 * 1024
+ assert mp.resolution_bucket(1536, 1536) == 4 * 1024 * 1024
+
+
+def test_record_keeps_max_and_survives_a_fresh_process(store):
+ kw = dict(family = "flux.2-klein", quant = "torchao:Int8Tensor", speed_mode = "default")
+ assert mp.lookup(**kw) is None
+ assert mp.record(**kw, width = 1024, height = 1024, peak_mib = 2000, device_name = "NVIDIA L4")
+ assert not mp.record(**kw, width = 1024, height = 1024, peak_mib = 1500) # never lowered
+ assert mp.record(**kw, width = 1024, height = 1024, peak_mib = 2600)
+ mp._cache.update(path = None, mtime = None, data = None) # what a new process sees
+ data = json.loads(store.read_text())
+ assert data["entries"]["flux.2-klein|torchao:Int8Tensor|default|f1|NVIDIA L4"]["peaks"] == {
+ "1048576": 2600
+ }
+ assert mp.lookup(**kw, width = 1024, height = 1024) == 2600
+ # a smaller request answers from the covering bucket, a larger one scales the largest linearly
+ assert mp.lookup(**kw, width = 512, height = 512) == 2600
+ assert mp.lookup(**kw, width = 2048, height = 2048) == 4 * 2600
+ # other quant / tier / frame count: separate keys
+ assert mp.lookup(family = "flux.2-klein", quant = "dense:bfloat16", speed_mode = "default") is None
+ assert mp.lookup(**{**kw, "speed_mode": "max"}) is None
+ assert mp.lookup(**kw, frames = 49) is None
+
+
+def test_store_off_and_corrupt_file(store, monkeypatch):
+ kw = dict(family = "z-image", quant = "q", speed_mode = "default")
+ store.write_text("{not json")
+ assert mp.lookup(**kw) is None
+ assert mp.record(**kw, width = 1024, height = 1024, peak_mib = 1000)
+ assert mp.lookup(**kw) == 1000
+ monkeypatch.setenv(mp.LEARNED_PEAKS_ENV, "0")
+ assert mp.lookup(**kw) is None
+ assert not mp.record(**kw, width = 1024, height = 1024, peak_mib = 5000)
+
+
+def test_learned_peak_raises_a_seed_but_never_lowers_it(store):
+ seed = dm.measured_image_runtime_mib(
+ "flux.2-klein", "default", phase_seeds = True, quant = "torchao:Int8Tensor"
+ )
+ assert seed == 3072
+ kw = dict(family = "flux.2-klein", quant = "torchao:Int8Tensor", speed_mode = "default")
+ mp.record(**kw, width = 1024, height = 1024, peak_mib = 1000)
+ assert (
+ dm.measured_image_runtime_mib(
+ "flux.2-klein", "default", phase_seeds = True, quant = "torchao:Int8Tensor"
+ )
+ == 3072
+ )
+ mp.record(**kw, width = 1024, height = 1024, peak_mib = 4000)
+ # 4000 x 1.15 = 4600 -> 4608
+ assert (
+ dm.measured_image_runtime_mib(
+ "flux.2-klein", "default", phase_seeds = True, quant = "torchao:Int8Tensor"
+ )
+ == 4608
+ )
+ # without a quant label (callers that never measured) the seed alone answers
+ assert dm.measured_image_runtime_mib("flux.2-klein", "default", phase_seeds = True) == 3072
+
+
+def test_unseeded_family_is_placed_from_its_learned_peak_on_the_next_load(klein_pipe):
+ plan = _klein_flat_plan()
+ assert (
+ _refine(klein_pipe, plan, family = "some-new-dit") is plan
+ ) # nothing measured yet: flat plan
+ mp.record(
+ family = "some-new-dit",
+ quant = "torchao:Int8Tensor",
+ speed_mode = "default",
+ width = 1024,
+ height = 1024,
+ peak_mib = 2600,
+ )
+ new = _refine(klein_pipe, plan, family = "some-new-dit")
+ assert new.resident_transformer_mib == 4091
+ assert new.estimates["measured_runtime_headroom_mib"] == 3072 # 2600 x 1.15 -> 3072
+
+
+def test_request_guard_reads_the_learned_peak(klein_pipe):
+ plan = _klein_flat_plan()
+ new = _refine(klein_pipe, plan)
+ klein_pipe._unsloth_measured_reserve = (3072, "flux.2-klein", "default", None, None)
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0
+ mp.record(
+ family = "flux.2-klein",
+ quant = "torchao:Int8Tensor",
+ speed_mode = "default",
+ width = 1024,
+ height = 1024,
+ peak_mib = 4000,
+ )
+ assert new is not plan
+ # seeded family: a peak recorded by this session's own render places the next load, never this one's requests
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0
+ # 2048x2048: 4 x 2645 x 1.15 = 12167 -> 12288
+ assert dm.measured_request_extra_mib(klein_pipe, width = 2048, height = 2048) == 12288 - 3072
+
+
+def test_request_guard_of_an_unseeded_family_reads_its_learned_peak(klein_pipe):
+ kw = dict(family = "some-new-dit", quant = "torchao:Int8Tensor", speed_mode = "default")
+ mp.record(**kw, width = 1024, height = 1024, peak_mib = 2600)
+ new = _refine(klein_pipe, _klein_flat_plan(), family = "some-new-dit")
+ assert new.estimates["measured_runtime_headroom_mib"] == 3072
+ klein_pipe._unsloth_measured_reserve = (3072, "some-new-dit", "default", None, None)
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0
+ mp.record(**kw, width = 1024, height = 1024, peak_mib = 4000)
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 4608 - 3072
+
+
+# ---------------------------------------------------------------------------------------------------- video
+
+
+def test_video_request_peaks_persist_across_processes(store, monkeypatch):
+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")
+ work = 1280 * 704 * 49
+ assert vsr.measured_extra_mib(key, work) is None
+
+ class _Torch:
+ class cuda: # noqa: N801
+ @staticmethod
+ def max_memory_allocated(dev):
+ return (3000 + 100) << 20
+
+ module = types.SimpleNamespace(_unsloth_video_pending = (work, 100 << 20, "cuda:0"))
+ module._unsloth_video_calibration_key = key
+ monkeypatch.setitem(__import__("sys").modules, "torch", _Torch)
+ assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) == 3000
+ monkeypatch.undo()
+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(store))
+ monkeypatch.setattr(vsr, "_PEAKS", {}) # a new process
+ mp._cache.update(path = None, mtime = None, data = None)
+ assert vsr.measured_extra_mib(key, work) == 3000
+ assert vsr.measured_extra_mib(key, work + 1) is None # a larger request is not covered
+
+
+# ---------------------------------------------------------------------------------------------------- OOM retry
+
+
+class _OOM(RuntimeError):
+ pass
+
+
+_OOM.__name__ = "OutOfMemoryError"
+
+
+def _resident_dit_plan():
+ mem = _memory(13132, 16380)
+ return dm.MemoryPlan(
+ requested_mode = "auto",
+ offload_policy = dm.OFFLOAD_GROUP,
+ vae_tiling = False,
+ vae_slicing = True,
+ device_memory = mem,
+ estimates = {"safe_device_budget_mib": 13132},
+ reasons = ("flat",),
+ stream_text_encoders = True,
+ stream_transformer = False,
+ )
+
+
+def test_transient_setup_oom_replans_once_at_lower_residency(monkeypatch):
+ calls = []
+
+ def fake_apply(pipe, device, logger, **kw):
+ calls.append(kw)
+ if len(calls) == 1:
+ dm._set_group_offload_failure(
+ pipe, _OOM("CUDA out of memory. Tried to allocate 20.00 MiB")
+ )
+ return False
+ return True
+
+ monkeypatch.setattr(dm, "_apply_group_offload", fake_apply)
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ pipe = types.SimpleNamespace(components = {})
+ plan = _resident_dit_plan()
+ policy, _ = dm.apply_memory_plan(pipe, plan, device = "cuda")
+ assert policy == dm.OFFLOAD_GROUP
+ assert calls[0] == {"stream_text_encoders": True, "stream_transformer": False}
+ assert calls[1] == {"stream_text_encoders": True} # the DiT streams on the retry
+ applied = dm.applied_plan(pipe, plan)
+ assert applied is not plan and applied.stream_transformer and applied.stream_text_encoders
+ assert applied.resident_transformer_mib is None and applied.estimates["setup_oom_retry"]
+ assert not dm.plan_keeps_transformer_resident(applied)
+
+
+def test_non_oom_setup_failure_still_refuses(monkeypatch):
+ def fake_apply(pipe, device, logger, **kw):
+ dm._set_group_offload_failure(pipe, ValueError("no block list"))
+ return False
+
+ monkeypatch.setattr(dm, "_apply_group_offload", fake_apply)
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ pipe = types.SimpleNamespace(components = {})
+ with pytest.raises(RuntimeError, match = "could not be streamed beside the resident quantised"):
+ dm.apply_memory_plan(pipe, _resident_dit_plan(), device = "cuda")
+ assert dm.applied_plan(pipe, "plan") == "plan"
+
+
+def test_oom_retry_that_fails_again_refuses(monkeypatch):
+ def fake_apply(pipe, device, logger, **kw):
+ dm._set_group_offload_failure(pipe, _OOM("out of memory"))
+ return False
+
+ monkeypatch.setattr(dm, "_apply_group_offload", fake_apply)
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ pipe = types.SimpleNamespace(components = {})
+ with pytest.raises(RuntimeError, match = "could not be streamed beside the resident quantised"):
+ dm.apply_memory_plan(pipe, _resident_dit_plan(), device = "cuda")
+
+
+# ---------------------------------------------------------------------------------------------------- GGUF pinning
+
+
+def test_background_pinner_pins_gguf_host_copies():
+ """GGUF host copies were skipped (``type(src) is torch.Tensor``), so a streamed GGUF denoiser never had a pinned
+ host copy and its CUDA graph refused capture ("a streamed group's host copy is not pinned")."""
+ torch = pytest.importorskip("torch")
+ gguf_utils = pytest.importorskip("diffusers.quantizers.gguf.utils")
+ gguf = gguf_utils.GGUFParameter(torch.zeros(4, 144, dtype = torch.uint8), quant_type = 12)
+ dense = torch.zeros(8)
+ group = types.SimpleNamespace(cpu_param_dict = {"w": gguf.data, "b": dense})
+ pinner = dm._GroupPinner.__new__(dm._GroupPinner)
+ picked = {key for key, _ in pinner._unpinned(group)}
+ assert picked == {"w", "b"}
+
+
+# ---------------------------------------------------------------------------------------------------- recording
+
+
[email protected](autouse = True)
+def _attention_scaling_known(request, monkeypatch):
+ """Recording tests run without a device target; the SDPA verdict counts as known unless a test says otherwise."""
+ if "attention_scaling_unknown" not in request.node.name:
+ monkeypatch.setattr(
+ "core.inference.diffusion._attention_scaling_known", lambda target, engaged = None: True
+ )
+
+
+class _Probe:
+ def __init__(
+ self,
+ peak,
+ failed = 0,
+ ):
+ self.device = "cuda:0"
+ self.failed_captures = failed
+ self._peak = peak
+
+ def finish(self):
+ return self._peak
+
+
+def test_render_records_its_peak_unless_a_capture_failed(store, monkeypatch):
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")
+ pipe = types.SimpleNamespace(
+ _unsloth_peak_identity = ("flux.2-klein", "torchao:Int8Tensor", "default")
+ )
+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default")
+ kw = dict(family = "flux.2-klein", quant = "torchao:Int8Tensor", speed_mode = "default")
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 1)
+ diffusion._record_peak_probe(
+ state, _Probe(5000, failed = 0), width = 1024, height = 1024, batch_size = 1
+ )
+ assert mp.lookup(**kw) is None # an absorbed capture OOM: the peak is the failed attempt's
+ diffusion._record_peak_probe(
+ state, _Probe(2400, failed = 1), width = 1024, height = 1024, batch_size = 1
+ )
+ assert mp.lookup(**kw) == 2400
+ assert (
+ json.loads(store.read_text())["entries"][
+ "flux.2-klein|torchao:Int8Tensor|default|f1|NVIDIA L4"
+ ]["device"]
+ == "NVIDIA L4"
+ )
+
+
+# ---------------------------------------------------------------------------------------------------- tied weights
+
+
+def test_tied_group_closures():
+ shared, a, b, c = object(), object(), object(), object()
+ g1 = types.SimpleNamespace(t = [shared, a])
+ g2 = types.SimpleNamespace(t = [b])
+ g3 = types.SimpleNamespace(t = [c, shared])
+ closures = dm._tied_group_closures([g1, g2, g3], lambda g: g.t)
+ assert closures[id(g1)] == [g1, g3] and closures[id(g3)] == [g1, g3]
+ assert id(g2) not in closures
+
+
+def _tied_tiny_encoder(torch):
+ """A Qwen3-style leaf: an embedding and an LM head sharing one weight (FLUX.2-klein's text encoder ties them)."""
+
+ class Tiny(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embed = torch.nn.Embedding(4096, 128)
+ self.mid = torch.nn.Linear(128, 128)
+ self.head = torch.nn.Linear(128, 4096, bias = False)
+ self.head.weight = self.embed.weight
+
+ def forward(self, ids):
+ return self.head(self.mid(self.embed(ids)))
+
+ return Tiny().to(torch.bfloat16)
+
+
+def test_partial_residency_keeps_tied_weights_together():
+ """Base: the embedding group went resident while the tied head streamed; the head's offload pointed the shared
+ weight back at the host and the next encode failed (index on cuda, weight on cpu)."""
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA and diffusers group offloading")
+ from diffusers.hooks import apply_group_offloading
+
+ te = _tied_tiny_encoder(torch)
+ dev = torch.device("cuda")
+ apply_group_offloading(
+ te,
+ onload_device = dev,
+ offload_device = torch.device("cpu"),
+ offload_type = "leaf_level",
+ use_stream = True,
+ non_blocking = True,
+ record_stream = True,
+ )
+ # 1 MiB: room for the 1 MiB shared weight once, not for the embedding and the head counted apart
+ dm._keep_groups_resident(te, 1, dev)
+ groups = dm._offload_groups(te)
+ tied = [g for g in groups if te.embed in g.modules or te.head in g.modules]
+ assert len(tied) == 2
+ assert len({bool(getattr(g, "_unsloth_resident", False)) for g in tied}) == 1
+ for n in (7, 11, 5):
+ with torch.no_grad():
+ te(torch.randint(0, 4096, (1, n), device = dev))
+ restore = dm.release_resident_groups(types.SimpleNamespace(components = {"text_encoder": te}), 1)
+ assert not any(getattr(g, "_unsloth_resident", False) for g in tied)
+ with torch.no_grad():
+ te(torch.randint(0, 4096, (1, 9), device = dev))
+ restore()
+ assert all(getattr(g, "_unsloth_resident", False) for g in tied)
+ with torch.no_grad():
+ te(torch.randint(0, 4096, (1, 4), device = dev))
+
+
+def test_video_peak_of_a_request_with_a_failed_capture_is_not_persisted(store, monkeypatch):
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "streaming", "cuda:0")
+ work = 832 * 480 * 33
+
+ class _Torch:
+ class cuda: # noqa: N801
+ @staticmethod
+ def max_memory_allocated(dev):
+ return 9000 << 20
+
+ module = types.SimpleNamespace(
+ _unsloth_video_pending = (work, 0, "cuda:0"),
+ _unsloth_video_calibration_key = key,
+ _unsloth_video_failed_captures = 0,
+ )
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 1)
+ monkeypatch.setitem(__import__("sys").modules, "torch", _Torch)
+ assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) == 9000
+ assert mp.work_peaks(key) == {} # kept in this process only
+
+
+def test_resident_dit_plan_clamps_encoder_room_and_says_so(klein_pipe):
+ """L4 (22.5 GB): the flat plan already kept the DiT resident; the encoder room exceeded the encoders and the log
+ read '0 MiB of the transformer and 10970 MiB of the 7673 MiB encoders'."""
+ plan = _klein_flat_plan(budget_mib = 19500, total_mib = 23034)
+ assert (
+ plan.offload_policy == dm.OFFLOAD_GROUP
+ and not plan.stream_transformer
+ and plan.stream_text_encoders
+ )
+ new = _refine(klein_pipe, plan)
+ assert new.resident_text_encoder_mib == 7629
+ assert "transformer is resident" in new.reasons[-1]
+
+
+# ---------------------------------------------------------------------------------------------------- review follow-ups
+
+
+def test_render_served_by_a_conditioning_cache_is_not_recorded(store, monkeypatch):
+ """A cache hit skips the text encoder: an unseeded family would place its next load from the smaller peak."""
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")
+ stats = {"hits": 0, "misses": 1}
+ pipe = types.SimpleNamespace(
+ _unsloth_peak_identity = ("some-new-dit", "torchao:Int8Tensor", "default"),
+ _unsloth_cond_cache_stats = stats,
+ )
+ state = types.SimpleNamespace(pipe = pipe, device = "cuda:0", speed_mode = "default")
+ probe = _Probe(1800)
+ probe.conditioning_hits = diffusion._conditioning_cache_hits(pipe)
+ stats["hits"] += 1 # this render's prompt came from the cache
+ diffusion._record_peak_probe(state, probe, width = 1024, height = 1024, batch_size = 1)
+ kw = dict(family = "some-new-dit", quant = "torchao:Int8Tensor", speed_mode = "default")
+ assert mp.lookup(**kw) is None
+ probe = _Probe(2600)
+ probe.conditioning_hits = diffusion._conditioning_cache_hits(pipe)
+ diffusion._record_peak_probe(state, probe, width = 1024, height = 1024, batch_size = 1)
+ assert mp.lookup(**kw) == 2600
+
+
+def test_learned_peaks_are_kept_per_gpu_model(store):
+ kw = dict(
+ family = "some-new-dit",
+ quant = "torchao:Int8Tensor",
+ speed_mode = "default",
+ width = 1024,
+ height = 1024,
+ )
+ mp.record(**kw, peak_mib = 2600, device_name = "NVIDIA L4")
+ assert mp.lookup(**kw, device = "NVIDIA L4") == 2600
+ assert mp.lookup(**kw, device = "NVIDIA GeForce RTX 4090") is None
+
+
+def test_peak_store_update_waits_for_another_process_lock(store):
+ """The read / merge / replace takes an interprocess lock, so a concurrent writer cannot replace a higher peak."""
+ fcntl = pytest.importorskip("fcntl")
+ import threading
+
+ store.parent.mkdir(parents = True, exist_ok = True)
+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default", width = 1024, height = 1024)
+ done = threading.Event()
+ with open(
+ str(store) + ".lock", "a+b"
+ ) as other: # a separate open file description, like another process
+ fcntl.flock(other.fileno(), fcntl.LOCK_EX)
+ worker = threading.Thread(
+ target = lambda: (mp.record(**kw, peak_mib = 2000, device_name = "g"), done.set())
+ )
+ worker.start()
+ assert not done.wait(0.5) # blocked behind the other writer's lock
+ fcntl.flock(other.fileno(), fcntl.LOCK_UN)
+ worker.join(10)
+ assert done.is_set() and mp.lookup(**kw, device = "g") == 2000
+
+
+def test_oom_after_partial_hooks_unwinds_and_replans(monkeypatch):
+ """An OOM on the second DiT after the first had hooks used to re-raise past the lower-residency retry."""
+ torch = pytest.importorskip("torch")
+ hooks = pytest.importorskip("diffusers.hooks")
+ from diffusers.hooks.hooks import HookRegistry, ModelHook
+
+ calls = []
+ marker = "_test_group_marker"
+
+ def fake_apply(module, **kw):
+ calls.append(module)
+ if module is pipe.transformer_2 and calls.count(module) == 1:
+ raise _OOM("CUDA out of memory. Tried to allocate 64.00 MiB")
+ HookRegistry.check_if_exists_or_initialize(module).register_hook(ModelHook(), marker)
+
+ monkeypatch.setattr(hooks, "apply_group_offloading", fake_apply)
+ monkeypatch.setattr(dm, "install_group_offload_buffer_restore", lambda: None)
+ monkeypatch.setattr(dm, "install_group_offload_hooks_eager", lambda: None)
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ real_remove = dm._remove_group_offload_hooks
+
+ def remove(module):
+ real_remove(module)
+ registry = getattr(module, "_diffusers_hook", None)
+ if registry is not None and registry.get_hook(marker) is not None:
+ registry.remove_hook(marker, recurse = True)
+
+ monkeypatch.setattr(dm, "_remove_group_offload_hooks", remove)
+ a, b, te = torch.nn.Linear(4, 4), torch.nn.Linear(4, 4), torch.nn.Linear(4, 4)
+ pipe = types.SimpleNamespace(
+ transformer = a,
+ transformer_2 = b,
+ components = {"transformer": a, "transformer_2": b, "text_encoder": te},
+ )
+ plan = dm.replace(_resident_dit_plan(), stream_transformer = True, stream_text_encoders = False)
+ policy, _ = dm.apply_memory_plan(pipe, plan, device = "cpu")
+ assert policy == dm.OFFLOAD_GROUP
+ assert dm.applied_plan(pipe, plan).estimates["setup_oom_retry"]
+ assert calls[:2] == [a, b] and calls[2:4] == [a, b] # unwound, then both DiTs streamed again
+
+
+# ---------------------------------------------------------------------------------------------------- review round 2
+
+
+def test_peak_is_scaled_to_its_bucket_ceiling(store):
+ kw = dict(family = "some-new-dit", quant = "torchao:Int8Tensor", speed_mode = "default")
+ # 1088x1024 lands in the 2 MP bucket; a 2 MP request must not get the ~1 MP peak back unchanged
+ mp.record(**kw, width = 1088, height = 1024, peak_mib = 1000)
+ assert mp.lookup(**kw, width = 1448, height = 1448) >= 1000 * 2 * 1024 * 1024 // (1088 * 1024)
+
+
+def test_request_guard_scales_the_learned_peak_the_load_placed_from(klein_pipe):
+ mp.record(
+ family = "flux.2-klein",
+ quant = "torchao:Int8Tensor",
+ speed_mode = "default",
+ width = 1024,
+ height = 1024,
+ peak_mib = 4000,
+ )
+ new = _refine(klein_pipe, _klein_flat_plan())
+ assert new.estimates["measured_runtime_headroom_mib"] == 4608
+ klein_pipe._unsloth_peak_identity = ("flux.2-klein", "torchao:Int8Tensor", "default")
+ klein_pipe._unsloth_measured_reserve = (
+ 4608,
+ "flux.2-klein",
+ "default",
+ None,
+ None,
+ new.estimates.get("measured_learned_peak_mib"),
+ )
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0
+ # 2 MP: 2 x 4000 x 1.15 = 9200 -> 9216, not the 2 x 2645 seed
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1448, height = 1448) >= 9216 - 4608
+
+
+def test_learned_peaks_are_keyed_by_attention_backend(klein_pipe):
+ mp.record(
+ family = "some-new-dit",
+ quant = "torchao:Int8Tensor|attn:flash",
+ speed_mode = "default",
+ width = 1024,
+ height = 1024,
+ peak_mib = 2600,
+ )
+ plan = _klein_flat_plan()
+ flash = dm.refine_plan_from_loaded_weights(
+ klein_pipe, plan, family = "some-new-dit", speed_mode = "default", attention = "flash"
+ )
+ assert flash.estimates["measured_runtime_headroom_mib"] == 3072
+ assert klein_pipe._unsloth_peak_identity[1] == "torchao:Int8Tensor|attn:flash"
+ native = dm.refine_plan_from_loaded_weights(
+ klein_pipe, plan, family = "some-new-dit", speed_mode = "default", attention = "native"
+ )
+ assert native is plan # a native-attention load never reuses the flash peak
+
+
+def test_load_passes_the_engaged_attention_backend_to_the_peak_identity():
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert (
+ "peak_model,\n peak_vae,\n peak_te,\n peak_cache,"
+ in src
+ )
+ assert 'attention = attention_engaged or "native",' in src
+ assert 'plan.estimates.get("measured_learned_peak_mib"),' in src
+
+
+def test_video_peaks_merge_other_processes_updates(store, monkeypatch):
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")
+ work = 1280 * 704 * 49
+ assert vsr.measured_extra_mib(key, work) is None # this process caches the empty answer
+ assert mp.record_work(vsr._store_key(key), work, 3000) # another process records
+ assert vsr.measured_extra_mib(key, work) == 3000
+
+
+def test_video_peaks_are_persisted_per_physical_gpu(store, monkeypatch):
+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")
+ work = 1280 * 704 * 49
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")
+ module = types.SimpleNamespace(_unsloth_video_pending = None, _unsloth_video_calibration_key = key)
+ vsr._PEAKS[key] = {work: 3000}
+ module._unsloth_video_pending = (work, 0, "cuda:0")
+
+ class _Torch:
+ class cuda: # noqa: N801
+ @staticmethod
+ def max_memory_allocated(dev):
+ return 3000 << 20
+
+ with monkeypatch.context() as m:
+ m.setitem(__import__("sys").modules, "torch", _Torch)
+ vsr.record_request_peak(types.SimpleNamespace(transformer = module))
+ stored = json.loads(store.read_text())["work"]
+ assert list(stored) == [repr((key, "NVIDIA L4"))]
+ vsr._PEAKS.clear()
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA H100") # another worker's cuda:0
+ assert vsr.measured_extra_mib(key, work) is None
+
+
+def test_store_cap_evicts_the_oldest_entry_not_the_alphabetically_first(store, monkeypatch):
+ monkeypatch.setattr(mp, "_MAX_ENTRIES", 3)
+ for fam in ("m", "n", "o"):
+ mp.record(
+ family = fam, quant = "q", speed_mode = "default", width = 1024, height = 1024, peak_mib = 100
+ )
+ mp.record(family = "a", quant = "q", speed_mode = "default", width = 1024, height = 1024, peak_mib = 100)
+ entries = json.loads(store.read_text())["entries"]
+ assert [k.split("|")[0] for k in entries] == ["n", "o", "a"]
+ for fam in ("m", "n", "o"):
+ assert mp.record_work(("k", fam), 10, 100)
+ assert mp.record_work(("k", "a"), 10, 100)
+ assert list(json.loads(store.read_text())["work"]) == [repr(("k", f)) for f in ("n", "o", "a")]
+
+
+# ---------------------------------------------------------------------------------------------------- review round 3
+
+
+def test_learned_peaks_are_keyed_by_the_loaded_model(klein_pipe):
+ mp.record(
+ family = "some-new-dit",
+ quant = "torchao:Int8Tensor|model:org/dit-4b|attn:native",
+ speed_mode = "default",
+ width = 1024,
+ height = 1024,
+ peak_mib = 2600,
+ )
+ plan = _klein_flat_plan()
+ kw = dict(family = "some-new-dit", speed_mode = "default", attention = "native")
+ small = dm.refine_plan_from_loaded_weights(klein_pipe, plan, model = "org/dit-4b", **kw)
+ assert small.estimates["measured_runtime_headroom_mib"] == 3072
+ assert klein_pipe._unsloth_peak_identity[1] == "torchao:Int8Tensor|model:org/dit-4b|attn:native"
+ # a larger variant of the same family never reuses the smaller one's peak
+ assert dm.refine_plan_from_loaded_weights(klein_pipe, plan, model = "org/dit-9b", **kw) is plan
+
+
+def test_load_passes_the_model_to_the_peak_identity():
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert (
+ "peak_model,\n peak_vae,\n peak_te,\n peak_cache,"
+ in src
+ )
+ assert "model = peak_model," in src
+ assert 'plan.estimates.get("measured_learned_peak_pixels"),' in src
+
+
+def test_request_guard_keeps_the_learned_peaks_source_bucket(klein_pipe):
+ mp.record(
+ family = "flux.2-klein",
+ quant = "torchao:Int8Tensor",
+ speed_mode = "default",
+ width = 1448,
+ height = 1448,
+ peak_mib = 8000,
+ )
+ new = _refine(klein_pipe, _klein_flat_plan())
+ learned = new.estimates["measured_learned_peak_mib"]
+ assert learned >= 8000
+ headroom = new.estimates["measured_runtime_headroom_mib"]
+ klein_pipe._unsloth_peak_identity = ("flux.2-klein", "torchao:Int8Tensor", "default")
+ klein_pipe._unsloth_measured_reserve = (
+ headroom,
+ "flux.2-klein",
+ "default",
+ None,
+ None,
+ learned,
+ new.estimates.get("measured_learned_peak_pixels"),
+ )
+ # the stored peak already covers 2 MP: a 2 MP request needs nothing beyond the load's reserve
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1448, height = 1448) == 0
+
+
+# ---------------------------------------------------------------------------------------------------- review round 4
+
+
+class _FakeCudaTorch:
+ """Allocator-wide peak stats, as torch keeps them: a reset by one render erases another's high-water mark."""
+
+ def __init__(self):
+ self.allocated = 0
+ self.peak = 0
+ torch_self = self
+
+ class _Dev:
+ def __init__(self, *args):
+ self.type = "cuda"
+ self.index = 0
+
+ def __str__(self):
+ return "cuda:0"
+
+ class cuda: # noqa: N801
+ is_available = staticmethod(lambda: True)
+ current_device = staticmethod(lambda: 0)
+ synchronize = staticmethod(lambda dev = None: None)
+ memory_allocated = staticmethod(lambda dev = None: torch_self.allocated)
+ max_memory_allocated = staticmethod(lambda dev = None: torch_self.peak)
+
+ @staticmethod
+ def reset_peak_memory_stats(dev = None):
+ torch_self.peak = torch_self.allocated
+
+ self.device = _Dev
+ self.cuda = cuda
+
+ def alloc(self, mib):
+ self.allocated += mib << 20
+ self.peak = max(self.peak, self.allocated)
+
+ def free(self, mib):
+ self.allocated -= mib << 20
+
+
+def test_overlapping_peak_probes_on_one_card_record_nothing(monkeypatch):
+ fake = _FakeCudaTorch()
+ monkeypatch.setitem(__import__("sys").modules, "torch", fake)
+ first = mp.PeakProbe("cuda:0")
+ fake.alloc(5000) # the first render's largest phase
+ fake.free(5000)
+ second = mp.PeakProbe("cuda:0") # a render on the other executor resets the card's peak
+ fake.alloc(100)
+ fake.free(100)
+ assert first.finish() is None # would read 100 MiB, not 5000
+ assert second.finish() is None
+ alone = mp.PeakProbe("cuda:0")
+ fake.alloc(700)
+ assert alone.finish() == 700
+ del alone
+ dropped = mp.PeakProbe("cuda:0") # a failed render never finishes its probe
+ del dropped
+ after = mp.PeakProbe("cuda:0")
+ fake.alloc(300)
+ assert after.finish() == 300 # not spoiled by the abandoned probe
+
+
+def test_partial_hook_failure_in_the_lower_residency_retry_propagates(monkeypatch):
+ calls = []
+
+ def fake_apply(pipe, device, logger, **kw):
+ calls.append(kw)
+ if len(calls) == 1:
+ dm._set_group_offload_failure(pipe, _OOM("CUDA out of memory"))
+ return False
+ raise RuntimeError("group offload failed after installing hooks on 1 module(s)")
+
+ monkeypatch.setattr(dm, "_apply_group_offload", fake_apply)
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ pipe = types.SimpleNamespace(components = {})
+ with pytest.raises(RuntimeError, match = "after installing hooks"):
+ dm.apply_memory_plan(pipe, _resident_dit_plan(), device = "cuda")
+ assert len(calls) == 2
+
+
+def test_learned_peaks_are_keyed_by_vae_savers(klein_pipe):
+ mp.record(
+ family = "some-new-dit",
+ quant = "torchao:Int8Tensor|model:org/dit|vae:tiled+sliced|attn:native",
+ speed_mode = "default",
+ width = 1024,
+ height = 1024,
+ peak_mib = 2600,
+ )
+ plan = _klein_flat_plan()
+ kw = dict(family = "some-new-dit", speed_mode = "default", attention = "native", model = "org/dit")
+ tiled = dm.refine_plan_from_loaded_weights(klein_pipe, plan, vae = "tiled+sliced", **kw)
+ assert tiled.estimates["measured_runtime_headroom_mib"] == 3072
+ # an untiled load never reuses the tiled decode's peak
+ assert dm.refine_plan_from_loaded_weights(klein_pipe, plan, vae = "full", **kw) is plan
+ assert (
+ dm.vae_saver_tag(True, True) == "tiled+sliced" and dm.vae_saver_tag(False, False) == "full"
+ )
+
+
+def test_render_with_a_vae_saver_the_key_lacks_is_not_probed(monkeypatch):
+ import core.inference.diffusion as diffusion
+
+ class _Probe:
+ def __init__(self, device):
+ self.device = "cuda:0"
+
+ monkeypatch.setattr(mp, "PeakProbe", _Probe)
+ monkeypatch.setattr(mp, "enabled", lambda: True)
+ vae = types.SimpleNamespace(use_tiling = False, use_slicing = False)
+ pipe = types.SimpleNamespace(
+ _unsloth_peak_identity = ("some-new-dit", "q|model:m|vae:full|attn:native", "default"),
+ vae = vae,
+ )
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)
+ state = types.SimpleNamespace(pipe = pipe, device = "cuda:0", speed_mode = "default")
+ assert diffusion._start_peak_probe(state) is not None
+ vae.use_tiling = True # the activation guard tiled this request's decode
+ assert diffusion._start_peak_probe(state) is None
+
+
+def test_load_passes_the_vae_savers_to_the_peak_identity():
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert "peak_vae = planned_vae_saver_tag(pipe, plan)" in src
+ assert "vae = peak_vae," in src
+
+
+def test_video_request_overlapped_by_another_render_records_nothing(store, monkeypatch):
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ work = 1280 * 704 * 49
+ window = mp.open_window("cuda:0")
+ image = mp.open_window("cuda:0") # an image render armed on the same card mid-request
+ module = types.SimpleNamespace(
+ _unsloth_video_pending = (work, 0, "cuda:0"),
+ _unsloth_video_window = ("cuda:0", window),
+ _unsloth_video_calibration_key = None,
+ )
+
+ class _Torch:
+ class cuda: # noqa: N801
+ @staticmethod
+ def max_memory_allocated(dev):
+ return 3000 << 20
+
+ with monkeypatch.context() as m:
+ m.setitem(__import__("sys").modules, "torch", _Torch)
+ assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) is None
+ assert not mp.close_window("cuda:0", image)
+
+
+# ---------------------------------------------------------------------------------------------------- review round 5
+
+
+def test_learned_peaks_are_keyed_by_text_encoder_precision(klein_pipe):
+ mp.record(
+ family = "some-new-dit",
+ quant = "torchao:Int8Tensor|model:org/dit|te:int8|vae:full|attn:native",
+ speed_mode = "default",
+ width = 1024,
+ height = 1024,
+ peak_mib = 2600,
+ )
+ plan = _klein_flat_plan()
+ kw = dict(
+ family = "some-new-dit", speed_mode = "default", attention = "native", model = "org/dit", vae = "full"
+ )
+ quantised = dm.refine_plan_from_loaded_weights(klein_pipe, plan, te = "int8", **kw)
+ assert quantised.estimates["measured_runtime_headroom_mib"] == 3072
+ # a dense encoder never reuses the quantised encoder's peak
+ assert dm.refine_plan_from_loaded_weights(klein_pipe, plan, te = "dense", **kw) is plan
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert "peak_te = _peak_te_label(te_quant, te_outcome)" in src and "te = peak_te," in src
+
+
+def test_vae_key_names_only_the_savers_the_pipeline_can_engage():
+ plan = types.SimpleNamespace(vae_tiling = True, vae_slicing = True)
+ untileable = types.SimpleNamespace(
+ vae = types.SimpleNamespace(enable_slicing = lambda: None, use_slicing = False)
+ )
+ assert dm.planned_vae_saver_tag(untileable, plan) == "sliced"
+ both = types.SimpleNamespace(
+ vae = types.SimpleNamespace(
+ enable_slicing = lambda: None,
+ enable_tiling = lambda: None,
+ use_slicing = False,
+ use_tiling = False,
+ )
+ )
+ assert dm.planned_vae_saver_tag(both, plan) == "tiled+sliced"
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ # after placement the key follows the savers that actually engaged, and the reserve follows the key
+ after_apply = src.index("plan = applied_plan(pipe, plan)")
+ assert (
+ 'with_label_part(identity[1], "vae", live_vae_saver_tag(pipe))'
+ in src[after_apply : after_apply + 800]
+ )
+ assert "pipe._unsloth_peak_static = True" in src[after_apply : after_apply + 800]
+
+
+def test_deferred_profile_sizes_requests_from_the_flat_estimate(klein_pipe):
+ klein_pipe._unsloth_phase_seeds = False
+ klein_pipe._unsloth_peak_identity = ("some-new-dit", "torchao:Int8Tensor", "default")
+ klein_pipe._unsloth_measured_reserve = (
+ 3072,
+ "some-new-dit",
+ "off",
+ None,
+ None,
+ 2600,
+ 1 << 20,
+ False,
+ )
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0
+ klein_pipe._unsloth_peak_static = True # the deferred profile engaged after placement
+ flat = dm.estimate_image_runtime_mib(
+ width = 1024, height = 1024, batch_size = 1, family = "some-new-dit"
+ )
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == flat - 3072
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ deferred = src.index("def _engage_deferred_profile")
+ assert "state.pipe._unsloth_peak_static = True" in src[deferred : deferred + 4000]
+
+
+def test_render_under_another_attention_backend_than_the_key_is_not_probed(monkeypatch):
+ import core.inference.diffusion as diffusion
+
+ class _Probe:
+ def __init__(self, device):
+ self.device = "cuda:0"
+
+ monkeypatch.setattr(mp, "PeakProbe", _Probe)
+ monkeypatch.setattr(mp, "enabled", lambda: True)
+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)
+ pipe = types.SimpleNamespace(
+ _unsloth_peak_identity = ("x", "q|vae:full|attn:native", "default"),
+ vae = types.SimpleNamespace(use_tiling = False, use_slicing = False),
+ )
+ state = types.SimpleNamespace(
+ pipe = pipe, device = "cuda:0", attention_backend = None, speed_mode = "default"
+ )
+ assert diffusion._start_peak_probe(state) is not None
+ state.attention_backend = "flash"
+ assert diffusion._start_peak_probe(state) is None
+
+
+def test_lower_residency_retry_returns_torchao_weights_without_module_to(monkeypatch):
+ torch = pytest.importorskip("torch")
+
+ class _Torchao(torch.nn.Module):
+ def to(self, *args, **kwargs):
+ raise RuntimeError("Module.to cannot move a torchao weight on its host copy")
+
+ returned = []
+ monkeypatch.setattr(
+ dm, "_return_module_tensors", lambda module, onload: returned.append(onload.type)
+ )
+ calls = []
+
+ def fake_apply(pipe, device, logger, **kw):
+ calls.append(kw)
+ if len(calls) == 1:
+ dm._set_group_offload_failure(pipe, _OOM("CUDA out of memory"))
+ return False
+ return True
+
+ monkeypatch.setattr(dm, "_apply_group_offload", fake_apply)
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ pipe = types.SimpleNamespace(components = {"transformer": _Torchao()})
+ policy, _ = dm.apply_memory_plan(pipe, _resident_dit_plan(), device = "cuda")
+ assert policy == dm.OFFLOAD_GROUP and len(calls) == 2
+ assert returned == ["cpu"]
+
+
+def test_quadratic_attention_peaks_scale_with_pixels_squared(store):
+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default")
+ mp.record(**kw, width = 1088, height = 1024, peak_mib = 1000, quadratic = True)
+ ratio = 2 * 1024 * 1024 / float(1088 * 1024)
+ assert mp.lookup(**kw, width = 1448, height = 1448) >= int(1000 * ratio * ratio)
+ # a request above every recorded bucket scales the largest one squared too
+ assert mp.lookup(**kw, width = 2048, height = 2048) >= int(1000 * ratio * ratio) * 4
+
+
+def test_record_passes_the_quadratic_attention_verdict():
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert "quadratic = _quadratic_attention(target, engaged, state.pipe)," in src
+ assert "if not _attention_scaling_known(target, engaged):" in src
+ assert "target = guard_target," in src
+
+
+def test_peak_measured_with_no_streamed_denoiser_is_raised_to_the_streamed_baseline(
+ store, monkeypatch
+):
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)
+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: 900)
+ resident = types.SimpleNamespace(components = {"transformer": object()})
+ monkeypatch.setattr(
+ dm, "_offload_groups", lambda module: [types.SimpleNamespace(_unsloth_resident = True)]
+ )
+ assert dm.stream_window_pad_mib(resident) == 900
+ monkeypatch.setattr(
+ dm, "_offload_groups", lambda module: [types.SimpleNamespace(_unsloth_resident = False)]
+ )
+ assert dm.stream_window_pad_mib(resident) == 0
+ pipe = types.SimpleNamespace(_unsloth_peak_identity = ("some-new-dit", "q", "default"))
+ probe = _Probe(2400)
+ probe.window_pad = 900
+ diffusion._record_peak_probe(
+ types.SimpleNamespace(pipe = pipe, speed_mode = "default"),
+ probe,
+ width = 1024,
+ height = 1024,
+ batch_size = 1,
+ )
+ assert mp.lookup(family = "some-new-dit", quant = "q", speed_mode = "default") == 3300
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert "window_pad = stream_window_pad_mib(pipe)" in src
+
+
+def test_residency_fit_registers_its_video_measurement_window():
+ src = (__import__("pathlib").Path(vsr.__file__)).read_text(encoding = "utf-8")
+ fit = src.index("def fit_for_request")
+ assert "dev = _open_window(module, dev)" in src[fit:]
+ assert src.count("dev = _open_window(module, dev)") == 2
+
+
+# ---------------------------------------------------------------------------------------------------- review round 6
+
+
+def test_request_guard_scales_a_reserved_peak_to_the_request_bucket(klein_pipe):
+ klein_pipe._unsloth_peak_identity = ("flux.2-klein", "torchao:Int8Tensor", "default")
+ klein_pipe._unsloth_measured_reserve = (
+ 4608,
+ "flux.2-klein",
+ "default",
+ None,
+ None,
+ 4000,
+ 1 << 20,
+ False,
+ )
+ # 1088x1024 crosses into the 2 MP bucket: 4000 x 2 x 1.15 = 9200 -> 9216, not 1.06x
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1088, height = 1024) == 9216 - 4608
+ klein_pipe._unsloth_measured_reserve = (
+ 4608,
+ "flux.2-klein",
+ "default",
+ None,
+ None,
+ 4000,
+ 1 << 20,
+ True,
+ )
+ # a math-attention entry scales squared: 4000 x 4 x 1.15 = 18400 -> 18432
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1088, height = 1024) == 18432 - 4608
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0
+
+
+def test_peak_window_stays_open_until_the_high_water_mark_is_read(monkeypatch):
+ fake = _FakeCudaTorch()
+ read = fake.cuda.max_memory_allocated
+ late = []
+
+ def racing_read(dev = None):
+ late.append(mp.open_window("cuda:0")) # a render arms between the sync and the read
+ return read(dev)
+
+ fake.cuda.max_memory_allocated = staticmethod(racing_read)
+ monkeypatch.setitem(__import__("sys").modules, "torch", fake)
+ probe = mp.PeakProbe("cuda:0")
+ fake.alloc(500)
+ assert probe.finish() is None
+ assert not mp.close_window("cuda:0", late[0])
+
+
+def test_unobservable_vae_savers_use_and_record_no_learned_peak(store):
+ pipe = types.SimpleNamespace(
+ vae = types.SimpleNamespace(enable_slicing = lambda: None, use_tiling = False)
+ )
+ plan = types.SimpleNamespace(vae_tiling = False, vae_slicing = True)
+ assert dm.planned_vae_saver_tag(pipe, plan) == "unknown"
+ assert dm.live_vae_saver_tag(pipe) == "unknown"
+ kw = dict(family = "some-new-dit", quant = "q|vae:unknown|attn:native", speed_mode = "default")
+ assert not mp.record(**kw, width = 1024, height = 1024, peak_mib = 2000)
+ assert mp.lookup(**kw) is None
+
+
+def test_peak_model_identity_names_the_whole_build(monkeypatch):
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_component_sizes as sizes
+
+ revisions = {("org/dit", "dit-Q4.gguf"): "abc", ("org/base", None): "def"}
+ monkeypatch.setattr(
+ sizes, "checkpoint_fingerprint", lambda repo, filename = None: revisions.get((repo, filename))
+ )
+ assert diffusion._peak_model_identity("org/dit", "dit-Q4.gguf", "gguf", "org/base") == (
+ "org/dit/dit-Q4.gguf@abc+org/base@def"
+ )
+ assert (
+ diffusion._peak_model_identity("org/dit", "dit-Q4.gguf", "gguf", "org/other") == "unknown"
+ )
+ assert diffusion._peak_model_identity("org/gone", None, "pipeline", None) == "unknown"
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert "peak_model = _peak_model_identity(repo_id, gguf_filename, kind, base)" in src
+
+
+def test_phase_table_seed_needs_its_measured_configuration(klein_pipe):
+ assert dm.measured_image_runtime_mib("flux.2-klein", "default") is None
+ plan = _klein_flat_plan()
+ klein_pipe._unsloth_phase_seeds = False # ROCm, or quadratic SDPA only
+ assert _refine(klein_pipe, plan) is plan
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert "pipe._unsloth_phase_seeds = _calibrated_activation(fam, target) is not None" in src
+
+
+def test_oom_inside_the_first_offload_apply_unwinds_its_hooks(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import diffusers.hooks as hooks
+
+ def fake_apply(module, **kwargs):
+ module._fake_hooked = True # hooked part of the module, then ran out of memory
+ raise _OOM("CUDA out of memory")
+
+ unwound = []
+
+ def fake_unwind(
+ pipe,
+ snapshots,
+ logger = None,
+ ):
+ for module, _ in snapshots:
+ module.__dict__.pop("_fake_hooked", None)
+ unwound.append(True)
+ return True
+
+ monkeypatch.setattr(hooks, "apply_group_offloading", fake_apply)
+ monkeypatch.setattr(
+ dm, "_offload_groups", lambda m: [object()] if getattr(m, "_fake_hooked", False) else []
+ )
+ monkeypatch.setattr(dm, "_unwind_group_offload", fake_unwind)
+ dit = torch.nn.Linear(4, 4)
+ pipe = types.SimpleNamespace(transformer = dit, components = {"transformer": dit})
+ assert dm._apply_group_offload(pipe, "cpu", None) is False
+ assert unwound and not getattr(dit, "_fake_hooked", False)
+ assert dm._is_oom(getattr(pipe, dm._GROUP_OFFLOAD_FAILURE_ATTR))
+
+
+def test_video_window_stays_open_until_the_high_water_mark_is_read(store, monkeypatch):
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ work = 1280 * 704 * 49
+ module = types.SimpleNamespace(
+ _unsloth_video_pending = (work, 0, "cuda:0"),
+ _unsloth_video_window = ("cuda:0", mp.open_window("cuda:0")),
+ _unsloth_video_calibration_key = None,
+ )
+
+ class _Torch:
+ class cuda: # noqa: N801
+ @staticmethod
+ def max_memory_allocated(dev):
+ mp.open_window("cuda:0") # an image render arms between the decode and the read
+ return 3000 << 20
+
+ with monkeypatch.context() as m:
+ m.setitem(__import__("sys").modules, "torch", _Torch)
+ assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) is None
+
+
+# ---------------------------------------------------------------------------------------------------- review round 7
+
+
+def _probe_state(monkeypatch, label, **live):
+ import core.inference.diffusion as diffusion
+
+ class _Probe:
+ def __init__(self, device):
+ self.device = "cuda:0"
+
+ monkeypatch.setattr(mp, "PeakProbe", _Probe)
+ monkeypatch.setattr(mp, "enabled", lambda: True)
+ pipe = types.SimpleNamespace(
+ _unsloth_peak_identity = ("x", label, "off"),
+ vae = types.SimpleNamespace(use_tiling = False, use_slicing = False),
+ )
+ state = types.SimpleNamespace(
+ pipe = pipe, device = "cuda:0", attention_backend = None, speed_mode = "off", transformer_cache = None
+ )
+ for name, value in live.items():
+ setattr(state, name, value)
+ return diffusion, state
+
+
+def test_local_pipeline_identity_covers_its_weight_files(tmp_path):
+ import core.inference.diffusion as diffusion
+
+ (tmp_path / "transformer").mkdir()
+ (tmp_path / "model_index.json").write_text("{}")
+ (tmp_path / "transformer" / "config.json").write_text("{}")
+ weights = tmp_path / "transformer" / "diffusion_pytorch_model.safetensors"
+ weights.write_bytes(b"0" * 64)
+ before = diffusion._peak_model_identity(str(tmp_path), None, "pipeline", None)
+ weights.write_bytes(b"0" * 128) # a heavier build, same manifests
+ after = diffusion._peak_model_identity(str(tmp_path), None, "pipeline", None)
+ assert before != "unknown" and after != "unknown" and before != after
+
+
+def test_unknown_stream_window_records_no_peak(monkeypatch):
+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: -1)
+ assert dm.stream_window_pad_mib(types.SimpleNamespace(components = {})) is None
+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")
+ state.pipe.components = {}
+ assert diffusion._start_peak_probe(state) is None
+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: 700)
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", dm.stream_window_pad_mib)
+ assert diffusion._start_peak_probe(state).window_pad == 700
+
+
+def test_bare_unknown_or_empty_format_is_unidentified(store):
+ for label in ("unknown", "", None, "unknown|model:x|attn:native"):
+ assert not mp.identified(label)
+ assert not mp.record(
+ family = "some-new-dit",
+ quant = label,
+ speed_mode = "default",
+ width = 1024,
+ height = 1024,
+ peak_mib = 900,
+ )
+ assert mp.identified("torchao:Int8Tensor|model:x|attn:native")
+
+
+def test_render_with_a_toggled_step_cache_is_not_probed(monkeypatch):
+ diffusion, state = _probe_state(monkeypatch, "q|cache:off|vae:full|attn:native")
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)
+ assert diffusion._start_peak_probe(state) is not None
+ state.transformer_cache = "fbcache" # a high-step request auto-engaged FBCache
+ assert diffusion._start_peak_probe(state) is None
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert 'peak_cache = "off" if cache_may_toggle else str(cache_engaged or "off")' in src
+
+
+def test_deferred_profile_retags_the_speed_tier(monkeypatch):
+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)
+ state.speed_mode = "default" # the deferred profile engaged the default tier
+ assert (
+ diffusion._start_peak_probe(state) is None
+ ) # an off-tier key never takes a compiled render
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ deferred = src.index("def _engage_deferred_profile")
+ block = src[deferred : deferred + 5000]
+ assert (
+ 'with_label_part(identity[1], "attn", attention_engaged or "native"),\n SPEED_DEFAULT,'
+ in block
+ )
+
+
+# ---------------------------------------------------------------------------------------------------- review round 8
+
+
+def test_failed_video_request_closes_its_measurement_window():
+ window = mp.open_window("cuda:0")
+ module = types.SimpleNamespace(
+ _unsloth_video_pending = (1, 0, "cuda:0"), _unsloth_video_window = ("cuda:0", window)
+ )
+ vsr.discard_request(types.SimpleNamespace(transformer = module))
+ assert module._unsloth_video_pending is None and module._unsloth_video_window is None
+ later = mp.open_window("cuda:0") # an image render after the cancelled video
+ assert mp.close_window("cuda:0", later)
+ src = (__import__("pathlib").Path(vsr.__file__).parent / "video.py").read_text(encoding = "utf-8")
+ assert 'video_stream_residency.discard_request(getattr(state, "pipe", None))' in src
+
+
+def test_whole_module_offload_records_no_streamed_baseline_peak(monkeypatch):
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: 700)
+ monkeypatch.setattr(dm, "_offload_groups", lambda module: [])
+ dit = torch.nn.Linear(
+ 4, 4
+ ) # on the host at render start: whole-module offload onloads it inside the render
+ assert dm.stream_window_pad_mib(types.SimpleNamespace(components = {"transformer": dit})) is None
+ dit._hf_hook = object()
+ assert dm.stream_window_pad_mib(types.SimpleNamespace(components = {"transformer": dit})) is None
+
+
+def test_compile_fallback_during_the_render_records_nothing(store, monkeypatch):
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)
+ pipe = types.SimpleNamespace(_unsloth_peak_identity = ("some-new-dit", "q", "default"))
+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default", speed_optims = ("compiled",))
+ state.speed_optims = ("compile_fallback_eager",) # settle_compile_fallback mid-render
+ diffusion._record_peak_probe(state, _Probe(2400), width = 1024, height = 1024, batch_size = 1)
+ assert mp.lookup(family = "some-new-dit", quant = "q", speed_mode = "default") is None
+ state.speed_optims = ("compiled",)
+ diffusion._record_peak_probe(state, _Probe(2400), width = 1024, height = 1024, batch_size = 1)
+ assert mp.lookup(family = "some-new-dit", quant = "q", speed_mode = "default") == 2400
+
+
+def test_failed_capture_video_peak_never_reaches_the_keyed_or_persisted_peaks(store, monkeypatch):
+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")
+ work = 1280 * 704 * 49
+ captures = {"n": 0}
+ monkeypatch.setattr(vsr, "_failed_captures", lambda: captures["n"])
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: captures["n"])
+
+ def render(peak_mib):
+ module = types.SimpleNamespace(
+ _unsloth_video_pending = (work, 0, "cuda:0"),
+ _unsloth_video_calibration_key = key,
+ _unsloth_video_failed_captures = captures["n"],
+ )
+
+ class _Torch:
+ class cuda: # noqa: N801
+ @staticmethod
+ def max_memory_allocated(dev):
+ return peak_mib << 20
+
+ return module, _Torch
+
+ module, torch_stub = render(15000)
+ captures["n"] += 1 # the capture failed during this render
+ with monkeypatch.context() as m:
+ m.setitem(__import__("sys").modules, "torch", torch_stub)
+ vsr.record_request_peak(types.SimpleNamespace(transformer = module))
+ module, torch_stub = render(3000)
+ with monkeypatch.context() as m:
+ m.setitem(__import__("sys").modules, "torch", torch_stub)
+ vsr.record_request_peak(types.SimpleNamespace(transformer = module))
+ assert mp.work_peaks(vsr._store_key(key)) == {work: 3000}
+ assert vsr.measured_extra_mib(key, work) == 3000
+
+
+# ---------------------------------------------------------------------------------------------------- review round 9
+
+
+def test_graphs_left_by_an_unrecorded_render_stop_image_sampling(store, monkeypatch):
+ import core.inference.diffusion_cuda_graph as cg
+
+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)
+ monkeypatch.setattr(diffusion, "_peak_device_index", lambda state: 0)
+ live = {0: 3, 1: 0}
+ monkeypatch.setattr(cg, "live_graph_count", lambda index: live.get(index, 0))
+ assert diffusion._start_peak_probe(state) is not None # the load's own graphs are the baseline
+ live[1] = 5 # another card's graphs never count
+ assert diffusion._start_peak_probe(state) is not None
+ live[0] = 5 # a cache-served render recorded graphs on this card and was not sampled
+ assert diffusion._start_peak_probe(state) is None
+ live[0] = 0 # an OOM reset freed every graph: nothing unaccounted is left
+ assert diffusion._start_peak_probe(state) is not None
+
+
+def test_graphs_a_recorded_render_captured_keep_sampling_open(store, monkeypatch):
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)
+ monkeypatch.setattr(
+ cg, "live_graph_count", lambda index: 4
+ ) # recorded during this sampled render
+ pipe = types.SimpleNamespace(
+ _unsloth_peak_identity = ("some-new-dit", "q", "default"), _unsloth_peak_graphs = 0
+ )
+ diffusion._record_peak_probe(
+ types.SimpleNamespace(pipe = pipe, speed_mode = "default"),
+ _Probe(2400),
+ width = 1024,
+ height = 1024,
+ batch_size = 1,
+ )
+ assert pipe._unsloth_peak_graphs == 4
+
+
+def test_graphs_left_by_a_discarded_video_request_stop_video_sampling(store, monkeypatch):
+ torch = pytest.importorskip("torch")
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ live = {0: 1}
+ monkeypatch.setattr(cg, "live_graph_count", lambda index: live.get(index, 0))
+ module = types.SimpleNamespace(_unsloth_video_calibration_key = None)
+ pipe = types.SimpleNamespace(transformer = module)
+ vsr._open_window(module, torch.device("cuda", 0))
+ live[0] += 1 # the request recorded a graph, then was cancelled
+ vsr.discard_request(pipe)
+ vsr._open_window(module, torch.device("cuda", 0))
+ module._unsloth_video_pending = (1280 * 704 * 49, 0, torch.device("cuda", 0))
+
+ class _Torch:
+ class cuda: # noqa: N801
+ @staticmethod
+ def max_memory_allocated(dev):
+ return 3000 << 20
+
+ with monkeypatch.context() as m:
+ m.setitem(__import__("sys").modules, "torch", _Torch)
+ assert vsr.record_request_peak(pipe) is None
+
+
+def test_live_graph_count_is_per_device_and_drops_reset_graphs():
+ import core.inference.diffusion_block_graph as bg
+ import core.inference.diffusion_cuda_graph as cg
+
+ class _Wrapper:
+ def __init__(self, device, n):
+ self._unsloth_capture_device = device
+ self.cache = dict.fromkeys(range(n))
+
+ a, b = _Wrapper(0, 2), _Wrapper(1, 3)
+ blocks = _Wrapper(0, 4)
+ cg._LIVE_WRAPPERS.add(a)
+ cg._LIVE_WRAPPERS.add(b)
+ bg._LIVE_BLOCKS.add(blocks)
+ try:
+ assert cg.live_graph_count(0) == 6 and cg.live_graph_count(1) == 3
+ a.cache.clear() # reset_all
+ assert cg.live_graph_count(0) == 4
+ finally:
+ cg._LIVE_WRAPPERS.discard(a)
+ cg._LIVE_WRAPPERS.discard(b)
+ bg._LIVE_BLOCKS.discard(blocks)
+
+
+def test_render_without_the_familys_cfg_is_not_probed(monkeypatch):
+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)
+ state.family = types.SimpleNamespace(uses_negative_prompt = True)
+ state.pipe._unsloth_peak_cfg = True
+ assert diffusion._start_peak_probe(state) is not None
+ state.pipe._unsloth_peak_cfg = False # guidance off: half the denoiser batch
+ assert diffusion._start_peak_probe(state) is None
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert "state.pipe._unsloth_peak_cfg = _negative_prompt_engaged(state.family, guidance)" in src
+
+
+def test_render_forced_eager_while_the_compile_is_pending_is_not_probed(monkeypatch):
+ import core.inference.diffusion_bg_compile as bg
+
+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)
+ assert diffusion._start_peak_probe(state) is not None
+ token = bg._FORCE_EAGER.set(True)
+ try:
+ assert diffusion._start_peak_probe(state) is None
+ finally:
+ bg._FORCE_EAGER.reset(token)
+
+
+# ---------------------------------------------------------------------------------------------------- review round 10
+
+
+def test_render_with_lora_adapters_is_not_probed(monkeypatch):
+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)
+ assert diffusion._start_peak_probe(state) is not None
+ state.pipe._unsloth_loras = (("style", "/loras/style.safetensors", 0.8),)
+ assert diffusion._start_peak_probe(state) is None
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert '"unknown" # adapters baked in at load' in src
+
+
+def test_learned_reserve_falls_back_to_the_flat_estimate_with_loras_or_a_failed_saver(klein_pipe):
+ klein_pipe._unsloth_phase_seeds = (
+ False # unseeded: the reserve came from the learned peak alone
+ )
+ klein_pipe._unsloth_peak_identity = ("some-new-dit", "torchao:Int8Tensor", "default")
+ klein_pipe._unsloth_measured_reserve = (
+ 3072,
+ "some-new-dit",
+ "default",
+ None,
+ None,
+ 2600,
+ 1 << 20,
+ False,
+ )
+ flat = dm.estimate_image_runtime_mib(
+ width = 1024, height = 1024, batch_size = 1, family = "some-new-dit"
+ )
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0
+ klein_pipe._unsloth_loras = (("style", "/loras/style.safetensors", 0.8),)
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == flat - 3072
+ klein_pipe._unsloth_loras = ()
+ klein_pipe._unsloth_peak_static = True # a planned VAE saver failed to engage
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == flat - 3072
+
+
+def test_store_written_under_other_runtimes_is_ignored(store, monkeypatch):
+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default")
+ assert mp.record(**kw, width = 1024, height = 1024, peak_mib = 2000)
+ assert mp.lookup(**kw) == 2000
+ monkeypatch.setattr(mp, "_runtime", ["torch=9.9|diffusers=9.9|cuda=99"]) # an upgraded runtime
+ mp._cache.update(path = None, mtime = None, data = None)
+ assert mp.lookup(**kw) is None
+
+
+def test_quadratic_peaks_scale_linearly_across_the_batch(store, klein_pipe):
+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default")
+ assert not mp.record(**kw, width = 1024, height = 1024, batch_size = 2, peak_mib = 3000, quadratic = True)
+ assert mp.record(**kw, width = 1024, height = 1024, peak_mib = 1000, quadratic = True)
+ assert mp.lookup(**kw, width = 1024, height = 1024, batch_size = 2) == 2000 # not 4000
+ assert mp.lookup(**kw, width = 1448, height = 1448) >= 4000 # pixels still square
+ klein_pipe._unsloth_peak_identity = ("flux.2-klein", "torchao:Int8Tensor", "default")
+ klein_pipe._unsloth_measured_reserve = (
+ 4608,
+ "flux.2-klein",
+ "default",
+ None,
+ None,
+ 4000,
+ 1 << 20,
+ True,
+ )
+ # batch 2 at 1 MP: 4000 x 2 x 1.15 = 9200 -> 9216, not 4000 x 4
+ assert (
+ dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024, batch_size = 2)
+ == 9216 - 4608
+ )
+
+
+# ---------------------------------------------------------------------------------------------------- review round 11
+
+
+def test_resident_text_encoders_add_their_streaming_window(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import core.inference.diffusion_offload_prefetch as prefetch
+
+ monkeypatch.setattr(prefetch, "prefetch_depth", lambda: 1)
+ dit, encoder = torch.nn.Linear(4, 4), torch.nn.Linear(1024, 1024, bias = False) # 4 MiB fp32 leaf
+ streamed = types.SimpleNamespace(_unsloth_resident = False)
+ resident = types.SimpleNamespace(_unsloth_resident = True)
+ groups = {id(dit): [streamed], id(encoder): [resident]} # streamed denoiser, resident encoder
+ monkeypatch.setattr(dm, "_offload_groups", lambda module: groups.get(id(module), []))
+ pipe = types.SimpleNamespace(components = {"transformer": dit, "text_encoder": encoder})
+ assert dm.stream_window_pad_mib(pipe) == 8 # (depth + 1) x the largest encoder leaf
+ groups[id(encoder)] = [streamed] # streamed during the render: its window is in the peak
+ assert dm.stream_window_pad_mib(pipe) == 0
+ groups[id(encoder)] = [] # no hooks, weights on the host: placement unknown, record nothing
+ assert dm.stream_window_pad_mib(pipe) is None
+
+
+def test_video_peaks_are_persisted_on_the_streamed_baseline(store, monkeypatch):
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")
+ work = 1280 * 704 * 49
+
+ class _Torch:
+ class cuda: # noqa: N801
+ @staticmethod
+ def max_memory_allocated(dev):
+ return 3000 << 20
+
+ def render(pad):
+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: pad)
+ module = types.SimpleNamespace(
+ _unsloth_video_pending = (work, 0, "cuda:0"), _unsloth_video_calibration_key = key
+ )
+ with monkeypatch.context() as m:
+ m.setitem(__import__("sys").modules, "torch", _Torch)
+ vsr.record_request_peak(types.SimpleNamespace(transformer = module))
+
+ render(None) # placement unknown: nothing persisted
+ assert mp.work_peaks(vsr._store_key(key)) == {}
+ render(700) # every group resident: the prefetch window a streamed load holds is added
+ assert mp.work_peaks(vsr._store_key(key)) == {work: 3700}
+ assert vsr.measured_extra_mib(key, work) == 3700
+
+
+# --------------------------------------------------------------------------------------------------- review round 12
+
+
+def test_a_graph_replaced_in_a_full_cache_stops_image_sampling(store, monkeypatch):
+ import core.inference.diffusion_cuda_graph as cg
+
+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)
+ monkeypatch.setattr(diffusion, "_peak_device_index", lambda state: 0)
+ monkeypatch.setattr(cg, "live_graph_count", lambda index: 4)
+ gen = {0: 7, 1: 0}
+ monkeypatch.setattr(cg, "capture_generation", lambda index: gen.get(index, 0))
+ assert diffusion._start_peak_probe(state) is not None # the load's graphs are the baseline
+ gen[1] = 3 # another card's captures never count
+ assert diffusion._start_peak_probe(state) is not None
+ gen[0] = 8 # a full cache evicted one graph and captured its replacement: same count, new graph
+ assert diffusion._start_peak_probe(state) is None
+
+
+def test_capture_generation_counts_per_device(monkeypatch):
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(cg, "_CAPTURES", {})
+ cg.note_capture(0)
+ cg.note_capture(0)
+ cg.note_capture(1)
+ assert (
+ cg.capture_generation(0) == 2
+ and cg.capture_generation(1) == 1
+ and cg.capture_generation(2) == 0
+ )
+
+
+def test_runtime_stamp_names_installed_kernel_packages(monkeypatch):
+ import importlib.metadata as md
+
+ real = md.version
+
+ def fake(dist):
+ if dist == "sageattention":
+ return "2.2.0"
+ if dist in mp._KERNEL_DISTS:
+ raise md.PackageNotFoundError(dist)
+ return real(dist)
+
+ monkeypatch.setattr(md, "version", fake)
+ monkeypatch.setattr(mp, "_runtime", [])
+ stamp = mp.runtime_stamp()
+ assert "sageattention=2.2.0" in stamp and "xformers" not in stamp
+ monkeypatch.setattr(mp, "_runtime", [])
+ monkeypatch.setattr(md, "version", lambda d: "2.3.0" if d == "sageattention" else fake(d))
+ assert mp.runtime_stamp() != stamp # a kernel upgrade drops the stored peaks
+
+
+def test_attention_scaling_unknown_render_is_not_recorded(store, monkeypatch):
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)
+ monkeypatch.setattr(
+ diffusion, "_live_peak_identity", lambda state: ("some-new-dit", "q", "default")
+ )
+ monkeypatch.setattr(diffusion, "_conditioning_cache_hits", lambda pipe: 0)
+ answers = {"math": False, "fused": False}
+ monkeypatch.setattr(diffusion, "sdpa_math_only", lambda target: answers["math"])
+ monkeypatch.setattr(diffusion, "sdpa_subquadratic_confirmed", lambda target: answers["fused"])
+ target = types.SimpleNamespace(device = "cuda:0")
+ assert not diffusion._attention_scaling_known(target, "native") # the probe answered nothing
+ assert not diffusion._attention_scaling_known(None, None)
+ assert diffusion._attention_scaling_known(
+ None, "sage"
+ ) # an engaged fused backend is evidence on its own
+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default", device = "NVIDIA L4")
+
+ def render():
+ pipe = types.SimpleNamespace(_unsloth_peak_identity = ("some-new-dit", "q", "default"))
+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default", attention_backend = "native")
+ diffusion._record_peak_probe(
+ state, _Probe(2400), width = 1024, height = 1024, batch_size = 1, target = target
+ )
+ return pipe
+
+ pipe = render()
+ assert mp.lookup(**kw, width = 1024, height = 1024) is None
+ assert (
+ getattr(pipe, "_unsloth_peak_graphs", None) is None
+ ) # an unrecorded render moves no graph baseline
+ answers["fused"] = True
+ render()
+ assert mp.lookup(**kw, width = 1024, height = 1024) is not None
+
+
+def test_partially_resident_denoiser_pads_the_window_difference(monkeypatch):
+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: 900)
+ monkeypatch.setattr(dm, "_encoder_stream_window_mib", lambda pipe: 0)
+ pipe = types.SimpleNamespace(components = {"transformer": object()})
+ groups = [
+ types.SimpleNamespace(_unsloth_resident = True),
+ types.SimpleNamespace(_unsloth_resident = False),
+ ]
+ monkeypatch.setattr(dm, "_offload_groups", lambda module: groups)
+ monkeypatch.setattr(dm, "_streamed_group_window_mib", lambda pipe: 300)
+ assert (
+ dm.stream_window_pad_mib(pipe) == 600
+ ) # a fully streamed load holds the largest group, not the streamed one
+ monkeypatch.setattr(dm, "_streamed_group_window_mib", lambda pipe: 900)
+ assert dm.stream_window_pad_mib(pipe) == 0
+ monkeypatch.setattr(dm, "_streamed_group_window_mib", lambda pipe: -1)
+ assert dm.stream_window_pad_mib(pipe) is None # unsized: not recorded
+ groups[0]._unsloth_resident = False # fully streamed: already the baseline
+ assert dm.stream_window_pad_mib(pipe) == 0
+
+
+def test_streamed_group_window_counts_only_streamed_groups(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import core.inference.diffusion_offload_prefetch as pf
+
+ monkeypatch.setattr(pf, "prefetch_depth", lambda: 1)
+ big, small = torch.nn.Linear(1024, 1024, bias = False), torch.nn.Linear(512, 512, bias = False)
+ groups = [
+ types.SimpleNamespace(_unsloth_resident = True, modules = [big], parameters = None, buffers = None),
+ types.SimpleNamespace(
+ _unsloth_resident = False, modules = [small], parameters = None, buffers = None
+ ),
+ ]
+ monkeypatch.setattr(dm, "_offload_groups", lambda module: groups)
+ pipe = types.SimpleNamespace(components = {"transformer": torch.nn.Module()})
+ assert dm._streamed_group_window_mib(pipe) == 2 # 1 MiB (512 x 512 fp32) x (depth 1 + 1)
+
+
+def test_a_graph_replaced_in_a_full_cache_stops_video_sampling(monkeypatch):
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(vsr, "_live_graphs", lambda dev: 4)
+ gen = [7]
+ monkeypatch.setattr(vsr, "_capture_generation", lambda dev: gen[0])
+ monkeypatch.setattr("core.inference.diffusion_measured_peaks.open_window", lambda dev: 1)
+ monkeypatch.setattr(
+ "core.inference.diffusion_measured_peaks.close_window", lambda dev, token: True
+ )
+ module = types.SimpleNamespace()
+ dev = torch.device("cuda", 0)
+ vsr._open_window(module, dev)
+ assert module._unsloth_video_graphs_clean # the load's graphs are the baseline
+ vsr._open_window(module, dev)
+ assert module._unsloth_video_graphs_clean
+ gen[0] = (
+ 8 # a discarded request evicted one graph and captured its replacement: same count, new graph
+ )
+ vsr._open_window(module, dev)
+ assert not module._unsloth_video_graphs_clean
+
+
+# --------------------------------------------------------------------------------------------------- review round 14
+
+
+def test_unseeded_reserve_scales_without_the_store(store, klein_pipe):
+ klein_pipe._unsloth_phase_seeds = False
+ klein_pipe._unsloth_peak_identity = ("some-new-dit", "torchao:Int8Tensor", "default")
+ klein_pipe._unsloth_measured_reserve = (
+ 3072,
+ "some-new-dit",
+ "default",
+ None,
+ None,
+ 2600,
+ 1 << 20,
+ False,
+ )
+ assert (
+ mp.lookup(family = "some-new-dit", quant = "torchao:Int8Tensor", speed_mode = "default") is None
+ )
+ # the store holds nothing (evicted, deleted, unwritable): a larger request still scales the reserved peak
+ assert dm.measured_request_extra_mib(klein_pipe, width = 2048, height = 2048) > 0
+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0
+
+
+def test_another_pipelines_reset_keeps_this_pipes_capture_dirty(monkeypatch):
+ import core.inference.diffusion_cuda_graph as cg
+
+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")
+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)
+ monkeypatch.setattr(diffusion, "_peak_device_index", lambda state: 0)
+ live, gen = [6], [10]
+ monkeypatch.setattr(cg, "live_graph_count", lambda index: live[0])
+ monkeypatch.setattr(cg, "capture_generation", lambda index: gen[0])
+ assert diffusion._start_peak_probe(state) is not None # baseline: 6 graphs, generation 10
+ live[0], gen[0] = 7, 11 # this pipe captured one graph in a render that was not recorded
+ assert diffusion._start_peak_probe(state) is None
+ live[0] = (
+ 3 # a video pipeline on the same card reset its 4 graphs; this pipe's capture is still alive
+ )
+ assert diffusion._start_peak_probe(state) is None
+ live[0] = 0 # nothing alive on the card: nothing unaccounted can remain
+ assert diffusion._start_peak_probe(state) is not None
+
+
+# --------------------------------------------------------------------------------------------------- review round 15
+
+
+def test_record_reports_whether_the_store_holds_the_sample(store):
+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default", width = 1024, height = 1024)
+ held: list = []
+ assert mp.record(**kw, peak_mib = 2000, held = held) and held == [True]
+ held = []
+ assert not mp.record(**kw, peak_mib = 1500, held = held) and held == [True] # already covered
+ held = []
+ assert not mp.record_work(("k",), 10, 0, held = held) and held == []
+ assert mp.record_work(("k",), 10, 100, held = held) and held == [True]
+
+
+def test_unpersisted_image_sample_keeps_its_graphs_dirty(store, monkeypatch):
+ import core.inference.diffusion as diffusion
+ import core.inference.diffusion_cuda_graph as cg
+
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")
+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)
+ monkeypatch.setattr(cg, "live_graph_count", lambda index: 4)
+ monkeypatch.setattr(cg, "capture_generation", lambda index: 9)
+ pipe = types.SimpleNamespace(
+ _unsloth_peak_identity = ("some-new-dit", "q", "default"), _unsloth_peak_graphs = 0
+ )
+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default")
+ monkeypatch.setattr(mp, "record", lambda **kw: False) # the store is unavailable
+ diffusion._record_peak_probe(state, _Probe(2400), width = 1024, height = 1024, batch_size = 1)
+ assert pipe._unsloth_peak_graphs == 0
+ monkeypatch.setattr(mp, "record", lambda **kw: kw["held"].append(True) or True)
+ diffusion._record_peak_probe(state, _Probe(2400), width = 1024, height = 1024, batch_size = 1)
+ assert pipe._unsloth_peak_graphs == 4 and pipe._unsloth_peak_graph_gen == 9
+
+
+def test_unknown_local_video_build_persists_nothing():
+ known = vsr.calibration_key(
+ "wan", "/models/wan", "group", "cuda:0", revision = "/models/wan@local:ab:cd"
+ )
+ unknown = vsr.calibration_key("wan", "/models/wan", "group", "cuda:0", revision = "unknown")
+ unread = vsr.calibration_key("wan", "/models/wan", "group", "cuda:0", revision = None)
+ assert vsr._store_key(known) is not None
+ assert vsr._store_key(unknown) is None and vsr._store_key(unread) is None
+ src = (__import__("pathlib").Path(dm.__file__).parent / "video.py").read_text(encoding = "utf-8")
+ assert "revision = _peak_model_identity(" in src
+
+
+def test_local_weight_shards_change_the_video_revision(tmp_path, monkeypatch):
+ import core.inference.diffusion as diffusion
+
+ (tmp_path / "model_index.json").write_text("{}")
+ (tmp_path / "transformer").mkdir()
+ (tmp_path / "transformer" / "config.json").write_text("{}")
+ shard = tmp_path / "transformer" / "diffusion_pytorch_model.safetensors"
+ shard.write_bytes(b"a" * 8)
+ before = diffusion._peak_model_identity(str(tmp_path), None, "pipeline", None)
+ shard.write_bytes(b"b" * 16) # same manifests, new weights
+ after = diffusion._peak_model_identity(str(tmp_path), None, "pipeline", None)
+ assert before != "unknown" and before != after
+
+
+def test_learned_peak_label_carries_the_graph_mode(klein_pipe, monkeypatch):
+ import core.inference.diffusion as diffusion
+
+ label = dm.learned_peak_label(klein_pipe, "native", "m", "none", "dense", "off", "on")
+ assert label.endswith("|graphs:on")
+ assert dm.learned_peak_label(klein_pipe, "native", "m", "none", "dense", "off", "off") != label
+ state = types.SimpleNamespace(resolved = None, speed_optims = ("compiled",), cuda_graphs = ())
+ assert diffusion._live_graph_mode(state) == "off"
+ state.speed_optims = ("compiled", "cuda_graph")
+ assert diffusion._live_graph_mode(state) == "step"
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert 'label = with_label_part(label, "graphs", _live_graph_mode(state))' in src
+ assert "graphs = peak_graphs," in src
+
+
+# --------------------------------------------------------------------------------------------------- review round 16
+
+
+def test_whole_step_and_block_graphs_record_under_distinct_keys(store):
+ import core.inference.diffusion as diffusion
+
+ state = types.SimpleNamespace(
+ resolved = None,
+ speed_optims = ("compiled", "cuda_graph"),
+ cuda_graphs = (),
+ pipe = types.SimpleNamespace(_unsloth_cuda_graph_mode = "step"),
+ )
+ assert diffusion._live_graph_mode(state) == "step"
+ state.pipe._unsloth_cuda_graph_mode = "blocks"
+ assert diffusion._live_graph_mode(state) == "blocks"
+ assert diffusion._engaged_graph_mode(state.pipe, {"cuda_graph": False}) == "off"
+ # a load places before it knows the mode: it reads the larger of both modes' peaks
+ base = dict(family = "flux.2-klein", speed_mode = "default", width = 1024, height = 1024)
+ assert mp.record(**base, quant = "torchao:Int8Tensor|graphs:step", peak_mib = 3000)
+ assert mp.lookup(**base, quant = "torchao:Int8Tensor|graphs:on") == 3000
+ assert mp.record(**base, quant = "torchao:Int8Tensor|graphs:blocks", peak_mib = 4200)
+ assert mp.lookup(**base, quant = "torchao:Int8Tensor|graphs:on") == 4200
+ assert mp.lookup(**base, quant = "torchao:Int8Tensor|graphs:step") == 3000
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert "_engaged_graph_mode(pipe, speed_applied)" in src
+
+
+def test_torchao_variants_of_one_weight_class_do_not_share_a_label(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import core.inference.diffusion_compile_cache as cc
+
+ monkeypatch.setattr(dm, "_torchao_weight_classes", lambda module: {"Int8Tensor"})
+ pipe = types.SimpleNamespace(transformer = torch.nn.Linear(4, 4))
+ variant = {"weights": {"Linear/Int8Tensor": 1}}
+ monkeypatch.setattr(cc, "graph_variant", lambda dit: variant)
+ plain = dm.denoiser_quant_label(pipe)
+ assert plain.startswith("torchao:Int8Tensor#")
+ variant = {"weights": {"Linear/Int8Tensor/convrot256": 1}}
+ assert dm.denoiser_quant_label(pipe) != plain # ConvRot: same class, other kernels
+ variant = {"weights": {"Linear/Int8Tensor/act_int8": 1}}
+ assert dm.denoiser_quant_label(pipe) != plain # W8A8 vs weight-only
+ variant = {"weights": "unavailable: RuntimeError"}
+ assert not mp.identified(dm.denoiser_quant_label(pipe)) # unreadable: neither read nor recorded
+
+
+def test_partially_resident_text_encoder_pads_the_window_difference(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import core.inference.diffusion_offload_prefetch as pf
+
+ monkeypatch.setattr(pf, "prefetch_depth", lambda: 1)
+ big, small = torch.nn.Linear(1024, 1024, bias = False), torch.nn.Linear(512, 512, bias = False)
+ te = torch.nn.Sequential(big, small)
+ groups = [
+ types.SimpleNamespace(_unsloth_resident = True, modules = [big], parameters = None, buffers = None),
+ types.SimpleNamespace(
+ _unsloth_resident = False, modules = [small], parameters = None, buffers = None
+ ),
+ ]
+ monkeypatch.setattr(dm, "_offload_groups", lambda module: groups)
+ pipe = types.SimpleNamespace(components = {"text_encoder": te})
+ # largest leaf 4 MiB x 2 = 8 MiB when fully streamed; the render streamed only the 1 MiB leaf (2 MiB in flight)
+ assert dm._encoder_stream_window_mib(pipe) == 6
+ groups[0]._unsloth_resident = False # fully streamed: already the baseline
+ assert dm._encoder_stream_window_mib(pipe) == 0
+
+
+# --------------------------------------------------------------------------------------------------- review round 17
+
+
+def test_graph_mode_envelope_covers_either_mode(store):
+ base = dict(family = "flux.2-klein", speed_mode = "default")
+ # a 3000 MiB step peak covering 4 MP, a 2900 MiB block peak covering 1 MP
+ assert mp.record(**base, quant = "q|graphs:step", width = 2048, height = 2048, peak_mib = 3000)
+ assert mp.record(**base, quant = "q|graphs:blocks", width = 1024, height = 1024, peak_mib = 2900)
+ blocks = mp.lookup_bucket(**base, quant = "q|graphs:blocks", width = 1024, height = 1024)
+ peak, pixels, quadratic = mp.lookup_bucket(**base, quant = "q|graphs:on", width = 1024, height = 1024)
+ assert peak == 3000 and pixels == blocks[1] # the larger peak over the smaller covered bucket
+ assert not quadratic
+ # a later 2 MP request scaled from the envelope covers what the block entry alone asks for
+ want = mp.resolution_bucket(1448, 1448, 1)
+ assert peak * want / pixels >= mp.lookup(
+ **base, quant = "q|graphs:blocks", width = 1448, height = 1448
+ )
+
+
+def test_in_process_video_peak_is_on_the_streamed_baseline(monkeypatch):
+ monkeypatch.setenv(mp.LEARNED_PEAKS_ENV, "0") # persistence off: the keyed table alone answers
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")
+ work = 1280 * 704 * 49
+
+ class _Torch:
+ class cuda: # noqa: N801
+ @staticmethod
+ def max_memory_allocated(dev):
+ return (3000 + 100) << 20
+
+ def run(pad):
+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: pad)
+ module = types.SimpleNamespace(_unsloth_video_pending = (work, 100 << 20, "cuda:0"))
+ module._unsloth_video_calibration_key = key
+ with monkeypatch.context() as m:
+ m.setitem(__import__("sys").modules, "torch", _Torch)
+ assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) == 3000
+ return module
+
+ run(None) # unknown placement: nothing enters the keyed table
+ assert vsr.measured_extra_mib(key, work) is None
+ module = run(600) # groups kept resident: the reused peak includes the prefetch window
+ assert vsr.measured_extra_mib(key, work) == 3600
+ assert module._unsloth_video_peaks[work] == 3000 # the module's own diagnostic stays raw
+
+
+# --------------------------------------------------------------------------------------------------- review round 18
+
+
+def test_capture_generation_increments_are_serialized(monkeypatch):
+ import core.inference.diffusion_cuda_graph as dcg
+
+ seen: list = []
+
+ class _Counts(dict):
+ def get(self, *a):
+ lock = getattr(dcg, "_CAPTURES_LOCK", None)
+ seen.append(lock is not None and lock.locked())
+ return super().get(*a)
+
+ monkeypatch.setattr(dcg, "_CAPTURES", _Counts())
+ dcg.note_capture(0)
+ dcg.note_capture(0)
+ assert dcg.capture_generation(0) == 2
+ assert seen and all(seen[:2]) # each read-modify-write held the lock
+
+
+def test_poisoned_graph_load_records_no_video_calibration(monkeypatch):
+ import core.inference.diffusion_cuda_graph as dcg
+
+ monkeypatch.setenv(mp.LEARNED_PEAKS_ENV, "0")
+ monkeypatch.setattr(vsr, "_PEAKS", {})
+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)
+ key = vsr.calibration_key(
+ "wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_optims = ("compiled", "cuda_graph")
+ )
+ work = 1280 * 704 * 49
+ handle = types.SimpleNamespace(poisoned = False)
+ monkeypatch.setattr(
+ dcg,
+ "never_engaged",
+ lambda handles: "capture failed" if all(h.poisoned for h in handles) else None,
+ )
+
+ class _Torch:
+ class cuda: # noqa: N801
+ @staticmethod
+ def max_memory_allocated(dev):
+ return (3000 + 100) << 20
+
+ def run():
+ module = types.SimpleNamespace(_unsloth_video_pending = (work, 100 << 20, "cuda:0"))
+ module._unsloth_video_calibration_key = key
+ pipe = types.SimpleNamespace(transformer = module, _unsloth_cuda_graphs = (handle,))
+ with monkeypatch.context() as m:
+ m.setitem(__import__("sys").modules, "torch", _Torch)
+ vsr.record_request_peak(pipe)
+
+ handle.poisoned = True # a failed capture poisoned the wrapper: later requests run eager
+ run()
+ assert vsr.measured_extra_mib(key, work) is None
+ handle.poisoned = False # graphs live: the request records
+ run()
+ assert vsr.measured_extra_mib(key, work) == 3000
+
+
+# --------------------------------------------------------------------------------------------------- review round 19
+
+
+def test_partially_quantized_text_encoders_neither_read_nor_record(klein_pipe):
+ import core.inference.diffusion as diffusion
+ from core.inference.diffusion_precision import TEQuantOutcome
+
+ full = TEQuantOutcome("int8", None)
+ partial = full._replace(partial = True)
+ assert diffusion._peak_te_label("int8", full) == "int8"
+ assert diffusion._peak_te_label(None, None) == "dense"
+ te = diffusion._peak_te_label("int8", partial)
+ label = dm.learned_peak_label(klein_pipe, "native", "m", "none", te, "off", "off")
+ assert not mp.identified(label) # a mixed build never reuses the fully quantised build's peak
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert "peak_te = _peak_te_label(te_quant, te_outcome)" in src
diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.py
index f8a0271beb5..1b81a742128 100644
--- a/studio/backend/tests/test_diffusion_measured_placement.py
+++ b/studio/backend/tests/test_diffusion_measured_placement.py
@@ -109,11 +109,9 @@ def test_24gb_keeps_transformer_and_text_encoder_resident(q21_pipe):
assert new.offload_policy == dm.OFFLOAD_GROUP
assert new.stream_text_encoders and not new.stream_transformer
assert new.resident_transformer_mib is None
- assert (
- new.resident_text_encoder_mib
- == 21432 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644 - 6922
- >= 8959
- )
+ # the room (21432 - 2304 - overhead - 644 - 6922) covers the whole encoder; the plan records the encoder's size
+ assert 21432 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644 - 6922 >= 8959
+ assert new.resident_text_encoder_mib == 8959
assert new.estimates["measured_runtime_headroom_mib"] == 2304
# never under-reserved: loaded weights + measured peak x margin + base overhead within the safe budget
assert 6922 + 8959 + 644 + 2304 + dm.DEFAULT_BASE_OVERHEAD_MIB <= 21432
@@ -402,7 +400,9 @@ def test_only_measured_family_moves(q21_pipe, speed, budget, total):
plan = _flat_plan(budget, total)
for family in names:
new = _refine(q21_pipe, plan, family = family, speed = speed)
- if family in ("qwen-image-2.1", "flux.1", "z-image") and speed in ("default", "max"):
+ # seeded: the peak table, or the per-phase calibration for a family the peak table lacks
+ seeded = set(dm._MEASURED_IMAGE_PEAK_MIB) | set(dm._MEASURED_IMAGE_ACTIVATION_MIB)
+ if family in seeded and speed in ("default", "max"):
continue
assert new is plan, family