Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: learned activation peaks, OOM retry in group offload setup, pinned GGUF host copies, tied-weight groups (#13724)
Daniel Han1f74450757b6
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 5cf395b322f..da47d3fe037 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -160,6 +160,12 @@ from .diffusion_memory import (refine_balanced_plan_for_components,refine_memory_plan_for_components,refine_plan_from_loaded_weights,+ learned_peak_label,+ live_vae_saver_tag,+ planned_vae_saver_tag,+ stream_window_pad_mib,+ with_label_part,+ applied_plan,release_resident_groups,resident_group_mib,hook_resident_denoiser,@@ -1962,6 +1968,288 @@ def _quadratic_attention(return True+def _attention_scaling_known(target: Any, engaged_backend: Optional[str] = None) -> bool:+ """Whether ``_quadratic_attention``'s answer is evidence: a non-native backend engaged, or the SDPA probe saw math+ alone or a sub-quadratic kernel run. An unanswered probe (or no device target) is unknown."""+ if engaged_backend is not None and str(engaged_backend) != "native":+ return True+ if target is None:+ return False+ try:+ return bool(sdpa_math_only(target) or sdpa_subquadratic_confirmed(target))+ except Exception: # noqa: BLE001+ return False+++def _start_peak_probe(state: Any) -> Any:+ """Arm the learned-peak measurement for one render (CUDA loads that carry a peak identity), else None."""+ pipe = getattr(state, "pipe", None)+ identity = getattr(pipe, "_unsloth_peak_identity", None)+ if identity is None:+ return None+ try:+ from .diffusion_measured_peaks import PeakProbe, enabled++ index = _peak_device_index(state)+ graphs = cuda_graph.live_graph_count(index)+ generation = cuda_graph.capture_generation(index)+ accounted = getattr(pipe, "_unsloth_peak_graphs", None)+ if accounted is None or graphs < accounted:+ # the load's own graphs (every render starts above them), or fewer after a reset freed some+ first = accounted is None+ pipe._unsloth_peak_graphs = accounted = graphs+ if first or graphs == 0:+ # nothing alive on the card: no unrecorded capture can remain. A drop to a non-zero count may be+ # another pipeline's reset, so a capture this one made unrecorded keeps the generation dirty+ pipe._unsloth_peak_graph_gen = generation+ if getattr(pipe, "_unsloth_peak_graph_gen", None) is None:+ pipe._unsloth_peak_graph_gen = generation+ if graphs > accounted or generation > pipe._unsloth_peak_graph_gen:+ # an unrecorded render (cache-served, refused, failed) left graphs allocated, or replaced one in a full+ # cache (same count): a later peak above that start omits memory a fresh process allocates on its first+ # render+ return None+ if _live_peak_identity(state) != tuple(identity):+ # this render runs a configuration the key does not name (a speed tier or step cache that toggled, a VAE+ # saver the guard engaged for it, another attention backend): its peak belongs to no key+ return None+ if not enabled():+ return None+ window_pad = stream_window_pad_mib(pipe)+ if window_pad is None:+ return (+ None # the streamed baseline is unknown: the peak cannot be put on the common scale+ )+ probe = PeakProbe(getattr(state, "device", None))+ if probe.device is None:+ return None+ probe.window_pad = window_pad+ probe.failed_captures = cuda_graph.failed_capture_count()+ probe.conditioning_hits = _conditioning_cache_hits(pipe)+ return probe+ except Exception: # noqa: BLE001 - measurement only+ return None+++def _peak_device_index(state: Any) -> Optional[int]:+ """CUDA ordinal the pipeline renders on (``state.device`` may be un-indexed: the thread's current card)."""+ try:+ import torch+ dev = torch.device(getattr(state, "device", None) or "cuda")+ return dev.index if dev.index is not None else torch.cuda.current_device()+ except Exception: # noqa: BLE001+ return None+++def _live_peak_identity(state: Any) -> tuple:+ """The load's learned-peak identity (the canonical configuration captured at load) with every part that can change+ after the load read from the live state: speed tier, step cache, VAE savers, attention backend. A render records+ only when this equals the identity."""+ pipe = getattr(state, "pipe", None)+ family, label, _ = tuple(getattr(pipe, "_unsloth_peak_identity", ("", "", None)))[:3]+ label = with_label_part(+ str(label), "cache", str(getattr(state, "transformer_cache", None) or "off")+ )+ label = with_label_part(label, "vae", live_vae_saver_tag(pipe))+ label = with_label_part(label, "attn", getattr(state, "attention_backend", None) or "native")+ if "|graphs:" in label:+ label = with_label_part(label, "graphs", _live_graph_mode(state))+ speed = getattr(state, "speed_mode", None)+ if "compile_fallback_eager" in tuple(getattr(state, "speed_optims", ()) or ()):+ speed = None # the compiled tier fell back to eager: an execution path no key names+ try:+ from .diffusion_bg_compile import eager_forced+ if eager_forced():+ speed = None # a background compile is still pending: this render runs eager under a compiled tier+ except Exception: # noqa: BLE001+ speed = None+ # a family that can run classifier-free guidance records only renders that do (the doubled denoiser batch)+ family_obj = getattr(state, "family", None)+ expects_cfg = bool(getattr(family_obj, "uses_negative_prompt", False))+ if bool(getattr(pipe, "_unsloth_peak_cfg", expects_cfg)) != expects_cfg:+ speed = None+ if getattr(pipe, "_unsloth_loras", None):+ speed = None # LoRA adapters add rank-dependent activations no key names+ return (family, label, speed)+++def _live_graph_mode(state: Any) -> str:+ """``step`` / ``blocks`` (``_engaged_graph_mode``) while the load's CUDA graphs are armed and not dropped+ (``live_status``), else ``off``."""+ try:+ _, optims = cuda_graph.live_status(+ getattr(state, "resolved", None),+ getattr(state, "speed_optims", ()),+ getattr(state, "cuda_graphs", ()),+ )+ if "cuda_graph" not in optims:+ return "off"+ return _engaged_graph_mode(getattr(state, "pipe", None), {"cuda_graph": True})+ except Exception: # noqa: BLE001+ return "unknown"+++def _peak_te_label(te_quant: Any, outcome: Any) -> str:+ """The text encoders' precision as the learned-peak key names it. A partial cast (some encoders quantised, others+ left dense) is no build the key can name: ``unknown``, so it neither reads nor records a learned peak."""+ if getattr(outcome, "partial", False):+ return "unknown"+ return str(te_quant or "dense")+++def _engaged_graph_mode(pipe: Any, applied: Any) -> str:+ """The graph implementation placement armed: whole-step and per-block graphs hold different statics, slot rings+ and pools, so their peaks never share a key."""+ if not (applied or {}).get("cuda_graph"):+ return "off"+ return "blocks" if getattr(pipe, "_unsloth_cuda_graph_mode", None) == "blocks" else "step"+++def _local_weights_digest(path: Any) -> Optional[str]:+ """Size and mtime of every weight file under a local pipeline folder (``checkpoint_fingerprint`` reads only its+ manifests); None for a repo id or an unreadable folder."""+ try:+ import hashlib++ root = Path(str(path)).expanduser()+ if not root.is_dir():+ return None+ digest = hashlib.sha256()+ for f in sorted(root.rglob("*")):+ if f.is_file() and f.suffix in (+ ".safetensors",+ ".bin",+ ".gguf",+ ".pt",+ ".pth",+ ".ckpt",+ ):+ st = f.stat()+ digest.update(f"{f.relative_to(root)}:{st.st_size}:{st.st_mtime_ns}\0".encode())+ return digest.hexdigest()[:16]+ except Exception: # noqa: BLE001+ return None+++def _peak_model_identity(+ repo_id: Any, gguf_filename: Optional[str], kind: Optional[str], base: Any+) -> str:+ """The exact build for the learned-peak key: the denoiser source and its weights' revision / fingerprint, plus the+ companion base and its revision for a GGUF / single-file load. ``unknown`` (no learned peak used or recorded)+ when a revision cannot be read."""+ from .diffusion_component_sizes import checkpoint_fingerprint++ source = f"{repo_id}/{gguf_filename}" if gguf_filename else str(repo_id)+ revision = checkpoint_fingerprint(repo_id, gguf_filename)+ if not revision:+ return "unknown"+ if str(revision).startswith("local:"):+ weights = _local_weights_digest(repo_id)+ if not weights:+ return "unknown"+ revision = f"{revision}:{weights}"+ identity = f"{source}@{revision}"+ if (kind or "pipeline") != "pipeline" and base and str(base) != str(repo_id):+ base_revision = checkpoint_fingerprint(base)+ if not base_revision:+ return "unknown"+ if str(base_revision).startswith("local:"):+ weights = _local_weights_digest(base)+ if not weights:+ return "unknown"+ base_revision = f"{base_revision}:{weights}"+ identity += f"+{base}@{base_revision}"+ return identity+++def _conditioning_cache_hits(pipe: Any) -> int:+ """Encodes served from the in-memory prompt cache or the persistent conditioning cache so far: a render with a+ hit skipped the text-encoder phase, so its peak covers only part of what a new prompt needs."""+ hits = 0+ try:+ cache = prompt_cache.cache_for(pipe)+ if cache is not None:+ hits += int(cache.stats.get("hits", 0))+ stats = getattr(pipe, "_unsloth_cond_cache_stats", None)+ if isinstance(stats, dict):+ hits += int(stats.get("hits", 0))+ except Exception: # noqa: BLE001 - measurement only+ pass+ return hits+++def _record_peak_probe(+ state: Any,+ probe: Any,+ *,+ width: Optional[int],+ height: Optional[int],+ batch_size: int,+ target: Any = None,+) -> None:+ """Persist a completed render's peak above its starting allocation under the load's peak identity."""+ if probe is None or not width or not height:+ return+ identity = getattr(getattr(state, "pipe", None), "_unsloth_peak_identity", None)+ try:+ from .diffusion_measured_peaks import device_name, record++ peak = probe.finish()+ if peak is None or not isinstance(identity, tuple) or len(identity) != 3:+ return+ if _live_peak_identity(state) != tuple(identity):+ # the configuration moved during the render (a compile fallback, a toggled cache): no key names it+ return+ if cuda_graph.failed_capture_count() != getattr(probe, "failed_captures", 0):+ # a capture failed (an absorbed OOM): the peak is the failed attempt's, not the render's need+ return+ if _conditioning_cache_hits(state.pipe) != getattr(probe, "conditioning_hits", 0):+ # a cached prompt skipped the text encoder: the peak misses that phase, and an unseeded family places+ # its next load from this value alone+ return+ engaged = getattr(state, "attention_backend", None)+ if not _attention_scaling_known(target, engaged):+ # the SDPA probe could not tell math from a fused kernel: a quadratic peak stored as linear would+ # under-size every larger request+ return+ family, quant, speed_mode = identity+ pipe = state.pipe+ index = getattr(probe.device, "index", None)+ graphs = cuda_graph.live_graph_count(index)+ generation = cuda_graph.capture_generation(index)+ peak += int(getattr(probe, "window_pad", 0) or 0)+ held: list = []+ raised = record(+ family = family,+ quant = quant,+ speed_mode = speed_mode,+ width = width,+ height = height,+ batch_size = batch_size,+ peak_mib = peak,+ device_name = device_name(probe.device),+ quadratic = _quadratic_attention(target, engaged, state.pipe),+ held = held,+ )+ if held:+ # the store holds this sample, so it covers the graphs the render recorded; unpersisted, they stay dirty+ pipe._unsloth_peak_graphs = graphs+ pipe._unsloth_peak_graph_gen = generation+ if raised:+ logger.info(+ "diffusion.memory: learned activation peak %d MiB recorded for %s / %s / %s at %dx%d x%d",+ peak,+ family,+ quant,+ speed_mode,+ int(width),+ int(height),+ int(batch_size),+ )+ except Exception as exc: # noqa: BLE001 - recording must never fail a finished render+ logger.debug("diffusion.memory: learned activation peak not recorded (%s)", exc)++def _activation_guard_batch(chunks: Sequence[Sequence[Any]]) -> int:"""The batch size the generate-time activation guard budgets for.@@ -8206,6 +8494,45 @@ class DiffusionBackend:"; ".join(refined_plan.reasons),)plan = refined_plan+ peak_model = _peak_model_identity(repo_id, gguf_filename, kind, base)+ if getattr(pipe, "_unsloth_loras", None):+ peak_model = (+ "unknown" # adapters baked in at load: a build the key cannot name+ )+ try:+ # the phase-table seed holds only where it was measured+ pipe._unsloth_phase_seeds = _calibrated_activation(fam, target) is not None+ except Exception: # noqa: BLE001+ pass+ peak_vae = planned_vae_saver_tag(pipe, plan)+ peak_te = _peak_te_label(te_quant, te_outcome)+ # an auto step cache toggles per request: only uncached renders (the full transformer) record+ peak_cache = "off" if cache_may_toggle else str(cache_engaged or "off")+ # graph statics and pools sit above a graph-free peak; a graph armed after an offloaded placement+ # counts as on+ peak_graphs = (+ "on"+ if speed_applied.get("cuda_graph")+ or getattr(pipe, "_unsloth_cuda_graph_after_placement", False)+ else "off"+ )+ try:+ # the learned-peak key: recorded at every completed render, read by later loads+ pipe._unsloth_peak_identity = (+ fam.name,+ learned_peak_label(+ pipe,+ attention_engaged or "native",+ peak_model,+ peak_vae,+ peak_te,+ peak_cache,+ peak_graphs,+ ),+ effective_speed,+ )+ except Exception: # noqa: BLE001 - recording is optional+ pass# not under the legacy cpu_offload flag alone: that request asked for offloadif not cpu_offload or normalize_memory_mode(memory_mode) is not None:plan = refine_plan_from_loaded_weights(@@ -8214,6 +8541,12 @@ class DiffusionBackend:family = fam.name,speed_mode = effective_speed,logger = logger,+ attention = attention_engaged or "native",+ model = peak_model,+ vae = peak_vae,+ te = peak_te,+ cache = peak_cache,+ graphs = peak_graphs,)headroom = plan.estimates.get("measured_runtime_headroom_mib")if headroom and (@@ -8225,6 +8558,9 @@ class DiffusionBackend:effective_speed,plan.estimates.get("measured_dense_transformer_mib"),plan.estimates.get("measured_compute_bytes"),+ plan.estimates.get("measured_learned_peak_mib"),+ plan.estimates.get("measured_learned_peak_pixels"),+ plan.estimates.get("measured_learned_peak_quadratic"),)from .diffusion_qwenimage21_vision import configure_vision_attention@@ -8271,6 +8607,16 @@ class DiffusionBackend:placement_device = target.torch_device,logger = logger,)+ # a transient OOM during setup may have re-planned to lower residency+ plan = applied_plan(pipe, plan)+ identity = getattr(pipe, "_unsloth_peak_identity", None)+ if isinstance(identity, tuple) and len(identity) == 3:+ # a saver the plan asked for may not have engaged: key the renders by what did+ live_vae = with_label_part(identity[1], "vae", live_vae_saver_tag(pipe))+ if live_vae != identity[1]:+ pipe._unsloth_peak_identity = (identity[0], live_vae, identity[2])+ # placed from the planned saver's peak: requests size from the flat estimate instead+ pipe._unsloth_peak_static = True# streams the whole-resident denoiser back to the flat room while the encoders runinstall_encode_release(pipe, plan, logger)# the speed layer saw only the plan; placement may have pinned every denoiser group since@@ -8294,6 +8640,20 @@ class DiffusionBackend:speed_mode = effective_speed,logger = logger,)+ identity = getattr(pipe, "_unsloth_peak_identity", None)+ if (+ isinstance(identity, tuple)+ and len(identity) == 3+ and "graphs:on" in str(identity[1]).split("|")+ ):+ # placed from the larger of both graph modes' peaks: renders record under the one that armed+ pipe._unsloth_peak_identity = (+ identity[0],+ with_label_part(+ identity[1], "graphs", _engaged_graph_mode(pipe, speed_applied)+ ),+ identity[2],+ )# Per-control provenance for status. cpu_offload=False is the unset default, so only True is# explicit.@@ -10354,6 +10714,21 @@ class DiffusionBackend:target = target,)object.__setattr__(state, "attention_backend", attention_engaged)+ identity = getattr(state.pipe, "_unsloth_peak_identity", None)+ if isinstance(identity, tuple) and len(identity) == 3:+ # later renders record under the backend that now runs+ try:+ # the profile engages the default tier too (state.speed_mode below)+ state.pipe._unsloth_peak_identity = (+ identity[0],+ with_label_part(identity[1], "attn", attention_engaged or "native"),+ SPEED_DEFAULT,+ )+ except Exception: # noqa: BLE001 - recording is optional+ pass+ # the reserve was placed for the load's tier: until a reload re-places it, requests size from the flat+ # estimate+ state.pipe._unsloth_peak_static = True# Keep the badge in step with the state it describes: the top-level field moved, so leaving `resolved` on the# load-time value would make the panel report an attention backend that is no longer the one running.if isinstance(state.resolved, dict) and "attention_backend" in state.resolved:@@ -10932,6 +11307,8 @@ class DiffusionBackend:# cap; the OOM backoff below halves a failed chunk. Layered: one input per forward (the pipeline's# per-prompt layer stride is only right for one).chunks = [[job] for job in jobs] if layer_count else chunk_jobs(jobs, batch_size)+ guard_width = guard_height = guard_target = None+ guard_batch = 1try:# The dimensions the forward runs at, not the sliders: img2img / inpaint / upscale / edit take@@ -11169,6 +11546,11 @@ class DiffusionBackend:chunk_shapes: list[int] = []graphs_before = fresh_compile_count()compile_cache.note_use(state.compile_cache_ctx)+ try:+ state.pipe._unsloth_peak_cfg = _negative_prompt_engaged(state.family, guidance)+ except Exception: # noqa: BLE001 - an unknown guidance mode records nothing+ state.pipe._unsloth_peak_cfg = None+ peak_probe = _start_peak_probe(state)try:previewer = _start_previewer()pending = list(chunks)@@ -11290,6 +11672,7 @@ class DiffusionBackend:empty_cache()first_half, second_half = split_chunk(chunk)pending[:0] = [first_half, second_half]+ peak_probe = None # the failed attempt's peak is the card's limit, not this request'sgen.total_steps += steps # one extra chunk to runlogger.warning("diffusion.generate: batch of %d hit OOM; retrying as %d + %d",@@ -11331,6 +11714,14 @@ class DiffusionBackend:raiseif previewer is not None:previewer.finish()+ _record_peak_probe(+ state,+ peak_probe,+ width = guard_width,+ height = guard_height,+ batch_size = max(chunk_shapes or [1]),+ target = guard_target,+ )if static_skip:logger.debug("diffusion.step_skip: %s", static_skip_stats(state.pipe))# Keep progress ACTIVE through the post-denoise work: the route persists the image after this returns,diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.pyindex 371961f3e23..bea93715035 100644--- a/studio/backend/core/inference/diffusion_block_graph.py+++ b/studio/backend/core/inference/diffusion_block_graph.py@@ -371,6 +371,18 @@ def _meta(t: Any) -> tuple:return (tuple(t.shape), tuple(t.stride()), t.dtype, t.device, bool(t.is_inference()))+_LIVE_BLOCKS: "weakref.WeakSet" = weakref.WeakSet()+++def live_block_graph_count(device_index: Optional[int]) -> int:+ """Per-block graphs alive on CUDA device ``device_index`` (``diffusion_cuda_graph.live_graph_count``)."""+ return sum(+ len(b.cache)+ for b in tuple(_LIVE_BLOCKS)+ if getattr(b, "_unsloth_capture_device", None) == device_index+ )++class BlockGraph:"""Replaces one block's compute callable (below its offload hooks) with a recording keyed by inputs and weightplacement. Any refusal or failure runs ``compute`` itself, so the block is never worse than ungraphed."""@@ -594,6 +606,11 @@ class BlockGraph:self.stats["recaptures"] += 1self.cache[full] = entryself.stats["captures"] += 1+ from .diffusion_cuda_graph import capture_device, note_capture++ self._unsloth_capture_device = capture_device()+ note_capture(self._unsloth_capture_device)+ _LIVE_BLOCKS.add(self)while len(self.cache) > self.max_graphs:_, evicted = self.cache.popitem(last = False)self._forget(evicted)diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex c71733fc17e..3b0238b5e45 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -300,6 +300,50 @@ def captures_exhausted() -> bool:return len(_FAILED_GRAPHS) >= MAX_FAILED_CAPTURES+def capture_device() -> Optional[int]:+ """The CUDA ordinal a capture records on (the current device), or None when unreadable."""+ try:+ return int(_torch().cuda.current_device())+ except Exception: # noqa: BLE001+ return None+++def live_graph_count(device_index: Optional[int]) -> int:+ """Graphs alive on CUDA device ``device_index`` (whole-step and per-block). Their statics and pool stay allocated+ after the render that recorded them, so a later render's start allocation includes them; a reset frees them."""+ total = sum(+ len(w.cache)+ for w in tuple(_LIVE_WRAPPERS)+ if getattr(w, "_unsloth_capture_device", None) == device_index+ )+ from .diffusion_block_graph import live_block_graph_count++ return total + live_block_graph_count(device_index)+++_CAPTURES: dict = {}+# image and video render threads capture concurrently: an unserialized read-modify-write could lose an increment+_CAPTURES_LOCK = threading.Lock()+++def note_capture(device_index: Optional[int]) -> None:+ """Count a recorded graph (whole-step or per-block) on ``device_index``; see ``capture_generation``."""+ with _CAPTURES_LOCK:+ _CAPTURES[device_index] = _CAPTURES.get(device_index, 0) + 1+++def capture_generation(device_index: Optional[int]) -> int:+ """Graphs ever recorded on ``device_index``. A full cache evicts before it captures, so a replacement leaves+ ``live_graph_count`` unchanged; this count still moves."""+ return _CAPTURES.get(device_index, 0)+++def failed_capture_count() -> int:+ """Captures that failed in this process (an absorbed OOM among them): a render that saw one grow peaked at the+ failed attempt, not at what it needed."""+ return len(_FAILED_GRAPHS)++def refuse_if_exhausted(logger: Any = None) -> None:"""Raise (the caller's failed-capture path then runs eager) once the failure budget is spent; log that once."""global _EXHAUSTED_LOGGED@@ -1130,6 +1174,8 @@ class GraphedForward:return self._eager(eager_args, eager_kwargs)entry = capturedself.cache[key] = entry+ self._unsloth_capture_device = capture_device()+ note_capture(self._unsloth_capture_device)if self.logger is not None:self.logger.debug("diffusion.cuda_graph: captured %s graph %d/%d over %d input tensor(s)",diff --git a/studio/backend/core/inference/diffusion_measured_peaks.py b/studio/backend/core/inference/diffusion_measured_peaks.pynew file mode 100644index 00000000000..20486e6232b--- /dev/null+++ b/studio/backend/core/inference/diffusion_measured_peaks.py@@ -0,0 +1,523 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Learned activation peaks: the real CUDA peak above the resident weights, recorded per+(family, denoiser quant, speed tier, resolution bucket, frames, GPU name) at a successful render and persisted under+the Studio cache dir, so later loads of the same model place it from a measurement instead of the flat+8192 MiB/MP estimate.++The static tables in ``diffusion_memory`` stay the seed: a learned value only RAISES a seeded peak (a local+measurement above the seed means this machine needs more) and supplies one where no seed exists. It never lowers a+seed, because one render does not exercise every phase (a prompt-cache hit skips the encoder, a smaller request+decodes less), and the seed was measured on the worst phase.++``UNSLOTH_DIFFUSION_LEARNED_PEAKS=0`` turns lookups and recording off; ``UNSLOTH_DIFFUSION_LEARNED_PEAKS_FILE``+names the JSON file (default ``<studio cache>/diffusion_measured_peaks.json``).+"""++from __future__ import annotations++import contextlib+import json+import math+import os+import tempfile+import threading+import weakref+from pathlib import Path+from typing import Any, Optional++LEARNED_PEAKS_ENV = "UNSLOTH_DIFFUSION_LEARNED_PEAKS"+LEARNED_PEAKS_FILE_ENV = "UNSLOTH_DIFFUSION_LEARNED_PEAKS_FILE"+_FILE_NAME = "diffusion_measured_peaks.json"+_VERSION = 1+_MAX_ENTRIES = 512+_MEGAPIXEL = 1024 * 1024+# Smallest bucket: a quarter megapixel. Below it the activation is dominated by fixed costs.+_MIN_BUCKET_PIXELS = _MEGAPIXEL // 4++_lock = threading.Lock()+_cache: dict[str, Any] = {"path": None, "mtime": None, "data": None}+++def enabled() -> bool:+ return (os.environ.get(LEARNED_PEAKS_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def store_path() -> Optional[Path]:+ explicit = (os.environ.get(LEARNED_PEAKS_FILE_ENV) or "").strip()+ if explicit:+ return Path(explicit)+ try:+ from utils.paths.storage_roots import cache_root+ except Exception: # noqa: BLE001 - imported without studio/backend on sys.path: nowhere safe to write+ return None+ try:+ return cache_root() / _FILE_NAME+ except Exception: # noqa: BLE001+ return None+++def resolution_bucket(+ width: Optional[int],+ height: Optional[int],+ batch_size: int = 1,+) -> int:+ """Pixels of the power-of-two megapixel bucket covering ``width x height x batch`` (at least 1/4 MP)."""+ pixels = max(1, int(width or 1024)) * max(1, int(height or 1024)) * max(1, int(batch_size or 1))+ if pixels <= _MIN_BUCKET_PIXELS:+ return _MIN_BUCKET_PIXELS+ return int(_MIN_BUCKET_PIXELS * 2 ** math.ceil(math.log2(pixels / _MIN_BUCKET_PIXELS)))+++def model_key(+ family: Optional[str],+ quant: Optional[str],+ speed_mode: Optional[str],+ frames: int = 1,+ device: Optional[str] = None,+) -> str:+ """``device``: the GPU's name. Kernels and workspaces differ by card, so a heterogeneous host keeps one set of+ peaks per card model instead of placing one GPU from another's measurement."""+ return "|".join(+ (+ str(family or "unknown").lower(),+ str(quant or "unknown"),+ str(speed_mode or "unknown"),+ f"f{max(1, int(frames or 1))}",+ str(device or "unknown"),+ )+ )+++_runtime: list = []+_KERNEL_DISTS = (+ "triton",+ "torchao",+ "flash-attn",+ "flash_attn_3",+ "xformers",+ "sageattention",+ "kernels",+)+++def runtime_stamp() -> str:+ """The runtimes a peak depends on (kernel choice, workspaces): a store written under others is ignored."""+ if not _runtime:+ from importlib.metadata import version++ parts = []+ for dist in ("torch", "diffusers"):+ try:+ parts.append(f"{dist}={version(dist)}")+ except Exception: # noqa: BLE001+ parts.append(f"{dist}=?")+ # attention and quantization kernels (their workspaces are part of a peak); an absent one is left out+ for dist in _KERNEL_DISTS:+ try:+ parts.append(f"{dist}={version(dist)}")+ except Exception: # noqa: BLE001+ pass+ try:+ import torch+ parts.append(f"cuda={torch.version.cuda or torch.version.hip or '?'}")+ except Exception: # noqa: BLE001+ parts.append("cuda=?")+ _runtime.append("|".join(parts))+ return _runtime[0]+++def _empty() -> dict:+ return {"version": _VERSION, "runtime": runtime_stamp(), "entries": {}}+++def _read(path: Path, fresh: bool = False) -> dict:+ try:+ mtime = path.stat().st_mtime_ns+ except OSError:+ return _empty()+ if (+ not fresh+ and _cache["path"] == str(path)+ and _cache["mtime"] == mtime+ and _cache["data"] is not None+ ):+ return _cache["data"]+ try:+ data = json.loads(path.read_text())+ if (+ not isinstance(data, dict)+ or data.get("version") != _VERSION+ or data.get("runtime") != runtime_stamp()+ or not isinstance(data.get("entries"), dict)+ ):+ data = _empty()+ except Exception: # noqa: BLE001 - a corrupt file is an empty store, never a failed load+ data = _empty()+ _cache.update(path = str(path), mtime = mtime, data = data)+ return data+++def _file_lock(path: Path):+ """Exclusive lock on ``<store>.lock`` across processes sharing the cache root (two Studio servers, a worker), so a+ read / merge / replace cannot overwrite another process's higher peak. Advisory; held only for the merge."""+ path.parent.mkdir(parents = True, exist_ok = True)+ handle = open(str(path) + ".lock", "a+b")+ try:+ try:+ import fcntl+ fcntl.flock(handle.fileno(), fcntl.LOCK_EX)+ except ImportError: # Windows+ import msvcrt+ handle.seek(0)+ msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)+ yield+ finally:+ handle.close() # closing releases either lock+++def _write(path: Path, data: dict) -> None:+ path.parent.mkdir(parents = True, exist_ok = True)+ fd, tmp = tempfile.mkstemp(prefix = ".peaks.", dir = str(path.parent))+ try:+ with os.fdopen(fd, "w") as handle:+ json.dump(data, handle, indent = 1) # insertion order is the eviction order+ os.replace(tmp, path)+ except BaseException:+ try:+ os.unlink(tmp)+ except OSError:+ pass+ raise+ _cache.update(path = None, mtime = None, data = None)+++def record(+ *,+ family: Optional[str],+ quant: Optional[str],+ speed_mode: Optional[str],+ width: Optional[int],+ height: Optional[int],+ batch_size: int = 1,+ frames: int = 1,+ peak_mib: int,+ device_name: Optional[str] = None,+ quadratic: bool = False,+ held: Optional[list] = None,+) -> bool:+ """Keep the max peak seen for this key and bucket, scaled to the bucket's ceiling so a smaller render in the bucket+ still answers a full-size one (squared with ``quadratic``: SDPA math attention grows with pixels squared, and the+ key's entry keeps scaling that way). Returns True when the stored value rose. ``held`` (a list) gets True appended+ when the store holds at least this peak afterwards (raised, or already covered): False when nothing was persisted."""+ if (+ not enabled()+ or peak_mib is None+ or int(peak_mib) <= 0+ or not family+ or not identified(quant)+ ):+ return False+ if quadratic and int(batch_size or 1) > 1:+ return False # a math-attention entry holds single images: it scales squared in pixels, linearly in batch+ path = store_path()+ if path is None:+ return False+ device_name = device_name or current_device_name()+ key = model_key(family, quant, speed_mode, frames, device_name)+ bucket_pixels = resolution_bucket(width, height, batch_size)+ bucket = str(bucket_pixels)+ pixels = max(1, int(width or 1024)) * max(1, int(height or 1024)) * max(1, int(batch_size or 1))+ peak_mib = int(+ math.ceil(+ int(peak_mib)+ * (bucket_pixels / float(max(pixels, bucket_pixels // 2))) ** (2 if quadratic else 1)+ )+ )+ try:+ with _lock, _file_lock(path):+ data = _read(path, fresh = True)+ entries = data.setdefault("entries", {})+ entry = entries.pop(key, None) or {"peaks": {}}+ entries[key] = entry # most recently raised last: the cap evicts from the front+ peaks = entry.setdefault("peaks", {})+ old = int(peaks.get(bucket) or 0)+ if int(peak_mib) <= old:+ if held is not None:+ held.append(True)+ return False+ peaks[bucket] = int(peak_mib)+ if quadratic:+ entry["quadratic"] = True+ if device_name:+ entry["device"] = str(device_name) # also in the key; kept for readers of the file+ if len(entries) > _MAX_ENTRIES:+ for stale in list(entries)[: len(entries) - _MAX_ENTRIES]:+ entries.pop(stale, None)+ _write(path, data)+ if held is not None:+ held.append(True)+ return True+ except Exception: # noqa: BLE001 - persistence is best-effort+ return False+++def lookup(+ *,+ family: Optional[str],+ quant: Optional[str],+ speed_mode: Optional[str],+ width: Optional[int] = None,+ height: Optional[int] = None,+ batch_size: int = 1,+ frames: int = 1,+ device: Optional[str] = None,+) -> Optional[int]:+ """Learned peak MiB (no margin) for a request of this size, or None when nothing was recorded.+ ``device`` (GPU name) defaults to the current CUDA device, the one the planner snapshots.++ The smallest recorded bucket at or above the request's bucket answers as is (an overestimate); with only smaller+ buckets recorded, the largest one is scaled linearly by pixel count."""+ found = lookup_bucket(+ family = family,+ quant = quant,+ speed_mode = speed_mode,+ width = width,+ height = height,+ batch_size = batch_size,+ frames = frames,+ device = device,+ )+ return None if found is None else found[0]+++def identified(quant: Optional[str]) -> bool:+ """False when the label or a part of it could not be identified (empty, a bare ``unknown`` format,+ ``name:unknown``, a ``#unknown`` quant variant): such a load neither reads nor records a learned peak and keeps the static estimate."""+ parts = str(quant or "").split("|")+ return bool(parts[0]) and not any(+ part == "unknown" or part.endswith((":unknown", "#unknown")) for part in parts+ )+++def lookup_bucket(+ *,+ family: Optional[str],+ quant: Optional[str],+ speed_mode: Optional[str],+ width: Optional[int] = None,+ height: Optional[int] = None,+ batch_size: int = 1,+ frames: int = 1,+ device: Optional[str] = None,+) -> Optional[tuple[int, int]]:+ """``lookup`` plus the pixels the peak covers and whether it scales squared: (peak MiB, bucket pixels,+ quadratic), or None. A ``graphs:on`` label (a load placed before it knows whether placement keeps the whole-step+ graph or records per block) reads an envelope over the ``graphs:step`` and ``graphs:blocks`` entries."""+ if "graphs:on" in str(quant or "").split("|"):+ found = [+ lookup_bucket(+ family = family,+ quant = "|".join(+ f"graphs:{mode}" if part == "graphs:on" else part+ for part in str(quant).split("|")+ ),+ speed_mode = speed_mode,+ width = width,+ height = height,+ batch_size = batch_size,+ frames = frames,+ device = device,+ )+ for mode in ("step", "blocks")+ ]+ found = [f for f in found if f is not None]+ if not found:+ return None+ # one envelope over both modes (the placement can arm either): the larger peak over the smaller covered+ # bucket, scaled squared if either mode is, sizes every later request at least as large as either mode would+ return (+ max(f[0] for f in found),+ min(f[1] for f in found),+ any(f[2] for f in found),+ )+ if not enabled() or not family or not identified(quant):+ return None+ path = store_path()+ if path is None:+ return None+ try:+ with _lock:+ key = model_key(family, quant, speed_mode, frames, device or current_device_name())+ entry = (_read(path).get("entries") or {}).get(key)+ peaks = {+ int(k): int(v) for k, v in ((entry or {}).get("peaks") or {}).items() if int(v) > 0+ }+ except Exception: # noqa: BLE001+ return None+ if not peaks:+ return None+ quadratic = bool((entry or {}).get("quadratic"))+ batch = max(1, int(batch_size or 1))+ # a math-attention entry holds single images (``record``): its buckets are per image, the batch scales linearly+ want = resolution_bucket(width, height, 1 if quadratic else batch)+ times = batch if quadratic else 1+ above = [b for b in peaks if b >= want]+ if above:+ return peaks[min(above)] * times, min(above), quadratic+ largest = max(peaks)+ power = 2 if quadratic else 1+ return (+ int(math.ceil(peaks[largest] * (want / float(largest)) ** power)) * times,+ want,+ quadratic,+ )+++def record_work(+ key: Any,+ work: int,+ peak_mib: int,+ held: Optional[list] = None,+) -> bool:+ """Persist a peak under an opaque calibration key (the video path's work = pixels x frames table); max wins.+ ``held`` as in ``record``."""+ if not enabled() or key is None or peak_mib is None or int(peak_mib) <= 0 or int(work) <= 0:+ return False+ path = store_path()+ if path is None:+ return False+ try:+ with _lock, _file_lock(path):+ data = _read(path, fresh = True)+ table = data.setdefault("work", {})+ peaks = table.pop(repr(key), None) or {}+ table[repr(key)] = peaks+ old = int(peaks.get(str(int(work))) or 0)+ if int(peak_mib) <= old:+ if held is not None:+ held.append(True)+ return False+ peaks[str(int(work))] = int(peak_mib)+ if len(table) > _MAX_ENTRIES:+ for stale in list(table)[: len(table) - _MAX_ENTRIES]:+ table.pop(stale, None)+ _write(path, data)+ if held is not None:+ held.append(True)+ return True+ except Exception: # noqa: BLE001 - persistence is best-effort+ return False+++def work_peaks(key: Any) -> dict[int, int]:+ """Persisted {work: peak MiB} for an opaque calibration key ({} when none or disabled)."""+ if not enabled() or key is None:+ return {}+ path = store_path()+ if path is None:+ return {}+ try:+ with _lock:+ peaks = (_read(path).get("work") or {}).get(repr(key)) or {}+ return {int(w): int(v) for w, v in peaks.items() if int(v) > 0}+ except Exception: # noqa: BLE001+ return {}+++class _Window:+ overlapped = False+++_windows_lock = threading.Lock()+_windows: dict[str, "weakref.WeakSet[_Window]"] = {}+++def open_window(device: Any) -> _Window:+ """Register a peak measurement on ``device``. Resetting the peak is allocator-wide, so a second render arming on+ the same card (image and video run on separate threads) spoils both: each one's window reads as overlapped. An+ abandoned window drops out once nothing references it."""+ window = _Window()+ with _windows_lock:+ live = _windows.setdefault(str(device), weakref.WeakSet())+ if len(live):+ window.overlapped = True+ for other in live:+ other.overlapped = True+ live.add(window)+ return window+++def close_window(device: Any, window: Optional[_Window]) -> bool:+ """End a measurement; True when no other measurement on the card overlapped it."""+ if window is None:+ return True+ with _windows_lock:+ live = _windows.get(str(device))+ if live is not None:+ live.discard(window)+ return not window.overlapped+++class PeakProbe:+ """CUDA peak above the allocation at ``start``; ``finish`` returns MiB or None (non-CUDA / unreadable, or another+ measurement on the card reset the peak meanwhile)."""++ def __init__(self, device: Any) -> None:+ self.device = None+ self.base = None+ try:+ import torch++ dev = torch.device(device)+ if dev.type != "cuda" or not torch.cuda.is_available():+ return+ if dev.index is None:+ dev = torch.device("cuda", torch.cuda.current_device())+ self.window = open_window(dev)+ torch.cuda.synchronize(dev)+ torch.cuda.reset_peak_memory_stats(dev)+ self.base = int(torch.cuda.memory_allocated(dev))+ self.device = dev+ except Exception: # noqa: BLE001 - measurement only+ self.device = None++ def finish(self) -> Optional[int]:+ if self.device is None or self.base is None:+ return None+ try:+ import torch+ torch.cuda.synchronize(self.device)+ peak = int(torch.cuda.max_memory_allocated(self.device)) - self.base+ except Exception: # noqa: BLE001+ peak = None+ # closed only after the read: a render arming in between would otherwise reset the peak unseen+ if not close_window(self.device, getattr(self, "window", None)) or peak is None:+ return None+ return -(-peak // (1024 * 1024)) if peak > 0 else None+++def current_device_name() -> Optional[str]:+ """Name of the current CUDA device (None without CUDA); the planner's snapshot reads this device."""+ try:+ import torch+ if not torch.cuda.is_available():+ return None+ except Exception: # noqa: BLE001+ return None+ return device_name("cuda")+++def device_name(device: Any) -> Optional[str]:+ try:+ import torch+ return str(torch.cuda.get_device_name(torch.device(device)))+ except Exception: # noqa: BLE001+ return Nonediff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 37f53f03b86..28de185be7a 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -2284,37 +2284,325 @@ def measured_image_runtime_mib(batch_size: int = 1,dense_transformer_mib: Optional[int] = None,compute_bytes: int = 2,+ quant: Optional[str] = None,+ phase_seeds: bool = False,) -> Optional[int]:"""Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate).- ``dense_transformer_mib`` selects the dense eager table; ``compute_bytes`` scales it (fp32 Qwen-Image: 8480 vs 4248)."""+ ``dense_transformer_mib`` selects the dense eager table; ``compute_bytes`` scales it (fp32 Qwen-Image: 8480 vs 4248).++ Seeds: the peak tables above, then the per-phase calibration (``_MEASURED_IMAGE_ACTIVATION_MIB``, the same+ quantity per phase) for a quantised family the peak table lacks, only with ``phase_seeds`` (the target is where+ those were measured: NVIDIA with a sub-quadratic attention kernel, ``_calibrated_activation``). ``quant`` (``denoiser_quant_label``) adds the+ peak this machine recorded for (family, quant, speed tier, resolution bucket) at an earlier render+ (diffusion_measured_peaks): it raises a seed and stands in for a missing one, never lowers a seed."""if _env_off(MEASURED_ACTIVATION_ENV):return None+ seed: Optional[int] = Noneif dense_transformer_mib is not None:if _env_off(MEASURED_ACTIVATION_DENSE_ENV):return Noneentry = _MEASURED_DENSE_EAGER_PEAK_MIB.get(str(family or "").lower())- if entry is None or str(speed_mode or "") not in _MEASURED_DENSE_SPEED_MODES:- return None- peak, max_dit = entry- widen = max(2, int(compute_bytes or 2))- peak, max_dit = peak * widen // 2, max_dit * widen // 2- if int(dense_transformer_mib) <= 0 or int(dense_transformer_mib) > max_dit:- return None- else:- peak = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())- if peak is None or str(speed_mode or "") not in _MEASURED_PEAK_SPEED_MODES:- return None+ if entry is not None and str(speed_mode or "") in _MEASURED_DENSE_SPEED_MODES:+ peak, max_dit = entry+ widen = max(2, int(compute_bytes or 2))+ peak, max_dit = peak * widen // 2, max_dit * widen // 2+ if 0 < int(dense_transformer_mib) <= max_dit:+ seed = peak+ elif str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES:+ seed = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())+ if seed is None and phase_seeds:+ phases = _MEASURED_IMAGE_ACTIVATION_MIB.get(str(family or "").lower())+ if phases is not None:+ # worst of encoder / denoise / untiled decode at 1024x1024 for this tier+ seed = max(phases[1 if speed_mode == "max" else 0][:3])w = max(64, int(width or DEFAULT_IMAGE_WIDTH))h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))- scale = max(- 1.0,- (w * h * max(1, int(batch_size or 1))) / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT),- )- need = peak * scale * _MEASURED_PEAK_MARGIN+ batch = max(1, int(batch_size or 1))+ learned = None+ if quant is not None:+ from .diffusion_measured_peaks import lookup as learned_peak+ learned = learned_peak(+ family = family, quant = quant, speed_mode = speed_mode, width = w, height = h, batch_size = batch+ )+ if seed is None and learned is None:+ return None+ scale = max(1.0, (w * h * batch) / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT))+ need = max(float(seed or 0) * scale, float(learned or 0)) * _MEASURED_PEAK_MARGINstep = _MEASURED_PEAK_ROUND_MIBreturn int(-(-need // step) * step)+def learned_peak_label(+ pipe: Any,+ attention: Optional[str] = None,+ model: Optional[str] = None,+ vae: Optional[str] = None,+ te: Optional[str] = None,+ cache: Optional[str] = None,+ graphs: Optional[str] = None,+) -> str:+ """``denoiser_quant_label`` plus the loaded model (variants of one family differ in size), the text encoders'+ precision (a dense streamed encoder peaks above a quantised one), the VAE savers (an untiled decode peaks far+ above a tiled one) and the engaged attention backend (backends with different workspaces): none shares a learned+ peak. The VAE savers and the backend can change after the load; ``with_label_part`` rewrites them."""+ label = denoiser_quant_label(pipe)+ for name, value in (+ ("model", model),+ ("te", te),+ ("cache", cache),+ ("vae", vae),+ ("attn", attention),+ ("graphs", graphs),+ ):+ if value is not None:+ label = f"{label}|{name}:{value}"+ return label+++def with_label_part(label: str, name: str, value: str) -> str:+ """``label`` with its ``name:`` part replaced by ``value`` (unchanged when it has no such part)."""+ return "|".join(+ f"{name}:{value}" if part.startswith(f"{name}:") else part for part in str(label).split("|")+ )+++_VAE_SAVERS = (+ ("use_tiling", "enable_vae_tiling", "enable_tiling"),+ ("use_slicing", "enable_vae_slicing", "enable_slicing"),+)+++def _vae_saver_can(pipe: Any, pipe_method: str, vae_method: str) -> bool:+ return callable(getattr(pipe, pipe_method, None)) or callable(+ getattr(getattr(pipe, "vae", None), vae_method, None)+ )+++def _vae_savers_observable(pipe: Any) -> bool:+ """False when a saver can be engaged but the VAE keeps no ``use_*`` flag to read it back: its state is unknown."""+ vae = getattr(pipe, "vae", None)+ return all(+ hasattr(vae, flag) or not _vae_saver_can(pipe, pm, vm) for flag, pm, vm in _VAE_SAVERS+ )+++def planned_vae_saver_tag(pipe: Any, plan: Any) -> str:+ """The VAE savers ``plan`` asks for that this pipeline can engage (what ``apply_memory_plan`` will turn on);+ ``unknown`` (no learned peak used or recorded) when the VAE cannot report them."""+ if not _vae_savers_observable(pipe):+ return "unknown"+ return vae_saver_tag(+ getattr(plan, "vae_tiling", False) and _vae_saver_can(pipe, *_VAE_SAVERS[0][1:]),+ getattr(plan, "vae_slicing", False) and _vae_saver_can(pipe, *_VAE_SAVERS[1][1:]),+ )+++def live_vae_saver_tag(pipe: Any) -> str:+ """The VAE savers in force right now (``unknown`` as in ``planned_vae_saver_tag``)."""+ if not _vae_savers_observable(pipe):+ return "unknown"+ vae = getattr(pipe, "vae", None)+ return vae_saver_tag(getattr(vae, "use_tiling", False), getattr(vae, "use_slicing", False))+++def stream_window_pad_mib(pipe: Any) -> Optional[int]:+ """MiB a learned peak measured with no denoiser group streaming adds to reach the streamed baseline (the prefetch+ window a streamed placement holds above its start allocation), so every placement reads one conservative scale.+ None when that window cannot be sized, or the denoiser is neither group-streamed nor on the card (whole-module /+ sequential offload onloads it whole inside the render): the peak is then not recorded. Resident text encoders add+ their leaf-streaming window the same way (``_encoder_stream_window_mib``)."""+ encoders = _encoder_stream_window_mib(pipe)+ if encoders is None:+ return None+ streaming = partial = False+ for name, module in (getattr(pipe, "components", {}) or {}).items():+ if str(name) not in ("transformer", "transformer_2", "unconditional_transformer"):+ continue+ state = _streaming_state(module)+ if state is None:+ return None+ if state:+ streaming = True+ partial = partial or any(+ getattr(g, "_unsloth_resident", False) for g in _offload_groups(module)+ )+ if streaming and not partial:+ return encoders+ window = _stream_window_mib(pipe)+ if window < 0:+ return None+ if streaming:+ # some groups resident: the measured window is the largest STREAMED group's, a fully streamed load holds the+ # largest of all; pad the difference+ streamed = _streamed_group_window_mib(pipe)+ if streamed < 0:+ return None+ window = max(0, window - streamed)+ return window + encoders+++def _streamed_group_window_mib(pipe: Any) -> int:+ """Device MiB the prefetcher holds for the denoiser groups that stream now: ``depth + 1`` of the largest, or -1."""+ try:+ from .diffusion_offload_prefetch import prefetch_depth++ largest = 0+ for name, module in (getattr(pipe, "components", {}) or {}).items():+ if str(name) not in ("transformer", "transformer_2", "unconditional_transformer"):+ continue+ for group in _offload_groups(module):+ if getattr(group, "_unsloth_resident", False):+ continue+ largest = max(largest, _group_nbytes(group))+ if largest <= 0:+ return -1+ return -(-largest * (prefetch_depth() + 1) // (1024 * 1024))+ except Exception: # noqa: BLE001+ return -1+++def _group_nbytes(group: Any) -> int:+ """Storage bytes of one offload group's modules, parameters and buffers (shared tensors once)."""+ seen: set[int] = set()+ nbytes = 0+ for member in getattr(group, "modules", None) or ():+ for t in (*member.parameters(), *member.buffers()):+ if id(t) not in seen:+ seen.add(id(t))+ nbytes += sum(_storage_nbytes(t))+ for t in (+ *(getattr(group, "parameters", None) or ()),+ *(getattr(group, "buffers", None) or ()),+ ):+ if id(t) not in seen:+ seen.add(id(t))+ nbytes += sum(_storage_nbytes(t))+ return nbytes+++def _streaming_state(module: Any) -> Optional[bool]:+ """True when ``module`` streams offload groups, False when it sits on the card, None when neither can be told+ (whole-module / sequential offload, weights off the card without hooks)."""+ try:+ groups = _offload_groups(module) or ()+ if any(not getattr(g, "_unsloth_resident", False) for g in groups):+ return True+ if not groups:+ if getattr(module, "_hf_hook", None) is not None:+ return None+ first = next(iter(module.parameters()), None)+ if first is not None and getattr(first.device, "type", None) != "cuda":+ return None+ return False+ except Exception: # noqa: BLE001+ return None+++def _encoder_stream_window_mib(pipe: Any) -> Optional[int]:+ """MiB a load that streams the text encoders (leaf-level groups) holds in flight that a render with them resident+ did not: ``depth + 1`` of the largest leaf of each resident encoder, less the streamed leaves' window of a+ partially resident one. None when an encoder's placement is unknown."""+ try:+ import torch++ from .diffusion_offload_prefetch import prefetch_depth++ total = 0+ for name, module in (getattr(pipe, "components", {}) or {}).items():+ if not str(name).startswith("text_encoder") or not isinstance(module, torch.nn.Module):+ continue+ state = _streaming_state(module)+ if state is None:+ return None+ groups = list(_offload_groups(module) or ()) if state else []+ if state and not any(getattr(g, "_unsloth_resident", False) for g in groups):+ continue # fully streamed during the render: its whole window is in the peak+ largest = 0+ for sub in module.modules():+ seen: set[int] = set()+ nbytes = 0+ for t in (*sub.parameters(recurse = False), *sub.buffers(recurse = False)):+ if id(t) not in seen:+ seen.add(id(t))+ nbytes += sum(_storage_nbytes(t))+ largest = max(largest, nbytes)+ window = -(-largest * (prefetch_depth() + 1) // (1024 * 1024))+ if state:+ # some leaves resident: the peak holds the largest STREAMED leaf's window, a fully streamed encoder+ # the largest of all; pad the difference+ streamed = max(+ (+ _group_nbytes(g)+ for g in groups+ if not getattr(g, "_unsloth_resident", False)+ ),+ default = 0,+ )+ window = max(0, window - -(-streamed * (prefetch_depth() + 1) // (1024 * 1024)))+ total += window+ return total+ except Exception: # noqa: BLE001+ return None+++def vae_saver_tag(tiling: Any, slicing: Any) -> str:+ """The VAE savers in force, as the learned-peak key names them."""+ return "+".join(name for name, on in (("tiled", tiling), ("sliced", slicing)) if on) or "full"+++def denoiser_quant_label(pipe: Any) -> str:+ """Stable label of the denoiser weights' format for the learned-peak key: torchao subclasses, GGUF, int8 / uint8+ packed, or the dense dtype."""+ try:+ import torch++ dits = [+ getattr(pipe, name, None)+ for name in ("transformer", "transformer_2", "unconditional_transformer", "unet")+ ]+ dits = [d for d in dits if isinstance(d, torch.nn.Module)]+ if not dits:+ return "none"+ classes: set[str] = set()+ for dit in dits:+ classes |= _torchao_weight_classes(dit)+ if classes:+ return "torchao:" + "+".join(sorted(classes)) + _quant_variant_tag(dits)+ dtypes: set[str] = set()+ for dit in dits:+ for param in dit.parameters():+ if type(param).__name__ == "GGUFParameter":+ return "gguf"+ if param.dtype in (torch.int8, torch.uint8):+ dtypes.add("int8")+ elif param.is_floating_point():+ dtypes.add(str(param.dtype).replace("torch.", ""))+ if "int8" in dtypes:+ return "packed:int8" + _quant_variant_tag(dits)+ return "dense:" + "+".join(sorted(dtypes)) if dtypes else "unknown"+ except Exception: # noqa: BLE001+ return "unknown"+++def _quant_variant_tag(dits: list) -> str:+ """``#`` plus a short digest of the denoisers' quant graph variant (``graph_variant``, the compile cache's key):+ plain vs ConvRot, W8A8 vs weight-only, hosted vs runtime scales share a weight class but not their kernels or+ workspaces. ``#unknown`` when it cannot be read (the label then neither reads nor records a peak)."""+ try:+ import hashlib+ import json++ from .diffusion_compile_cache import graph_variant++ variants = [graph_variant(dit) for dit in dits]+ if any(isinstance(v.get("weights"), str) for v in variants):+ return "#unknown"+ blob = json.dumps(variants, sort_keys = True, default = str)+ return "#" + hashlib.sha256(blob.encode()).hexdigest()[:12]+ except Exception: # noqa: BLE001+ return "#unknown"++def _loaded_component_mib(pipe: Any) -> Optional[dict[str, tuple[int, str]]]:"""name -> (MiB, role in 'dit' / 'text_encoder' / 'other') from storage bytes (a torchao int8 weight counts itsint8 data + scales); None when unreadable."""@@ -2441,6 +2729,12 @@ def refine_plan_from_loaded_weights(family: Optional[str],speed_mode: Optional[str],logger: Any = None,+ attention: Optional[str] = None,+ model: Optional[str] = None,+ vae: Optional[str] = None,+ te: Optional[str] = None,+ cache: Optional[str] = None,+ graphs: Optional[str] = None,) -> MemoryPlan:"""Re-place a streamed ``auto`` load from its LOADED weights and the family's measured activation peak.@@ -2473,16 +2767,34 @@ def refine_plan_from_loaded_weights(if dense_mib is not None and (compute_bytes is None or _pipe_denoisers_hold_packed(pipe)):# measured on fp16 / fp32 cards only; bf16 cards can compile laterreturn plan+ quant = learned_peak_label(pipe, attention, model, vae, te, cache, graphs)+ try:+ pipe._unsloth_peak_identity = (family, quant, speed_mode)+ except Exception: # noqa: BLE001 - a pipe refusing attributes only skips recording+ passheadroom = measured_image_runtime_mib(- family, speed_mode, dense_transformer_mib = dense_mib, compute_bytes = compute_bytes or 2+ family,+ speed_mode,+ dense_transformer_mib = dense_mib,+ compute_bytes = compute_bytes or 2,+ quant = quant,+ phase_seeds = bool(getattr(pipe, "_unsloth_phase_seeds", False)),)if headroom is None:return plan+ from .diffusion_measured_peaks import lookup_bucket++ learned, learned_pixels, learned_quadratic = lookup_bucket(+ family = family, quant = quant, speed_mode = speed_mode+ ) or (None, None, False)overhead = int(plan.estimates.get("base_overhead_mib") or DEFAULT_BASE_OVERHEAD_MIB)floor = headroom + overhead + otherestimates = dict(plan.estimates)estimates.update(measured_runtime_headroom_mib = headroom,+ measured_learned_peak_mib = learned,+ measured_learned_peak_pixels = learned_pixels,+ measured_learned_peak_quadratic = learned_quadratic,measured_dense_transformer_mib = dense_mib,measured_compute_bytes = compute_bytes,loaded_transformer_mib = dit,@@ -2523,7 +2835,9 @@ def refine_plan_from_loaded_weights(encode_room = int(dit_room)dit_room, room, wide_dit = streamed_room, streamed_room, Trueestimates["stream_window_mib"] = window- te_room = max(0, room - dit_room) if stream_te and policy == OFFLOAD_GROUP else 0+ te_room = (+ min(max(0, room - dit_room), encoders) if stream_te and policy == OFFLOAD_GROUP else 0+ )if dit_room <= 0 and te_room <= 0:return planif whole_dit or wide_dit:@@ -2540,7 +2854,12 @@ def refine_plan_from_loaded_weights(},reasons = plan.reasons+ (- f"{int(dit_room)} MiB of the {dit} MiB transformer and {int(te_room)} MiB of the {encoders} MiB "+ (+ f"the {dit} MiB transformer is resident (not streamed by the plan)"+ if not bool(getattr(plan, "stream_transformer", True))+ else f"{int(dit_room)} MiB of the {dit} MiB transformer"+ )+ + f" and {int(te_room)} MiB of the {encoders} MiB "f"encoders stay resident (measured peak {headroom} MiB); the rest streams"+ (f" (whole transformer within free memory less a {_resident_dit_slack_mib(memory)} MiB slack;"@@ -2653,35 +2972,57 @@ def _keep_groups_resident(module._unsloth_resident_device = onloadexcept AttributeError:pass+ # Tied weights (an LM head sharing the embedding, as in Qwen3 text encoders) sit in two groups. A resident+ # group next to a streamed partner breaks: the partner's offload points the shared tensor back at its host+ # copy and the resident group's no-op onload never brings it back. Such groups go resident together or not.+ tied = _tied_group_closures(ordered, _tensors)+ done: set = set()for group in ordered:if getattr(group, "offload_to_disk_path", None):break- tensors = _tensors(group)+ if id(group) in done:+ continue+ closure = tied.get(id(group)) or [group]+ done.update(id(g) for g in closure)+ unique: dict = {}+ for member in closure:+ for t in _tensors(member):+ unique.setdefault(id(t), t)+ tensors = list(unique.values())need = sum(sum(_storage_nbytes(t)) for t in tensors)- if getattr(group, "_unsloth_resident", False):+ if all(getattr(g, "_unsloth_resident", False) for g in closure):left -= needcontinue- if need > left or (only is not None and id(group) not in only):+ if (+ need > left+ or any(getattr(g, "offload_to_disk_path", None) for g in closure)+ or (only is not None and any(id(g) not in only for g in closure))+ ):# too large (a later, smaller group may still fit), or another release's groupcontinue- cpu = getattr(group, "cpu_param_dict", None) or {}- for t in tensors:- if _placed_on(t, onload.type, is_torchao):+ for member in closure:+ cpu = getattr(member, "cpu_param_dict", None) or {}+ for t in _tensors(member):+ if _placed_on(t, onload.type, is_torchao):+ continue+ moved = cpu.get(t, t).to(onload)+ if is_torchao(t):+ go._swap_torchao_tensor(t, moved)+ else:+ t.data = moved+ for i, member in enumerate(closure):+ if getattr(member, "_unsloth_resident", False):continue- moved = cpu.get(t, t).to(onload)- if is_torchao(t):- go._swap_torchao_tensor(t, moved)- else:- t.data = moved- # host copies stay: release_resident_groups streams the group again for an oversized request- group._unsloth_streamed_hooks = (- group.__dict__.get("onload_"),- group.__dict__.get("offload_"),- )- group.onload_ = _resident_onload(getattr(group, "stream", None))- group.offload_ = noop- group._unsloth_resident = True- group._unsloth_resident_bytes = need+ # host copies stay: release_resident_groups streams the group again for an oversized request+ member._unsloth_streamed_hooks = (+ member.__dict__.get("onload_"),+ member.__dict__.get("offload_"),+ )+ member.onload_ = _resident_onload(getattr(member, "stream", None))+ member.offload_ = noop+ member._unsloth_resident = True+ member._unsloth_resident_bytes = need if i == 0 else 0+ member._unsloth_tied_groups = [g for g in closure if g is not member]left -= needkept += needstate["streamed"] = _streamed_group_count(ordered)@@ -2704,6 +3045,31 @@ def _keep_groups_resident(return 0+def _tied_group_closures(groups: list, tensors_of: Callable[[Any], list]) -> dict:+ """id(group) -> every group that shares a tensor with it, transitively (only for groups that share any)."""+ owners: dict[int, list] = {}+ for group in groups:+ for t in tensors_of(group):+ owners.setdefault(id(t), []).append(group)+ closures: dict = {}+ for group in groups:+ if id(group) in closures:+ continue+ stack, seen = [group], {id(group): group}+ while stack:+ current = stack.pop()+ for t in tensors_of(current):+ for other in owners.get(id(t), ()):+ if id(other) not in seen:+ seen[id(other)] = other+ stack.append(other)+ if len(seen) > 1:+ members = [g for g in groups if id(g) in seen]+ for g in members:+ closures[id(g)] = members+ return closures++def _bump_placement_epoch() -> None:try:from .diffusion_offload_prefetch import bump_placement_epoch@@ -2768,9 +3134,13 @@ def release_resident_groups(state = getattr(module, "_unsloth_stream_state", None)if isinstance(state, dict):state["streamed"] = 1- _release_group(group)- streamed.add(id(group))- left -= int(getattr(group, "_unsloth_resident_bytes", 0))+ # tied partners stream together (a resident partner would keep reading the host copy)+ for member in [group, *(getattr(group, "_unsloth_tied_groups", None) or [])]:+ if not getattr(member, "_unsloth_resident", False):+ continue+ _release_group(member)+ streamed.add(id(member))+ left -= int(getattr(member, "_unsloth_resident_bytes", 0))if module not in released:released.append(module)if not released:@@ -2941,7 +3311,32 @@ def _return_module_tensors(module: Any, onload: Any) -> None:go._swap_torchao_tensor(t, moved)else:t.data = t.data.to(onload)- torch.cuda.synchronize(onload)+ if onload.type == "cuda":+ torch.cuda.synchronize(onload)+++def _unwind_group_offload(+ pipe: Any,+ snapshots: list,+ logger: Any = None,+) -> bool:+ """Undo a partial ``_apply_group_offload``: group hooks, the prefetchers' torch hooks and attributes, deferred+ pins. The weights stay where the hooks left them (host copies or the card); the caller's retry moves them."""+ try:+ from .diffusion_offload_prefetch import PREFETCHER_ATTR+ from .diffusion_small_host import ENCODER_PREFETCH_ATTR++ for module, before in snapshots:+ _remove_group_offload_hooks(module)+ _drop_module_forward_hooks_since(module, before)+ for attr in (PREFETCHER_ATTR, ENCODER_PREFETCH_ATTR, "_unsloth_stream_state"):+ module.__dict__.pop(attr, None)+ _drop_deferred_pinning(pipe, module)+ return bool(snapshots)+ except Exception as exc: # noqa: BLE001 - the original failure propagates+ if logger is not None:+ logger.warning("diffusion.memory: could not unwind partial group offload (%s)", exc)+ return Falsedef _is_text_encoder_module(pipe: Any, module: Any) -> bool:@@ -3045,15 +3440,65 @@ def measured_request_extra_mib(headroom, family, speed_mode = reserve[:3]dense_mib = reserve[3] if len(reserve) > 3 else Nonecompute_bytes = reserve[4] if len(reserve) > 4 and reserve[4] else 2- need = measured_image_runtime_mib(- family,- speed_mode,+ identity = getattr(pipe, "_unsloth_peak_identity", None)+ sized = dict(width = width,height = height,batch_size = batch_size,dense_transformer_mib = dense_mib,compute_bytes = compute_bytes,)+ # A seeded family keeps the seed and the learned peak the load placed from: a peak this session's renders recorded+ # after the load only places the NEXT load, it must not stream resident groups for a request the same size as one+ # that just completed. An unseeded family was placed from its learned peak, which is all there is to check against.+ sized["phase_seeds"] = bool(getattr(pipe, "_unsloth_phase_seeds", False))+ need = measured_image_runtime_mib(family, speed_mode, **sized)+ learned = reserve[5] if len(reserve) > 5 else None+ if (learned or need is None) and (+ getattr(pipe, "_unsloth_peak_static", False) or getattr(pipe, "_unsloth_loras", None)+ ):+ # the learned peak no longer describes this pipeline (runtime LoRA adapters, a planned VAE saver that did not+ # engage): requests size from the flat estimate+ flat = estimate_image_runtime_mib(+ width = width,+ height = height,+ batch_size = batch_size,+ family = family,+ condition_pixels = condition_pixels,+ )+ return max(0, int(flat) - int(headroom))+ if learned:+ # the learned peak the load reserved, scaled here (not reread from the store, which another worker may evict+ # or a failed write may leave empty while this pipeline stays loaded)+ from .diffusion_measured_peaks import resolution_bucket++ w = max(64, int(width or DEFAULT_IMAGE_WIDTH))+ h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))+ batch = max(1, int(batch_size or 1))+ covered = (+ reserve[6]+ if len(reserve) > 6 and reserve[6]+ else DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT+ )+ # the learned store's own scaling: a request past the covered bucket takes its bucket's ceiling, squared over+ # the image's pixels for a math-attention entry (linear over the batch)+ scale = 1.0+ if len(reserve) > 7 and reserve[7]:+ if w * h > int(covered):+ scale = (resolution_bucket(w, h, 1) / float(covered)) ** 2+ scale *= batch+ elif w * h * batch > int(covered):+ scale = resolution_bucket(w, h, batch) / float(covered)+ step = _MEASURED_PEAK_ROUND_MIB+ scaled = int(-(-(float(learned) * scale * _MEASURED_PEAK_MARGIN) // step) * step)+ seeded = need is not None+ need = scaled if need is None else max(need, scaled)+ if not seeded and isinstance(identity, tuple) and len(identity) == 3:+ stored = measured_image_runtime_mib(family, speed_mode, quant = identity[1], **sized)+ if stored is not None:+ need = max(need, stored)+ if need is None and isinstance(identity, tuple) and len(identity) == 3:+ need = measured_image_runtime_mib(family, speed_mode, quant = identity[1], **sized)if need is None:return 0cond = max(0, int(condition_pixels or 0)) * max(1, int(batch_size or 1))@@ -3085,6 +3530,10 @@ def apply_memory_plan(the modules would page onto the very card the selection existed to avoid while generation ranon another. ``device`` stays bare for anything reading it as a policy string."""placement = placement_device or device+ try:+ pipe._unsloth_applied_plan = None+ except Exception: # noqa: BLE001+ passtiling_engaged = Falseif plan.vae_tiling:tiling_engaged = _enable_vae_saver(pipe, "enable_vae_tiling", "enable_tiling", logger)@@ -3117,7 +3566,18 @@ def apply_memory_plan(resident_te_mib = getattr(plan, "resident_text_encoder_mib", None)if resident_te_mib and group_kwargs["stream_text_encoders"]:group_kwargs["resident_text_encoder_mib"] = int(resident_te_mib)- if not _apply_group_offload(pipe, placement, logger, **group_kwargs):+ _set_group_offload_failure(pipe, None)+ applied = _apply_group_offload(pipe, placement, logger, **group_kwargs)+ failure = getattr(pipe, _GROUP_OFFLOAD_FAILURE_ATTR, None)+ if not applied and failure is not None and _is_oom(failure):+ lowered = _retry_group_offload_lower(pipe, plan, placement, logger, failure)+ if lowered is not None:+ applied = True+ try:+ pipe._unsloth_applied_plan = lowered+ except Exception: # noqa: BLE001+ pass+ if not applied:if "stream_transformer" in group_kwargs and _pipe_denoisers_hold_torchao(pipe):raise RuntimeError("the text encoder could not be streamed beside the resident quantised "@@ -3161,6 +3621,83 @@ def apply_memory_plan(return policy, tiling_engaged+_GROUP_OFFLOAD_FAILURE_ATTR = "_unsloth_group_offload_failure"+++def _set_group_offload_failure(pipe: Any, exc: Optional[BaseException]) -> None:+ """What made ``_apply_group_offload`` return False (it absorbs the exception so the caller can fall back)."""+ try:+ setattr(pipe, _GROUP_OFFLOAD_FAILURE_ATTR, exc)+ except Exception: # noqa: BLE001+ pass+++def _is_oom(exc: BaseException) -> bool:+ from .diffusion_batched import is_oom_error+ return is_oom_error(exc)+++def applied_plan(pipe: Any, plan: Any) -> Any:+ """The plan ``apply_memory_plan`` actually placed: ``plan``, or the lower-residency re-plan a transient setup OOM+ forced (callers gate graphs, fused GEMMs and the encode release on residency)."""+ return getattr(pipe, "_unsloth_applied_plan", None) or plan+++def _retry_group_offload_lower(+ pipe: Any, plan: MemoryPlan, placement: str, logger: Any, exc: BaseException+) -> Optional[MemoryPlan]:+ """One retry after an OOM while setting up group offload (another process can move VRAM between the plan's+ snapshot and placement): return every denoiser and text encoder to the host, empty the cache, and stream the+ denoisers and encoders with nothing kept resident. Returns the re-plan, or None (the caller refuses as before)."""+ try:+ import gc++ import torch++ for name, comp in (getattr(pipe, "components", {}) or {}).items():+ name = str(name)+ if not isinstance(comp, torch.nn.Module):+ continue+ if name in (+ "transformer",+ "transformer_2",+ "unconditional_transformer",+ ) or name.startswith("text_encoder"):+ _return_module_tensors(comp, torch.device("cpu"))+ gc.collect()+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+ if logger is not None:+ logger.warning(+ "diffusion.memory: group offload setup hit OOM (%s); emptied the cache and re-planning once with "+ "the transformer and text encoders streamed",+ str(exc).splitlines()[0][:200] if str(exc) else type(exc).__name__,+ )+ except Exception as retry_exc: # noqa: BLE001 - the caller's refusal stands+ if logger is not None:+ logger.warning(+ "diffusion.memory: lower-residency group offload retry failed (%s)", retry_exc+ )+ return None+ # outside the catch: a raise here left hooks on part of the pipe, which the caller's whole-module fallback rejects+ if not _apply_group_offload(pipe, placement, logger, stream_text_encoders = True):+ return None+ estimates = dict(plan.estimates or {})+ estimates.update(resident_transformer_mib = 0, resident_text_encoder_mib = 0, setup_oom_retry = True)+ return replace(+ plan,+ stream_text_encoders = True,+ stream_transformer = True,+ resident_transformer_mib = None,+ resident_text_encoder_mib = None,+ estimates = estimates,+ reasons = tuple(plan.reasons)+ + (+ "group offload setup hit OOM; re-planned once with the transformer and text encoders streamed",+ ),+ )++def _enable_vae_saver(pipe: Any, pipe_method: str, vae_method: str, logger: Any) -> bool:"""Turn on a VAE memory saver, trying the pipeline shortcut first then the VAE submodule(some pipelines, e.g. Z-Image, only expose it on ``pipe.vae``). Returns whether it engaged."""@@ -3399,6 +3936,13 @@ def denoisers_pinned_resident(pipe: Any) -> bool:return pinned+def _has_offload_groups(module: Any) -> bool:+ try:+ return bool(_offload_groups(module))+ except Exception: # noqa: BLE001 - an unreadable module counts as unhooked+ return False++def _offload_groups(module: Any) -> list:"""The diffusers offload groups hooked under ``module``, in registration (block) order."""try:@@ -3584,10 +4128,14 @@ class _GroupPinner:def _unpinned(self, group: Any) -> list:import torch++ # GGUF host copies are GGUFParameter (packed bytes + quant type): pin_memory keeps the subclass, which the+ # per-onload fallback already relies on. Leaving them unpinned kept every CUDA graph off the GGUF stream.+ gguf = _gguf_parameter_class()return [(tensor, src)for tensor, src in list(group.cpu_param_dict.items())- if type(src) is torch.Tensor+ if (type(src) is torch.Tensor or (gguf is not None and type(src) is gguf))and src.device.type == "cpu"and src.numel() > 0and not src.is_pinned()@@ -3627,7 +4175,16 @@ class _GroupPinner:)for group in self.groups:copies[id(group)] = [- (tensor, src, pool.submit(registered_host_copy, src))+ (+ tensor,+ src,+ pool.submit(+ registered_host_copy+ if type(src) is torch.Tensor+ else src.pin_memory,+ *((src,) if type(src) is torch.Tensor else ()),+ ),+ )for tensor, src in self._unpinned(group)]for group in self.groups:@@ -4084,6 +4641,9 @@ def _apply_group_offload(if not stream_transformer and not stream_text_encoders:return Falseinstalled = 0+ # every component's torch forward hooks before this call: an OOM after hooks exist unwinds to this state+ hook_snapshots: list = []+ grouped_before: set = set()try:import inspect@@ -4092,6 +4652,12 @@ def _apply_group_offload(install_group_offload_buffer_restore()install_group_offload_hooks_eager()+ hook_snapshots = [+ (comp, _module_forward_hook_ids(comp))+ for comp in (getattr(pipe, "components", {}) or {}).values()+ if isinstance(comp, torch.nn.Module)+ ]+ grouped_before = {id(comp) for comp, _ in hook_snapshots if _has_offload_groups(comp)}# A dual-DiT pipeline (Ideogram 4) carries a second denoiser as large as the first, so stream every DiT and keep# only smaller companions resident.@@ -4266,6 +4832,16 @@ def _apply_group_offload(module.to(onload)return Trueexcept Exception as exc: # noqa: BLE001 - fall back to whole-module offload+ # apply_group_offloading can raise after hooking part of a module, before ``installed`` counts it+ if not installed and any(+ id(comp) not in grouped_before and _has_offload_groups(comp)+ for comp, _ in hook_snapshots+ ):+ installed = 1+ if installed and _is_oom(exc) and _unwind_group_offload(pipe, hook_snapshots, logger):+ # An OOM after some hooks exist (a later DiT, an encoder moved resident): every hook is gone again, so+ # the caller's one lower-residency retry can run instead of the load failing.+ installed = 0if installed:# An earlier streamed module already has hooks but a later one failed: the pipe is in a PARTIAL# group-offload state enable_model_cpu_offload rejects, so propagate the real failure instead of a@@ -4278,6 +4854,7 @@ def _apply_group_offload(exc,)raise+ _set_group_offload_failure(pipe, exc)if logger is not None:logger.warning("diffusion.memory: group offload failed (%s); falling back to "diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex d762072ec29..9671e50a6ae 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -124,6 +124,7 @@ from .diffusion_memory import (reclaim_host_memory,reclaim_offload_host_memory,refine_memory_plan_for_components,+ applied_plan,finish_background_pins,release_pinned_host_memory,request_background_pins,@@ -2675,19 +2676,16 @@ def _calibration_revision(state: Any) -> Optional[str]:cached = getattr(pipe, "_unsloth_calibration_revision", None)if cached is not None:return cached or None- from .diffusion_component_sizes import checkpoint_fingerprint-- revision = checkpoint_fingerprint(- getattr(state, "repo_id", None), getattr(state, "gguf_filename", None)+ from .diffusion import _peak_model_identity++ # the image peak identity's rule: snapshot commit, or a local folder's manifests plus its weight files' size and+ # mtime, plus the companion base for a GGUF / single-file load; "unknown" persists nothing+ revision = _peak_model_identity(+ getattr(state, "repo_id", None),+ getattr(state, "gguf_filename", None),+ getattr(state, "kind", None),+ getattr(state, "base_repo", None),)- if getattr(state, "kind", "pipeline") != "pipeline":- # a GGUF / single-file load takes its companions from a base repo: its id and exact weights are part of it- base = getattr(state, "base_repo", None)- revision = (- f"{revision or ''}|base={base or ''}@{checkpoint_fingerprint(base) or ''}"- if base- else revision- )try:pipe._unsloth_calibration_revision = revision or ""except Exception: # noqa: BLE001@@ -7995,6 +7993,8 @@ class VideoBackend:placement_device = target.torch_device,logger = logger,)+ # a transient OOM during setup may have re-planned to lower residency+ plan = applied_plan(pipe, plan)ltx25_upsampler = getattr(pipe, "_unsloth_ltx25_upsampler", None)if ltx25_upsampler is not None and offload_policy == OFFLOAD_NONE:# ~1 GB; resident with the rest, else render() moves it in for the upsample only.@@ -10836,7 +10836,11 @@ class VideoBackend:_log_failed_generation(request_shape, exc)raisefinally:- i2v_swap.close()+ try:+ from . import video_stream_residency+ video_stream_residency.discard_request(getattr(state, "pipe", None))+ finally:+ i2v_swap.close()if bg_token is not None:bg_compile._FORCE_EAGER.reset(bg_token)with self._lock:diff --git a/studio/backend/core/inference/video_stream_residency.py b/studio/backend/core/inference/video_stream_residency.pyindex f754b8989cb..e8353c255cc 100644--- a/studio/backend/core/inference/video_stream_residency.py+++ b/studio/backend/core/inference/video_stream_residency.py@@ -187,10 +187,39 @@ def _covering(peaks: dict, work: int) -> Optional[int]:return max(covering) if covering else None+def _store_key(key: Any) -> Any:+ """``key`` as persisted: the device in it is a process-local ordinal (``cuda:0``), so the GPU's name rides along+ and workers with different ``CUDA_VISIBLE_DEVICES`` never share one card's peaks with another."""+ if not isinstance(key, tuple) or len(key) != 5:+ return key+ if dict(key[4]).get("revision", "?") in ("", "None", "unknown"):+ return None # the exact weights are unknown (an unreadable local build): nothing persists for them+ try:+ from .diffusion_measured_peaks import device_name+ name = device_name(key[3]) if key[3] else None+ except Exception: # noqa: BLE001+ name = None+ return (key, name)+++def _key_peaks(key: Any) -> dict:+ """This process's peaks for ``key``, merged with the persisted store's (other processes' measurements) at every+ read; the store's own mtime cache keeps that a stat when nothing changed."""+ peaks = _PEAKS.setdefault(key, {})+ try:+ from .diffusion_measured_peaks import work_peaks+ for work, extra in work_peaks(_store_key(key)).items():+ if extra > peaks.get(work, 0):+ peaks[work] = extra+ except Exception: # noqa: BLE001 - no store: this process's peaks only+ pass+ return peaks++def measured_extra_mib(key: Any, work: int) -> Optional[int]:if key is None or not measured_enabled():return None- return _covering(_PEAKS.get(key) or {}, int(work))+ return _covering(_key_peaks(key), int(work))def _measured_extra_mib(module: Any, work: int) -> Optional[int]:@@ -199,7 +228,7 @@ def _measured_extra_mib(module: Any, work: int) -> Optional[int]:peaks it recorded itself; the module's own mode-blind table is the fallback for an untagged one."""key = getattr(module, "_unsloth_video_calibration_key", None)if key is not None:- return _covering(_PEAKS.get(key) or {}, work)+ return _covering(_key_peaks(key), work)return _covering(getattr(module, "_unsloth_video_peaks", None) or {}, work)@@ -227,14 +256,107 @@ def arm_request(module._unsloth_video_calibration_key = keyif not rearm:return True+ dev = _open_window(module, dev)torch.cuda.reset_peak_memory_stats(dev)work = int(width) * int(height) * int(frames)module._unsloth_video_pending = (work, int(torch.cuda.memory_allocated(dev)), dev)+ module._unsloth_video_failed_captures = _failed_captures()return Trueexcept Exception: # noqa: BLE001 -- an unarmed request keeps the estimatereturn False+def _open_window(module: Any, dev: Any) -> Any:+ """Register this request's peak measurement on its card (``diffusion_measured_peaks.open_window``), replacing a+ stale one; returns the indexed device."""+ import torch++ from .diffusion_measured_peaks import close_window, open_window++ if dev.index is None:+ dev = torch.device("cuda", torch.cuda.current_device())+ stale = getattr(module, "_unsloth_video_window", None)+ if stale is not None:+ close_window(stale[0], stale[1])+ module._unsloth_video_window = (dev, open_window(dev))+ graphs = _live_graphs(dev)+ generation = _capture_generation(dev)+ accounted = getattr(module, "_unsloth_video_graphs", None)+ if accounted is None or graphs < accounted:+ # the load's own graphs (every request starts above them), or fewer after a reset freed some+ first = accounted is None+ module._unsloth_video_graphs = accounted = graphs+ if first or graphs <= 0:+ # nothing alive on the card; a drop to a non-zero count may be another pipeline's reset, so a capture+ # this one made unrecorded keeps the generation dirty+ module._unsloth_video_graph_gen = generation+ if getattr(module, "_unsloth_video_graph_gen", None) is None:+ module._unsloth_video_graph_gen = generation+ # graphs a discarded request recorded stay allocated, also one that replaced an evicted graph (same count): a+ # peak above that start would omit them+ module._unsloth_video_graphs_clean = (+ graphs == accounted and generation <= module._unsloth_video_graph_gen+ )+ return dev+++def _capture_generation(dev: Any) -> int:+ try:+ from .diffusion_cuda_graph import capture_generation+ return capture_generation(getattr(dev, "index", None))+ except Exception: # noqa: BLE001+ return -1+++def _live_graphs(dev: Any) -> int:+ try:+ from .diffusion_cuda_graph import live_graph_count+ return live_graph_count(getattr(dev, "index", None))+ except Exception: # noqa: BLE001+ return -1+++def discard_request(pipe: Any) -> None:+ """Drop a request's armed measurement without recording it (cancel, failure, refusal): its window would otherwise+ mark every later measurement on the card as overlapped. A no-op after ``record_request_peak``."""+ module = getattr(pipe, "transformer", None)+ if module is None:+ return+ try:+ module._unsloth_video_pending = None+ window = getattr(module, "_unsloth_video_window", None)+ module._unsloth_video_window = None+ if window is not None:+ from .diffusion_measured_peaks import close_window+ close_window(window[0], window[1])+ except Exception: # noqa: BLE001+ pass+++def _failed_captures() -> int:+ try:+ from .diffusion_cuda_graph import failed_capture_count+ return failed_capture_count()+ except Exception: # noqa: BLE001+ return 0+++def _keyed_graphs_live(pipe: Any, key: Any) -> bool:+ """False when ``key`` names CUDA graphs that ``live_status`` reports dropped or never engaged for this load."""+ try:+ optims = dict(key[4]).get("speed_optims", "")+ if "cuda_graph" not in str(optims):+ return True+ from .diffusion_cuda_graph import live_status++ _, live = live_status(+ None, ("cuda_graph",), tuple(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())+ )+ return "cuda_graph" in live+ except Exception: # noqa: BLE001 - unreadable: no key names this render+ return False++def record_request_peak(pipe: Any, *, logger: Any = None) -> Optional[int]:"""Record the request's peak allocation above its start; call only after decode and export."""module = getattr(pipe, "transformer", None)@@ -247,7 +369,15 @@ def record_request_peak(pipe: Any, *, logger: Any = None) -> Optional[int]:module._unsloth_video_pending = Nonework, start, dev = pendingextra = (int(torch.cuda.max_memory_allocated(dev)) - int(start)) >> 20- if extra <= 0:+ window = getattr(module, "_unsloth_video_window", None)+ module._unsloth_video_window = None+ if window is not None:+ from .diffusion_measured_peaks import close_window++ # closed after the read: a render arming in between would otherwise reset the peak unseen+ if not close_window(window[0], window[1]):+ return None # another render on the card reset the peak meanwhile+ if extra <= 0 or not getattr(module, "_unsloth_video_graphs_clean", True):return Nonepeaks = getattr(module, "_unsloth_video_peaks", None)if not isinstance(peaks, dict):@@ -255,9 +385,42 @@ def record_request_peak(pipe: Any, *, logger: Any = None) -> Optional[int]:module._unsloth_video_peaks = peakspeaks[work] = max(extra, peaks.get(work, 0))key = getattr(module, "_unsloth_video_calibration_key", None)- if key is not None:- shared = _PEAKS.setdefault(key, {})- shared[work] = max(extra, shared.get(work, 0))+ # a failed capture absorbed an OOM: this peak is that attempt's, keep it out of the keyed and persisted peaks+ # a load keyed with CUDA graphs whose graphs no longer run (a poisoned wrapper after a failed capture runs+ # eager without counting a new failure) records an eager peak a graph-capturing load would undershoot+ if (+ key is not None+ and _failed_captures() == getattr(module, "_unsloth_video_failed_captures", 0)+ and _keyed_graphs_live(pipe, key)+ ):+ graphs, generation = _live_graphs(dev), _capture_generation(dev)+ stored_key = _store_key(key)+ try:+ from .diffusion_memory import stream_window_pad_mib++ # on the streamed baseline, as image peaks: a later request of the key may stream groups this request+ # kept resident; an unknown placement enters neither the keyed table nor the store+ pad = stream_window_pad_mib(pipe)+ except Exception: # noqa: BLE001+ pad = None+ held: list = []+ if pad is not None:+ shared = _key_peaks(key)+ shared[work] = max(extra + pad, shared.get(work, 0))+ # an unpersistable key keeps this process's table only, where max wins: nothing lower can be stored+ # later+ if stored_key is None:+ held.append(True)+ else:+ try:+ from .diffusion_measured_peaks import record_work+ record_work(stored_key, work, extra + pad, held = held)+ except Exception: # noqa: BLE001 - persistence is best-effort+ pass+ if held:+ # the store holds this sample: it covers the graphs the request recorded; unpersisted, they stay dirty+ module._unsloth_video_graphs = graphs+ module._unsloth_video_graph_gen = generation_COUNTS["recorded"] += 1if logger is not None:logger.info("video.dit_resident: request peak %d MiB above its start recorded", extra)@@ -332,8 +495,10 @@ def fit_for_request(_keep_groups_resident(module, room, dev, logger)promoted = max(0, resident_mib(module) - after_release)kept = resident_mib(module)+ dev = _open_window(module, dev)torch.cuda.reset_peak_memory_stats(dev)module._unsloth_video_pending = (work, int(torch.cuda.memory_allocated(dev)), dev)+ module._unsloth_video_failed_captures = _failed_captures()try:# release_resident_groups' restore reads it back; keep it at this request's roommodule._unsloth_resident_room = room if room > 0 else Nonediff --git a/studio/backend/tests/test_diffusion_learned_peaks.py b/studio/backend/tests/test_diffusion_learned_peaks.pynew file mode 100644index 00000000000..8e27bc2d792--- /dev/null+++ b/studio/backend/tests/test_diffusion_learned_peaks.py@@ -0,0 +1,2162 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Learned activation peaks (``diffusion_measured_peaks``), the per-phase seed for families the peak table lacks,+and the one lower-residency retry after a transient OOM during group-offload setup (``diffusion_memory``).++The FLUX.2-klein-4B case uses the estimates a real auto load logged on a 16 GB card (int8 DiT, bf16 text encoder):+``safe_device_budget_mib 13132``, group offload with the text encoders streamed, and the whole DiT streamed although+the load peaked at 4.6 GiB. CPU-only; loaded sizes and the torchao check are stubbed.+"""++from __future__ import annotations++import json+import types++import pytest++import core.inference.diffusion_measured_peaks as mp+import core.inference.diffusion_memory as dm+import core.inference.video_stream_residency as vsr++KLEIN_LOADED = {+ "transformer": (4091, "dit"),+ "text_encoder": (7629, "text_encoder"),+ "vae": (160, "other"),+}+KLEIN_FLAT = dict(model_dense_mib = 11911, companion_dense_mib = 7820, text_encoder_dense_mib = 7629)+++def _memory(budget_mib: int, total_mib: int) -> "dm.DeviceMemory":+ reserve = dm._reserve_mib("discrete_vram", total_mib)+ return dm.DeviceMemory("cuda", "cuda", "discrete_vram", budget_mib + reserve, total_mib)+++def _klein_flat_plan(budget_mib: int = 13132, total_mib: int = 16380):+ plan = dm.plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = _memory(budget_mib, total_mib),+ runtime_headroom_mib = dm.estimate_image_runtime_mib(+ width = None, height = None, family = "flux.2-klein"+ ),+ requested_mode = None,+ **KLEIN_FLAT,+ )+ if plan.offload_policy == dm.OFFLOAD_MODEL:+ plan = dm.torchao_streaming_plan(plan)+ return plan+++def store(tmp_path, monkeypatch):+ path = tmp_path / "peaks.json"+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(path))+ monkeypatch.delenv(mp.LEARNED_PEAKS_ENV, raising = False)+ monkeypatch.delenv(dm.MEASURED_ACTIVATION_ENV, raising = False)+ monkeypatch.delenv(dm.PARTIAL_RESIDENT_ENV, raising = False)+ mp._cache.update(path = None, mtime = None, data = None)+ monkeypatch.setattr(+ mp, "current_device_name", lambda: "NVIDIA L4", raising = False+ ) # the planner's card+ return path+++def klein_pipe(monkeypatch, store):+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(KLEIN_LOADED))+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ monkeypatch.setattr(dm, "denoiser_quant_label", lambda pipe: "torchao:Int8Tensor")+ return types.SimpleNamespace(_unsloth_phase_seeds = True)+++def _refine(+ pipe,+ plan,+ family = "flux.2-klein",+ speed = "default",+):+ return dm.refine_plan_from_loaded_weights(pipe, plan, family = family, speed_mode = speed)+++# ---------------------------------------------------------------------------------------------------- seeds+++def test_family_missing_from_peak_table_is_seeded_from_phase_calibration(store):+ assert "flux.2-klein" not in dm._MEASURED_IMAGE_PEAK_MIB+ # worst of encoder 1516 / denoise 1160 / untiled decode 2645 x 1.15, rounded up to 256 MiB+ assert dm.measured_image_runtime_mib("flux.2-klein", "default", phase_seeds = True) == 3072+ assert dm.measured_image_runtime_mib("flux.2-klein", "max", phase_seeds = True) == 3328+ # eager tiers and unseeded families still keep the flat estimate+ assert dm.measured_image_runtime_mib("flux.2-klein", "off") is None+ assert dm.measured_image_runtime_mib("hidream", "default") is None+ # the peak table still wins where it has the family (Qwen-Image-2.1 decode is tiled: 1849, not 7648)+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "default") == 2304+++def test_klein_16gb_keeps_whole_dit_resident(klein_pipe):+ plan = _klein_flat_plan()+ assert (+ plan.offload_policy == dm.OFFLOAD_GROUP+ and plan.stream_text_encoders+ and plan.stream_transformer+ )+ assert plan.resident_transformer_mib is None+ new = _refine(klein_pipe, plan)+ assert new is not plan+ assert new.resident_transformer_mib == 4091 # every DiT group resident+ assert new.estimates["measured_runtime_headroom_mib"] == 3072+ assert klein_pipe._unsloth_peak_identity == ("flux.2-klein", "torchao:Int8Tensor", "default")+++# ---------------------------------------------------------------------------------------------------- store+++def test_bucket_is_power_of_two_megapixels():+ assert mp.resolution_bucket(256, 256) == mp._MIN_BUCKET_PIXELS+ assert mp.resolution_bucket(1024, 1024) == 1024 * 1024+ assert mp.resolution_bucket(1280, 720) == 1024 * 1024+ assert mp.resolution_bucket(1024, 1024, 2) == 2 * 1024 * 1024+ assert mp.resolution_bucket(1536, 1536) == 4 * 1024 * 1024+++def test_record_keeps_max_and_survives_a_fresh_process(store):+ kw = dict(family = "flux.2-klein", quant = "torchao:Int8Tensor", speed_mode = "default")+ assert mp.lookup(**kw) is None+ assert mp.record(**kw, width = 1024, height = 1024, peak_mib = 2000, device_name = "NVIDIA L4")+ assert not mp.record(**kw, width = 1024, height = 1024, peak_mib = 1500) # never lowered+ assert mp.record(**kw, width = 1024, height = 1024, peak_mib = 2600)+ mp._cache.update(path = None, mtime = None, data = None) # what a new process sees+ data = json.loads(store.read_text())+ assert data["entries"]["flux.2-klein|torchao:Int8Tensor|default|f1|NVIDIA L4"]["peaks"] == {+ "1048576": 2600+ }+ assert mp.lookup(**kw, width = 1024, height = 1024) == 2600+ # a smaller request answers from the covering bucket, a larger one scales the largest linearly+ assert mp.lookup(**kw, width = 512, height = 512) == 2600+ assert mp.lookup(**kw, width = 2048, height = 2048) == 4 * 2600+ # other quant / tier / frame count: separate keys+ assert mp.lookup(family = "flux.2-klein", quant = "dense:bfloat16", speed_mode = "default") is None+ assert mp.lookup(**{**kw, "speed_mode": "max"}) is None+ assert mp.lookup(**kw, frames = 49) is None+++def test_store_off_and_corrupt_file(store, monkeypatch):+ kw = dict(family = "z-image", quant = "q", speed_mode = "default")+ store.write_text("{not json")+ assert mp.lookup(**kw) is None+ assert mp.record(**kw, width = 1024, height = 1024, peak_mib = 1000)+ assert mp.lookup(**kw) == 1000+ monkeypatch.setenv(mp.LEARNED_PEAKS_ENV, "0")+ assert mp.lookup(**kw) is None+ assert not mp.record(**kw, width = 1024, height = 1024, peak_mib = 5000)+++def test_learned_peak_raises_a_seed_but_never_lowers_it(store):+ seed = dm.measured_image_runtime_mib(+ "flux.2-klein", "default", phase_seeds = True, quant = "torchao:Int8Tensor"+ )+ assert seed == 3072+ kw = dict(family = "flux.2-klein", quant = "torchao:Int8Tensor", speed_mode = "default")+ mp.record(**kw, width = 1024, height = 1024, peak_mib = 1000)+ assert (+ dm.measured_image_runtime_mib(+ "flux.2-klein", "default", phase_seeds = True, quant = "torchao:Int8Tensor"+ )+ == 3072+ )+ mp.record(**kw, width = 1024, height = 1024, peak_mib = 4000)+ # 4000 x 1.15 = 4600 -> 4608+ assert (+ dm.measured_image_runtime_mib(+ "flux.2-klein", "default", phase_seeds = True, quant = "torchao:Int8Tensor"+ )+ == 4608+ )+ # without a quant label (callers that never measured) the seed alone answers+ assert dm.measured_image_runtime_mib("flux.2-klein", "default", phase_seeds = True) == 3072+++def test_unseeded_family_is_placed_from_its_learned_peak_on_the_next_load(klein_pipe):+ plan = _klein_flat_plan()+ assert (+ _refine(klein_pipe, plan, family = "some-new-dit") is plan+ ) # nothing measured yet: flat plan+ mp.record(+ family = "some-new-dit",+ quant = "torchao:Int8Tensor",+ speed_mode = "default",+ width = 1024,+ height = 1024,+ peak_mib = 2600,+ )+ new = _refine(klein_pipe, plan, family = "some-new-dit")+ assert new.resident_transformer_mib == 4091+ assert new.estimates["measured_runtime_headroom_mib"] == 3072 # 2600 x 1.15 -> 3072+++def test_request_guard_reads_the_learned_peak(klein_pipe):+ plan = _klein_flat_plan()+ new = _refine(klein_pipe, plan)+ klein_pipe._unsloth_measured_reserve = (3072, "flux.2-klein", "default", None, None)+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0+ mp.record(+ family = "flux.2-klein",+ quant = "torchao:Int8Tensor",+ speed_mode = "default",+ width = 1024,+ height = 1024,+ peak_mib = 4000,+ )+ assert new is not plan+ # seeded family: a peak recorded by this session's own render places the next load, never this one's requests+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0+ # 2048x2048: 4 x 2645 x 1.15 = 12167 -> 12288+ assert dm.measured_request_extra_mib(klein_pipe, width = 2048, height = 2048) == 12288 - 3072+++def test_request_guard_of_an_unseeded_family_reads_its_learned_peak(klein_pipe):+ kw = dict(family = "some-new-dit", quant = "torchao:Int8Tensor", speed_mode = "default")+ mp.record(**kw, width = 1024, height = 1024, peak_mib = 2600)+ new = _refine(klein_pipe, _klein_flat_plan(), family = "some-new-dit")+ assert new.estimates["measured_runtime_headroom_mib"] == 3072+ klein_pipe._unsloth_measured_reserve = (3072, "some-new-dit", "default", None, None)+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0+ mp.record(**kw, width = 1024, height = 1024, peak_mib = 4000)+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 4608 - 3072+++# ---------------------------------------------------------------------------------------------------- video+++def test_video_request_peaks_persist_across_processes(store, monkeypatch):+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)+ monkeypatch.setattr(vsr, "_PEAKS", {})+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")+ work = 1280 * 704 * 49+ assert vsr.measured_extra_mib(key, work) is None++ class _Torch:+ class cuda: # noqa: N801+ @staticmethod+ def max_memory_allocated(dev):+ return (3000 + 100) << 20++ module = types.SimpleNamespace(_unsloth_video_pending = (work, 100 << 20, "cuda:0"))+ module._unsloth_video_calibration_key = key+ monkeypatch.setitem(__import__("sys").modules, "torch", _Torch)+ assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) == 3000+ monkeypatch.undo()+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(store))+ monkeypatch.setattr(vsr, "_PEAKS", {}) # a new process+ mp._cache.update(path = None, mtime = None, data = None)+ assert vsr.measured_extra_mib(key, work) == 3000+ assert vsr.measured_extra_mib(key, work + 1) is None # a larger request is not covered+++# ---------------------------------------------------------------------------------------------------- OOM retry+++class _OOM(RuntimeError):+ pass+++_OOM.__name__ = "OutOfMemoryError"+++def _resident_dit_plan():+ mem = _memory(13132, 16380)+ return dm.MemoryPlan(+ requested_mode = "auto",+ offload_policy = dm.OFFLOAD_GROUP,+ vae_tiling = False,+ vae_slicing = True,+ device_memory = mem,+ estimates = {"safe_device_budget_mib": 13132},+ reasons = ("flat",),+ stream_text_encoders = True,+ stream_transformer = False,+ )+++def test_transient_setup_oom_replans_once_at_lower_residency(monkeypatch):+ calls = []++ def fake_apply(pipe, device, logger, **kw):+ calls.append(kw)+ if len(calls) == 1:+ dm._set_group_offload_failure(+ pipe, _OOM("CUDA out of memory. Tried to allocate 20.00 MiB")+ )+ return False+ return True++ monkeypatch.setattr(dm, "_apply_group_offload", fake_apply)+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ pipe = types.SimpleNamespace(components = {})+ plan = _resident_dit_plan()+ policy, _ = dm.apply_memory_plan(pipe, plan, device = "cuda")+ assert policy == dm.OFFLOAD_GROUP+ assert calls[0] == {"stream_text_encoders": True, "stream_transformer": False}+ assert calls[1] == {"stream_text_encoders": True} # the DiT streams on the retry+ applied = dm.applied_plan(pipe, plan)+ assert applied is not plan and applied.stream_transformer and applied.stream_text_encoders+ assert applied.resident_transformer_mib is None and applied.estimates["setup_oom_retry"]+ assert not dm.plan_keeps_transformer_resident(applied)+++def test_non_oom_setup_failure_still_refuses(monkeypatch):+ def fake_apply(pipe, device, logger, **kw):+ dm._set_group_offload_failure(pipe, ValueError("no block list"))+ return False++ monkeypatch.setattr(dm, "_apply_group_offload", fake_apply)+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ pipe = types.SimpleNamespace(components = {})+ with pytest.raises(RuntimeError, match = "could not be streamed beside the resident quantised"):+ dm.apply_memory_plan(pipe, _resident_dit_plan(), device = "cuda")+ assert dm.applied_plan(pipe, "plan") == "plan"+++def test_oom_retry_that_fails_again_refuses(monkeypatch):+ def fake_apply(pipe, device, logger, **kw):+ dm._set_group_offload_failure(pipe, _OOM("out of memory"))+ return False++ monkeypatch.setattr(dm, "_apply_group_offload", fake_apply)+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ pipe = types.SimpleNamespace(components = {})+ with pytest.raises(RuntimeError, match = "could not be streamed beside the resident quantised"):+ dm.apply_memory_plan(pipe, _resident_dit_plan(), device = "cuda")+++# ---------------------------------------------------------------------------------------------------- GGUF pinning+++def test_background_pinner_pins_gguf_host_copies():+ """GGUF host copies were skipped (``type(src) is torch.Tensor``), so a streamed GGUF denoiser never had a pinned+ host copy and its CUDA graph refused capture ("a streamed group's host copy is not pinned")."""+ torch = pytest.importorskip("torch")+ gguf_utils = pytest.importorskip("diffusers.quantizers.gguf.utils")+ gguf = gguf_utils.GGUFParameter(torch.zeros(4, 144, dtype = torch.uint8), quant_type = 12)+ dense = torch.zeros(8)+ group = types.SimpleNamespace(cpu_param_dict = {"w": gguf.data, "b": dense})+ pinner = dm._GroupPinner.__new__(dm._GroupPinner)+ picked = {key for key, _ in pinner._unpinned(group)}+ assert picked == {"w", "b"}+++# ---------------------------------------------------------------------------------------------------- recording++[email protected](autouse = True)+def _attention_scaling_known(request, monkeypatch):+ """Recording tests run without a device target; the SDPA verdict counts as known unless a test says otherwise."""+ if "attention_scaling_unknown" not in request.node.name:+ monkeypatch.setattr(+ "core.inference.diffusion._attention_scaling_known", lambda target, engaged = None: True+ )+++class _Probe:+ def __init__(+ self,+ peak,+ failed = 0,+ ):+ self.device = "cuda:0"+ self.failed_captures = failed+ self._peak = peak++ def finish(self):+ return self._peak+++def test_render_records_its_peak_unless_a_capture_failed(store, monkeypatch):+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")+ pipe = types.SimpleNamespace(+ _unsloth_peak_identity = ("flux.2-klein", "torchao:Int8Tensor", "default")+ )+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default")+ kw = dict(family = "flux.2-klein", quant = "torchao:Int8Tensor", speed_mode = "default")+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 1)+ diffusion._record_peak_probe(+ state, _Probe(5000, failed = 0), width = 1024, height = 1024, batch_size = 1+ )+ assert mp.lookup(**kw) is None # an absorbed capture OOM: the peak is the failed attempt's+ diffusion._record_peak_probe(+ state, _Probe(2400, failed = 1), width = 1024, height = 1024, batch_size = 1+ )+ assert mp.lookup(**kw) == 2400+ assert (+ json.loads(store.read_text())["entries"][+ "flux.2-klein|torchao:Int8Tensor|default|f1|NVIDIA L4"+ ]["device"]+ == "NVIDIA L4"+ )+++# ---------------------------------------------------------------------------------------------------- tied weights+++def test_tied_group_closures():+ shared, a, b, c = object(), object(), object(), object()+ g1 = types.SimpleNamespace(t = [shared, a])+ g2 = types.SimpleNamespace(t = [b])+ g3 = types.SimpleNamespace(t = [c, shared])+ closures = dm._tied_group_closures([g1, g2, g3], lambda g: g.t)+ assert closures[id(g1)] == [g1, g3] and closures[id(g3)] == [g1, g3]+ assert id(g2) not in closures+++def _tied_tiny_encoder(torch):+ """A Qwen3-style leaf: an embedding and an LM head sharing one weight (FLUX.2-klein's text encoder ties them)."""++ class Tiny(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.embed = torch.nn.Embedding(4096, 128)+ self.mid = torch.nn.Linear(128, 128)+ self.head = torch.nn.Linear(128, 4096, bias = False)+ self.head.weight = self.embed.weight++ def forward(self, ids):+ return self.head(self.mid(self.embed(ids)))++ return Tiny().to(torch.bfloat16)+++def test_partial_residency_keeps_tied_weights_together():+ """Base: the embedding group went resident while the tied head streamed; the head's offload pointed the shared+ weight back at the host and the next encode failed (index on cuda, weight on cpu)."""+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA and diffusers group offloading")+ from diffusers.hooks import apply_group_offloading++ te = _tied_tiny_encoder(torch)+ dev = torch.device("cuda")+ apply_group_offloading(+ te,+ onload_device = dev,+ offload_device = torch.device("cpu"),+ offload_type = "leaf_level",+ use_stream = True,+ non_blocking = True,+ record_stream = True,+ )+ # 1 MiB: room for the 1 MiB shared weight once, not for the embedding and the head counted apart+ dm._keep_groups_resident(te, 1, dev)+ groups = dm._offload_groups(te)+ tied = [g for g in groups if te.embed in g.modules or te.head in g.modules]+ assert len(tied) == 2+ assert len({bool(getattr(g, "_unsloth_resident", False)) for g in tied}) == 1+ for n in (7, 11, 5):+ with torch.no_grad():+ te(torch.randint(0, 4096, (1, n), device = dev))+ restore = dm.release_resident_groups(types.SimpleNamespace(components = {"text_encoder": te}), 1)+ assert not any(getattr(g, "_unsloth_resident", False) for g in tied)+ with torch.no_grad():+ te(torch.randint(0, 4096, (1, 9), device = dev))+ restore()+ assert all(getattr(g, "_unsloth_resident", False) for g in tied)+ with torch.no_grad():+ te(torch.randint(0, 4096, (1, 4), device = dev))+++def test_video_peak_of_a_request_with_a_failed_capture_is_not_persisted(store, monkeypatch):+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(vsr, "_PEAKS", {})+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "streaming", "cuda:0")+ work = 832 * 480 * 33++ class _Torch:+ class cuda: # noqa: N801+ @staticmethod+ def max_memory_allocated(dev):+ return 9000 << 20++ module = types.SimpleNamespace(+ _unsloth_video_pending = (work, 0, "cuda:0"),+ _unsloth_video_calibration_key = key,+ _unsloth_video_failed_captures = 0,+ )+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 1)+ monkeypatch.setitem(__import__("sys").modules, "torch", _Torch)+ assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) == 9000+ assert mp.work_peaks(key) == {} # kept in this process only+++def test_resident_dit_plan_clamps_encoder_room_and_says_so(klein_pipe):+ """L4 (22.5 GB): the flat plan already kept the DiT resident; the encoder room exceeded the encoders and the log+ read '0 MiB of the transformer and 10970 MiB of the 7673 MiB encoders'."""+ plan = _klein_flat_plan(budget_mib = 19500, total_mib = 23034)+ assert (+ plan.offload_policy == dm.OFFLOAD_GROUP+ and not plan.stream_transformer+ and plan.stream_text_encoders+ )+ new = _refine(klein_pipe, plan)+ assert new.resident_text_encoder_mib == 7629+ assert "transformer is resident" in new.reasons[-1]+++# ---------------------------------------------------------------------------------------------------- review follow-ups+++def test_render_served_by_a_conditioning_cache_is_not_recorded(store, monkeypatch):+ """A cache hit skips the text encoder: an unseeded family would place its next load from the smaller peak."""+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")+ stats = {"hits": 0, "misses": 1}+ pipe = types.SimpleNamespace(+ _unsloth_peak_identity = ("some-new-dit", "torchao:Int8Tensor", "default"),+ _unsloth_cond_cache_stats = stats,+ )+ state = types.SimpleNamespace(pipe = pipe, device = "cuda:0", speed_mode = "default")+ probe = _Probe(1800)+ probe.conditioning_hits = diffusion._conditioning_cache_hits(pipe)+ stats["hits"] += 1 # this render's prompt came from the cache+ diffusion._record_peak_probe(state, probe, width = 1024, height = 1024, batch_size = 1)+ kw = dict(family = "some-new-dit", quant = "torchao:Int8Tensor", speed_mode = "default")+ assert mp.lookup(**kw) is None+ probe = _Probe(2600)+ probe.conditioning_hits = diffusion._conditioning_cache_hits(pipe)+ diffusion._record_peak_probe(state, probe, width = 1024, height = 1024, batch_size = 1)+ assert mp.lookup(**kw) == 2600+++def test_learned_peaks_are_kept_per_gpu_model(store):+ kw = dict(+ family = "some-new-dit",+ quant = "torchao:Int8Tensor",+ speed_mode = "default",+ width = 1024,+ height = 1024,+ )+ mp.record(**kw, peak_mib = 2600, device_name = "NVIDIA L4")+ assert mp.lookup(**kw, device = "NVIDIA L4") == 2600+ assert mp.lookup(**kw, device = "NVIDIA GeForce RTX 4090") is None+++def test_peak_store_update_waits_for_another_process_lock(store):+ """The read / merge / replace takes an interprocess lock, so a concurrent writer cannot replace a higher peak."""+ fcntl = pytest.importorskip("fcntl")+ import threading++ store.parent.mkdir(parents = True, exist_ok = True)+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default", width = 1024, height = 1024)+ done = threading.Event()+ with open(+ str(store) + ".lock", "a+b"+ ) as other: # a separate open file description, like another process+ fcntl.flock(other.fileno(), fcntl.LOCK_EX)+ worker = threading.Thread(+ target = lambda: (mp.record(**kw, peak_mib = 2000, device_name = "g"), done.set())+ )+ worker.start()+ assert not done.wait(0.5) # blocked behind the other writer's lock+ fcntl.flock(other.fileno(), fcntl.LOCK_UN)+ worker.join(10)+ assert done.is_set() and mp.lookup(**kw, device = "g") == 2000+++def test_oom_after_partial_hooks_unwinds_and_replans(monkeypatch):+ """An OOM on the second DiT after the first had hooks used to re-raise past the lower-residency retry."""+ torch = pytest.importorskip("torch")+ hooks = pytest.importorskip("diffusers.hooks")+ from diffusers.hooks.hooks import HookRegistry, ModelHook++ calls = []+ marker = "_test_group_marker"++ def fake_apply(module, **kw):+ calls.append(module)+ if module is pipe.transformer_2 and calls.count(module) == 1:+ raise _OOM("CUDA out of memory. Tried to allocate 64.00 MiB")+ HookRegistry.check_if_exists_or_initialize(module).register_hook(ModelHook(), marker)++ monkeypatch.setattr(hooks, "apply_group_offloading", fake_apply)+ monkeypatch.setattr(dm, "install_group_offload_buffer_restore", lambda: None)+ monkeypatch.setattr(dm, "install_group_offload_hooks_eager", lambda: None)+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ real_remove = dm._remove_group_offload_hooks++ def remove(module):+ real_remove(module)+ registry = getattr(module, "_diffusers_hook", None)+ if registry is not None and registry.get_hook(marker) is not None:+ registry.remove_hook(marker, recurse = True)++ monkeypatch.setattr(dm, "_remove_group_offload_hooks", remove)+ a, b, te = torch.nn.Linear(4, 4), torch.nn.Linear(4, 4), torch.nn.Linear(4, 4)+ pipe = types.SimpleNamespace(+ transformer = a,+ transformer_2 = b,+ components = {"transformer": a, "transformer_2": b, "text_encoder": te},+ )+ plan = dm.replace(_resident_dit_plan(), stream_transformer = True, stream_text_encoders = False)+ policy, _ = dm.apply_memory_plan(pipe, plan, device = "cpu")+ assert policy == dm.OFFLOAD_GROUP+ assert dm.applied_plan(pipe, plan).estimates["setup_oom_retry"]+ assert calls[:2] == [a, b] and calls[2:4] == [a, b] # unwound, then both DiTs streamed again+++# ---------------------------------------------------------------------------------------------------- review round 2+++def test_peak_is_scaled_to_its_bucket_ceiling(store):+ kw = dict(family = "some-new-dit", quant = "torchao:Int8Tensor", speed_mode = "default")+ # 1088x1024 lands in the 2 MP bucket; a 2 MP request must not get the ~1 MP peak back unchanged+ mp.record(**kw, width = 1088, height = 1024, peak_mib = 1000)+ assert mp.lookup(**kw, width = 1448, height = 1448) >= 1000 * 2 * 1024 * 1024 // (1088 * 1024)+++def test_request_guard_scales_the_learned_peak_the_load_placed_from(klein_pipe):+ mp.record(+ family = "flux.2-klein",+ quant = "torchao:Int8Tensor",+ speed_mode = "default",+ width = 1024,+ height = 1024,+ peak_mib = 4000,+ )+ new = _refine(klein_pipe, _klein_flat_plan())+ assert new.estimates["measured_runtime_headroom_mib"] == 4608+ klein_pipe._unsloth_peak_identity = ("flux.2-klein", "torchao:Int8Tensor", "default")+ klein_pipe._unsloth_measured_reserve = (+ 4608,+ "flux.2-klein",+ "default",+ None,+ None,+ new.estimates.get("measured_learned_peak_mib"),+ )+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0+ # 2 MP: 2 x 4000 x 1.15 = 9200 -> 9216, not the 2 x 2645 seed+ assert dm.measured_request_extra_mib(klein_pipe, width = 1448, height = 1448) >= 9216 - 4608+++def test_learned_peaks_are_keyed_by_attention_backend(klein_pipe):+ mp.record(+ family = "some-new-dit",+ quant = "torchao:Int8Tensor|attn:flash",+ speed_mode = "default",+ width = 1024,+ height = 1024,+ peak_mib = 2600,+ )+ plan = _klein_flat_plan()+ flash = dm.refine_plan_from_loaded_weights(+ klein_pipe, plan, family = "some-new-dit", speed_mode = "default", attention = "flash"+ )+ assert flash.estimates["measured_runtime_headroom_mib"] == 3072+ assert klein_pipe._unsloth_peak_identity[1] == "torchao:Int8Tensor|attn:flash"+ native = dm.refine_plan_from_loaded_weights(+ klein_pipe, plan, family = "some-new-dit", speed_mode = "default", attention = "native"+ )+ assert native is plan # a native-attention load never reuses the flash peak+++def test_load_passes_the_engaged_attention_backend_to_the_peak_identity():+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert (+ "peak_model,\n peak_vae,\n peak_te,\n peak_cache,"+ in src+ )+ assert 'attention = attention_engaged or "native",' in src+ assert 'plan.estimates.get("measured_learned_peak_mib"),' in src+++def test_video_peaks_merge_other_processes_updates(store, monkeypatch):+ monkeypatch.setattr(vsr, "_PEAKS", {})+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")+ work = 1280 * 704 * 49+ assert vsr.measured_extra_mib(key, work) is None # this process caches the empty answer+ assert mp.record_work(vsr._store_key(key), work, 3000) # another process records+ assert vsr.measured_extra_mib(key, work) == 3000+++def test_video_peaks_are_persisted_per_physical_gpu(store, monkeypatch):+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)+ monkeypatch.setattr(vsr, "_PEAKS", {})+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")+ work = 1280 * 704 * 49+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")+ module = types.SimpleNamespace(_unsloth_video_pending = None, _unsloth_video_calibration_key = key)+ vsr._PEAKS[key] = {work: 3000}+ module._unsloth_video_pending = (work, 0, "cuda:0")++ class _Torch:+ class cuda: # noqa: N801+ @staticmethod+ def max_memory_allocated(dev):+ return 3000 << 20++ with monkeypatch.context() as m:+ m.setitem(__import__("sys").modules, "torch", _Torch)+ vsr.record_request_peak(types.SimpleNamespace(transformer = module))+ stored = json.loads(store.read_text())["work"]+ assert list(stored) == [repr((key, "NVIDIA L4"))]+ vsr._PEAKS.clear()+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA H100") # another worker's cuda:0+ assert vsr.measured_extra_mib(key, work) is None+++def test_store_cap_evicts_the_oldest_entry_not_the_alphabetically_first(store, monkeypatch):+ monkeypatch.setattr(mp, "_MAX_ENTRIES", 3)+ for fam in ("m", "n", "o"):+ mp.record(+ family = fam, quant = "q", speed_mode = "default", width = 1024, height = 1024, peak_mib = 100+ )+ mp.record(family = "a", quant = "q", speed_mode = "default", width = 1024, height = 1024, peak_mib = 100)+ entries = json.loads(store.read_text())["entries"]+ assert [k.split("|")[0] for k in entries] == ["n", "o", "a"]+ for fam in ("m", "n", "o"):+ assert mp.record_work(("k", fam), 10, 100)+ assert mp.record_work(("k", "a"), 10, 100)+ assert list(json.loads(store.read_text())["work"]) == [repr(("k", f)) for f in ("n", "o", "a")]+++# ---------------------------------------------------------------------------------------------------- review round 3+++def test_learned_peaks_are_keyed_by_the_loaded_model(klein_pipe):+ mp.record(+ family = "some-new-dit",+ quant = "torchao:Int8Tensor|model:org/dit-4b|attn:native",+ speed_mode = "default",+ width = 1024,+ height = 1024,+ peak_mib = 2600,+ )+ plan = _klein_flat_plan()+ kw = dict(family = "some-new-dit", speed_mode = "default", attention = "native")+ small = dm.refine_plan_from_loaded_weights(klein_pipe, plan, model = "org/dit-4b", **kw)+ assert small.estimates["measured_runtime_headroom_mib"] == 3072+ assert klein_pipe._unsloth_peak_identity[1] == "torchao:Int8Tensor|model:org/dit-4b|attn:native"+ # a larger variant of the same family never reuses the smaller one's peak+ assert dm.refine_plan_from_loaded_weights(klein_pipe, plan, model = "org/dit-9b", **kw) is plan+++def test_load_passes_the_model_to_the_peak_identity():+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert (+ "peak_model,\n peak_vae,\n peak_te,\n peak_cache,"+ in src+ )+ assert "model = peak_model," in src+ assert 'plan.estimates.get("measured_learned_peak_pixels"),' in src+++def test_request_guard_keeps_the_learned_peaks_source_bucket(klein_pipe):+ mp.record(+ family = "flux.2-klein",+ quant = "torchao:Int8Tensor",+ speed_mode = "default",+ width = 1448,+ height = 1448,+ peak_mib = 8000,+ )+ new = _refine(klein_pipe, _klein_flat_plan())+ learned = new.estimates["measured_learned_peak_mib"]+ assert learned >= 8000+ headroom = new.estimates["measured_runtime_headroom_mib"]+ klein_pipe._unsloth_peak_identity = ("flux.2-klein", "torchao:Int8Tensor", "default")+ klein_pipe._unsloth_measured_reserve = (+ headroom,+ "flux.2-klein",+ "default",+ None,+ None,+ learned,+ new.estimates.get("measured_learned_peak_pixels"),+ )+ # the stored peak already covers 2 MP: a 2 MP request needs nothing beyond the load's reserve+ assert dm.measured_request_extra_mib(klein_pipe, width = 1448, height = 1448) == 0+++# ---------------------------------------------------------------------------------------------------- review round 4+++class _FakeCudaTorch:+ """Allocator-wide peak stats, as torch keeps them: a reset by one render erases another's high-water mark."""++ def __init__(self):+ self.allocated = 0+ self.peak = 0+ torch_self = self++ class _Dev:+ def __init__(self, *args):+ self.type = "cuda"+ self.index = 0++ def __str__(self):+ return "cuda:0"++ class cuda: # noqa: N801+ is_available = staticmethod(lambda: True)+ current_device = staticmethod(lambda: 0)+ synchronize = staticmethod(lambda dev = None: None)+ memory_allocated = staticmethod(lambda dev = None: torch_self.allocated)+ max_memory_allocated = staticmethod(lambda dev = None: torch_self.peak)++ @staticmethod+ def reset_peak_memory_stats(dev = None):+ torch_self.peak = torch_self.allocated++ self.device = _Dev+ self.cuda = cuda++ def alloc(self, mib):+ self.allocated += mib << 20+ self.peak = max(self.peak, self.allocated)++ def free(self, mib):+ self.allocated -= mib << 20+++def test_overlapping_peak_probes_on_one_card_record_nothing(monkeypatch):+ fake = _FakeCudaTorch()+ monkeypatch.setitem(__import__("sys").modules, "torch", fake)+ first = mp.PeakProbe("cuda:0")+ fake.alloc(5000) # the first render's largest phase+ fake.free(5000)+ second = mp.PeakProbe("cuda:0") # a render on the other executor resets the card's peak+ fake.alloc(100)+ fake.free(100)+ assert first.finish() is None # would read 100 MiB, not 5000+ assert second.finish() is None+ alone = mp.PeakProbe("cuda:0")+ fake.alloc(700)+ assert alone.finish() == 700+ del alone+ dropped = mp.PeakProbe("cuda:0") # a failed render never finishes its probe+ del dropped+ after = mp.PeakProbe("cuda:0")+ fake.alloc(300)+ assert after.finish() == 300 # not spoiled by the abandoned probe+++def test_partial_hook_failure_in_the_lower_residency_retry_propagates(monkeypatch):+ calls = []++ def fake_apply(pipe, device, logger, **kw):+ calls.append(kw)+ if len(calls) == 1:+ dm._set_group_offload_failure(pipe, _OOM("CUDA out of memory"))+ return False+ raise RuntimeError("group offload failed after installing hooks on 1 module(s)")++ monkeypatch.setattr(dm, "_apply_group_offload", fake_apply)+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ pipe = types.SimpleNamespace(components = {})+ with pytest.raises(RuntimeError, match = "after installing hooks"):+ dm.apply_memory_plan(pipe, _resident_dit_plan(), device = "cuda")+ assert len(calls) == 2+++def test_learned_peaks_are_keyed_by_vae_savers(klein_pipe):+ mp.record(+ family = "some-new-dit",+ quant = "torchao:Int8Tensor|model:org/dit|vae:tiled+sliced|attn:native",+ speed_mode = "default",+ width = 1024,+ height = 1024,+ peak_mib = 2600,+ )+ plan = _klein_flat_plan()+ kw = dict(family = "some-new-dit", speed_mode = "default", attention = "native", model = "org/dit")+ tiled = dm.refine_plan_from_loaded_weights(klein_pipe, plan, vae = "tiled+sliced", **kw)+ assert tiled.estimates["measured_runtime_headroom_mib"] == 3072+ # an untiled load never reuses the tiled decode's peak+ assert dm.refine_plan_from_loaded_weights(klein_pipe, plan, vae = "full", **kw) is plan+ assert (+ dm.vae_saver_tag(True, True) == "tiled+sliced" and dm.vae_saver_tag(False, False) == "full"+ )+++def test_render_with_a_vae_saver_the_key_lacks_is_not_probed(monkeypatch):+ import core.inference.diffusion as diffusion++ class _Probe:+ def __init__(self, device):+ self.device = "cuda:0"++ monkeypatch.setattr(mp, "PeakProbe", _Probe)+ monkeypatch.setattr(mp, "enabled", lambda: True)+ vae = types.SimpleNamespace(use_tiling = False, use_slicing = False)+ pipe = types.SimpleNamespace(+ _unsloth_peak_identity = ("some-new-dit", "q|model:m|vae:full|attn:native", "default"),+ vae = vae,+ )+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)+ state = types.SimpleNamespace(pipe = pipe, device = "cuda:0", speed_mode = "default")+ assert diffusion._start_peak_probe(state) is not None+ vae.use_tiling = True # the activation guard tiled this request's decode+ assert diffusion._start_peak_probe(state) is None+++def test_load_passes_the_vae_savers_to_the_peak_identity():+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert "peak_vae = planned_vae_saver_tag(pipe, plan)" in src+ assert "vae = peak_vae," in src+++def test_video_request_overlapped_by_another_render_records_nothing(store, monkeypatch):+ monkeypatch.setattr(vsr, "_PEAKS", {})+ work = 1280 * 704 * 49+ window = mp.open_window("cuda:0")+ image = mp.open_window("cuda:0") # an image render armed on the same card mid-request+ module = types.SimpleNamespace(+ _unsloth_video_pending = (work, 0, "cuda:0"),+ _unsloth_video_window = ("cuda:0", window),+ _unsloth_video_calibration_key = None,+ )++ class _Torch:+ class cuda: # noqa: N801+ @staticmethod+ def max_memory_allocated(dev):+ return 3000 << 20++ with monkeypatch.context() as m:+ m.setitem(__import__("sys").modules, "torch", _Torch)+ assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) is None+ assert not mp.close_window("cuda:0", image)+++# ---------------------------------------------------------------------------------------------------- review round 5+++def test_learned_peaks_are_keyed_by_text_encoder_precision(klein_pipe):+ mp.record(+ family = "some-new-dit",+ quant = "torchao:Int8Tensor|model:org/dit|te:int8|vae:full|attn:native",+ speed_mode = "default",+ width = 1024,+ height = 1024,+ peak_mib = 2600,+ )+ plan = _klein_flat_plan()+ kw = dict(+ family = "some-new-dit", speed_mode = "default", attention = "native", model = "org/dit", vae = "full"+ )+ quantised = dm.refine_plan_from_loaded_weights(klein_pipe, plan, te = "int8", **kw)+ assert quantised.estimates["measured_runtime_headroom_mib"] == 3072+ # a dense encoder never reuses the quantised encoder's peak+ assert dm.refine_plan_from_loaded_weights(klein_pipe, plan, te = "dense", **kw) is plan+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert "peak_te = _peak_te_label(te_quant, te_outcome)" in src and "te = peak_te," in src+++def test_vae_key_names_only_the_savers_the_pipeline_can_engage():+ plan = types.SimpleNamespace(vae_tiling = True, vae_slicing = True)+ untileable = types.SimpleNamespace(+ vae = types.SimpleNamespace(enable_slicing = lambda: None, use_slicing = False)+ )+ assert dm.planned_vae_saver_tag(untileable, plan) == "sliced"+ both = types.SimpleNamespace(+ vae = types.SimpleNamespace(+ enable_slicing = lambda: None,+ enable_tiling = lambda: None,+ use_slicing = False,+ use_tiling = False,+ )+ )+ assert dm.planned_vae_saver_tag(both, plan) == "tiled+sliced"+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ # after placement the key follows the savers that actually engaged, and the reserve follows the key+ after_apply = src.index("plan = applied_plan(pipe, plan)")+ assert (+ 'with_label_part(identity[1], "vae", live_vae_saver_tag(pipe))'+ in src[after_apply : after_apply + 800]+ )+ assert "pipe._unsloth_peak_static = True" in src[after_apply : after_apply + 800]+++def test_deferred_profile_sizes_requests_from_the_flat_estimate(klein_pipe):+ klein_pipe._unsloth_phase_seeds = False+ klein_pipe._unsloth_peak_identity = ("some-new-dit", "torchao:Int8Tensor", "default")+ klein_pipe._unsloth_measured_reserve = (+ 3072,+ "some-new-dit",+ "off",+ None,+ None,+ 2600,+ 1 << 20,+ False,+ )+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0+ klein_pipe._unsloth_peak_static = True # the deferred profile engaged after placement+ flat = dm.estimate_image_runtime_mib(+ width = 1024, height = 1024, batch_size = 1, family = "some-new-dit"+ )+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == flat - 3072+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ deferred = src.index("def _engage_deferred_profile")+ assert "state.pipe._unsloth_peak_static = True" in src[deferred : deferred + 4000]+++def test_render_under_another_attention_backend_than_the_key_is_not_probed(monkeypatch):+ import core.inference.diffusion as diffusion++ class _Probe:+ def __init__(self, device):+ self.device = "cuda:0"++ monkeypatch.setattr(mp, "PeakProbe", _Probe)+ monkeypatch.setattr(mp, "enabled", lambda: True)+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)+ pipe = types.SimpleNamespace(+ _unsloth_peak_identity = ("x", "q|vae:full|attn:native", "default"),+ vae = types.SimpleNamespace(use_tiling = False, use_slicing = False),+ )+ state = types.SimpleNamespace(+ pipe = pipe, device = "cuda:0", attention_backend = None, speed_mode = "default"+ )+ assert diffusion._start_peak_probe(state) is not None+ state.attention_backend = "flash"+ assert diffusion._start_peak_probe(state) is None+++def test_lower_residency_retry_returns_torchao_weights_without_module_to(monkeypatch):+ torch = pytest.importorskip("torch")++ class _Torchao(torch.nn.Module):+ def to(self, *args, **kwargs):+ raise RuntimeError("Module.to cannot move a torchao weight on its host copy")++ returned = []+ monkeypatch.setattr(+ dm, "_return_module_tensors", lambda module, onload: returned.append(onload.type)+ )+ calls = []++ def fake_apply(pipe, device, logger, **kw):+ calls.append(kw)+ if len(calls) == 1:+ dm._set_group_offload_failure(pipe, _OOM("CUDA out of memory"))+ return False+ return True++ monkeypatch.setattr(dm, "_apply_group_offload", fake_apply)+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ pipe = types.SimpleNamespace(components = {"transformer": _Torchao()})+ policy, _ = dm.apply_memory_plan(pipe, _resident_dit_plan(), device = "cuda")+ assert policy == dm.OFFLOAD_GROUP and len(calls) == 2+ assert returned == ["cpu"]+++def test_quadratic_attention_peaks_scale_with_pixels_squared(store):+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default")+ mp.record(**kw, width = 1088, height = 1024, peak_mib = 1000, quadratic = True)+ ratio = 2 * 1024 * 1024 / float(1088 * 1024)+ assert mp.lookup(**kw, width = 1448, height = 1448) >= int(1000 * ratio * ratio)+ # a request above every recorded bucket scales the largest one squared too+ assert mp.lookup(**kw, width = 2048, height = 2048) >= int(1000 * ratio * ratio) * 4+++def test_record_passes_the_quadratic_attention_verdict():+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert "quadratic = _quadratic_attention(target, engaged, state.pipe)," in src+ assert "if not _attention_scaling_known(target, engaged):" in src+ assert "target = guard_target," in src+++def test_peak_measured_with_no_streamed_denoiser_is_raised_to_the_streamed_baseline(+ store, monkeypatch+):+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: 900)+ resident = types.SimpleNamespace(components = {"transformer": object()})+ monkeypatch.setattr(+ dm, "_offload_groups", lambda module: [types.SimpleNamespace(_unsloth_resident = True)]+ )+ assert dm.stream_window_pad_mib(resident) == 900+ monkeypatch.setattr(+ dm, "_offload_groups", lambda module: [types.SimpleNamespace(_unsloth_resident = False)]+ )+ assert dm.stream_window_pad_mib(resident) == 0+ pipe = types.SimpleNamespace(_unsloth_peak_identity = ("some-new-dit", "q", "default"))+ probe = _Probe(2400)+ probe.window_pad = 900+ diffusion._record_peak_probe(+ types.SimpleNamespace(pipe = pipe, speed_mode = "default"),+ probe,+ width = 1024,+ height = 1024,+ batch_size = 1,+ )+ assert mp.lookup(family = "some-new-dit", quant = "q", speed_mode = "default") == 3300+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert "window_pad = stream_window_pad_mib(pipe)" in src+++def test_residency_fit_registers_its_video_measurement_window():+ src = (__import__("pathlib").Path(vsr.__file__)).read_text(encoding = "utf-8")+ fit = src.index("def fit_for_request")+ assert "dev = _open_window(module, dev)" in src[fit:]+ assert src.count("dev = _open_window(module, dev)") == 2+++# ---------------------------------------------------------------------------------------------------- review round 6+++def test_request_guard_scales_a_reserved_peak_to_the_request_bucket(klein_pipe):+ klein_pipe._unsloth_peak_identity = ("flux.2-klein", "torchao:Int8Tensor", "default")+ klein_pipe._unsloth_measured_reserve = (+ 4608,+ "flux.2-klein",+ "default",+ None,+ None,+ 4000,+ 1 << 20,+ False,+ )+ # 1088x1024 crosses into the 2 MP bucket: 4000 x 2 x 1.15 = 9200 -> 9216, not 1.06x+ assert dm.measured_request_extra_mib(klein_pipe, width = 1088, height = 1024) == 9216 - 4608+ klein_pipe._unsloth_measured_reserve = (+ 4608,+ "flux.2-klein",+ "default",+ None,+ None,+ 4000,+ 1 << 20,+ True,+ )+ # a math-attention entry scales squared: 4000 x 4 x 1.15 = 18400 -> 18432+ assert dm.measured_request_extra_mib(klein_pipe, width = 1088, height = 1024) == 18432 - 4608+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0+++def test_peak_window_stays_open_until_the_high_water_mark_is_read(monkeypatch):+ fake = _FakeCudaTorch()+ read = fake.cuda.max_memory_allocated+ late = []++ def racing_read(dev = None):+ late.append(mp.open_window("cuda:0")) # a render arms between the sync and the read+ return read(dev)++ fake.cuda.max_memory_allocated = staticmethod(racing_read)+ monkeypatch.setitem(__import__("sys").modules, "torch", fake)+ probe = mp.PeakProbe("cuda:0")+ fake.alloc(500)+ assert probe.finish() is None+ assert not mp.close_window("cuda:0", late[0])+++def test_unobservable_vae_savers_use_and_record_no_learned_peak(store):+ pipe = types.SimpleNamespace(+ vae = types.SimpleNamespace(enable_slicing = lambda: None, use_tiling = False)+ )+ plan = types.SimpleNamespace(vae_tiling = False, vae_slicing = True)+ assert dm.planned_vae_saver_tag(pipe, plan) == "unknown"+ assert dm.live_vae_saver_tag(pipe) == "unknown"+ kw = dict(family = "some-new-dit", quant = "q|vae:unknown|attn:native", speed_mode = "default")+ assert not mp.record(**kw, width = 1024, height = 1024, peak_mib = 2000)+ assert mp.lookup(**kw) is None+++def test_peak_model_identity_names_the_whole_build(monkeypatch):+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_component_sizes as sizes++ revisions = {("org/dit", "dit-Q4.gguf"): "abc", ("org/base", None): "def"}+ monkeypatch.setattr(+ sizes, "checkpoint_fingerprint", lambda repo, filename = None: revisions.get((repo, filename))+ )+ assert diffusion._peak_model_identity("org/dit", "dit-Q4.gguf", "gguf", "org/base") == (+ "org/dit/dit-Q4.gguf@abc+org/base@def"+ )+ assert (+ diffusion._peak_model_identity("org/dit", "dit-Q4.gguf", "gguf", "org/other") == "unknown"+ )+ assert diffusion._peak_model_identity("org/gone", None, "pipeline", None) == "unknown"+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert "peak_model = _peak_model_identity(repo_id, gguf_filename, kind, base)" in src+++def test_phase_table_seed_needs_its_measured_configuration(klein_pipe):+ assert dm.measured_image_runtime_mib("flux.2-klein", "default") is None+ plan = _klein_flat_plan()+ klein_pipe._unsloth_phase_seeds = False # ROCm, or quadratic SDPA only+ assert _refine(klein_pipe, plan) is plan+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert "pipe._unsloth_phase_seeds = _calibrated_activation(fam, target) is not None" in src+++def test_oom_inside_the_first_offload_apply_unwinds_its_hooks(monkeypatch):+ torch = pytest.importorskip("torch")+ import diffusers.hooks as hooks++ def fake_apply(module, **kwargs):+ module._fake_hooked = True # hooked part of the module, then ran out of memory+ raise _OOM("CUDA out of memory")++ unwound = []++ def fake_unwind(+ pipe,+ snapshots,+ logger = None,+ ):+ for module, _ in snapshots:+ module.__dict__.pop("_fake_hooked", None)+ unwound.append(True)+ return True++ monkeypatch.setattr(hooks, "apply_group_offloading", fake_apply)+ monkeypatch.setattr(+ dm, "_offload_groups", lambda m: [object()] if getattr(m, "_fake_hooked", False) else []+ )+ monkeypatch.setattr(dm, "_unwind_group_offload", fake_unwind)+ dit = torch.nn.Linear(4, 4)+ pipe = types.SimpleNamespace(transformer = dit, components = {"transformer": dit})+ assert dm._apply_group_offload(pipe, "cpu", None) is False+ assert unwound and not getattr(dit, "_fake_hooked", False)+ assert dm._is_oom(getattr(pipe, dm._GROUP_OFFLOAD_FAILURE_ATTR))+++def test_video_window_stays_open_until_the_high_water_mark_is_read(store, monkeypatch):+ monkeypatch.setattr(vsr, "_PEAKS", {})+ work = 1280 * 704 * 49+ module = types.SimpleNamespace(+ _unsloth_video_pending = (work, 0, "cuda:0"),+ _unsloth_video_window = ("cuda:0", mp.open_window("cuda:0")),+ _unsloth_video_calibration_key = None,+ )++ class _Torch:+ class cuda: # noqa: N801+ @staticmethod+ def max_memory_allocated(dev):+ mp.open_window("cuda:0") # an image render arms between the decode and the read+ return 3000 << 20++ with monkeypatch.context() as m:+ m.setitem(__import__("sys").modules, "torch", _Torch)+ assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) is None+++# ---------------------------------------------------------------------------------------------------- review round 7+++def _probe_state(monkeypatch, label, **live):+ import core.inference.diffusion as diffusion++ class _Probe:+ def __init__(self, device):+ self.device = "cuda:0"++ monkeypatch.setattr(mp, "PeakProbe", _Probe)+ monkeypatch.setattr(mp, "enabled", lambda: True)+ pipe = types.SimpleNamespace(+ _unsloth_peak_identity = ("x", label, "off"),+ vae = types.SimpleNamespace(use_tiling = False, use_slicing = False),+ )+ state = types.SimpleNamespace(+ pipe = pipe, device = "cuda:0", attention_backend = None, speed_mode = "off", transformer_cache = None+ )+ for name, value in live.items():+ setattr(state, name, value)+ return diffusion, state+++def test_local_pipeline_identity_covers_its_weight_files(tmp_path):+ import core.inference.diffusion as diffusion++ (tmp_path / "transformer").mkdir()+ (tmp_path / "model_index.json").write_text("{}")+ (tmp_path / "transformer" / "config.json").write_text("{}")+ weights = tmp_path / "transformer" / "diffusion_pytorch_model.safetensors"+ weights.write_bytes(b"0" * 64)+ before = diffusion._peak_model_identity(str(tmp_path), None, "pipeline", None)+ weights.write_bytes(b"0" * 128) # a heavier build, same manifests+ after = diffusion._peak_model_identity(str(tmp_path), None, "pipeline", None)+ assert before != "unknown" and after != "unknown" and before != after+++def test_unknown_stream_window_records_no_peak(monkeypatch):+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: -1)+ assert dm.stream_window_pad_mib(types.SimpleNamespace(components = {})) is None+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")+ state.pipe.components = {}+ assert diffusion._start_peak_probe(state) is None+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: 700)+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", dm.stream_window_pad_mib)+ assert diffusion._start_peak_probe(state).window_pad == 700+++def test_bare_unknown_or_empty_format_is_unidentified(store):+ for label in ("unknown", "", None, "unknown|model:x|attn:native"):+ assert not mp.identified(label)+ assert not mp.record(+ family = "some-new-dit",+ quant = label,+ speed_mode = "default",+ width = 1024,+ height = 1024,+ peak_mib = 900,+ )+ assert mp.identified("torchao:Int8Tensor|model:x|attn:native")+++def test_render_with_a_toggled_step_cache_is_not_probed(monkeypatch):+ diffusion, state = _probe_state(monkeypatch, "q|cache:off|vae:full|attn:native")+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)+ assert diffusion._start_peak_probe(state) is not None+ state.transformer_cache = "fbcache" # a high-step request auto-engaged FBCache+ assert diffusion._start_peak_probe(state) is None+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert 'peak_cache = "off" if cache_may_toggle else str(cache_engaged or "off")' in src+++def test_deferred_profile_retags_the_speed_tier(monkeypatch):+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)+ state.speed_mode = "default" # the deferred profile engaged the default tier+ assert (+ diffusion._start_peak_probe(state) is None+ ) # an off-tier key never takes a compiled render+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ deferred = src.index("def _engage_deferred_profile")+ block = src[deferred : deferred + 5000]+ assert (+ 'with_label_part(identity[1], "attn", attention_engaged or "native"),\n SPEED_DEFAULT,'+ in block+ )+++# ---------------------------------------------------------------------------------------------------- review round 8+++def test_failed_video_request_closes_its_measurement_window():+ window = mp.open_window("cuda:0")+ module = types.SimpleNamespace(+ _unsloth_video_pending = (1, 0, "cuda:0"), _unsloth_video_window = ("cuda:0", window)+ )+ vsr.discard_request(types.SimpleNamespace(transformer = module))+ assert module._unsloth_video_pending is None and module._unsloth_video_window is None+ later = mp.open_window("cuda:0") # an image render after the cancelled video+ assert mp.close_window("cuda:0", later)+ src = (__import__("pathlib").Path(vsr.__file__).parent / "video.py").read_text(encoding = "utf-8")+ assert 'video_stream_residency.discard_request(getattr(state, "pipe", None))' in src+++def test_whole_module_offload_records_no_streamed_baseline_peak(monkeypatch):+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: 700)+ monkeypatch.setattr(dm, "_offload_groups", lambda module: [])+ dit = torch.nn.Linear(+ 4, 4+ ) # on the host at render start: whole-module offload onloads it inside the render+ assert dm.stream_window_pad_mib(types.SimpleNamespace(components = {"transformer": dit})) is None+ dit._hf_hook = object()+ assert dm.stream_window_pad_mib(types.SimpleNamespace(components = {"transformer": dit})) is None+++def test_compile_fallback_during_the_render_records_nothing(store, monkeypatch):+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)+ pipe = types.SimpleNamespace(_unsloth_peak_identity = ("some-new-dit", "q", "default"))+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default", speed_optims = ("compiled",))+ state.speed_optims = ("compile_fallback_eager",) # settle_compile_fallback mid-render+ diffusion._record_peak_probe(state, _Probe(2400), width = 1024, height = 1024, batch_size = 1)+ assert mp.lookup(family = "some-new-dit", quant = "q", speed_mode = "default") is None+ state.speed_optims = ("compiled",)+ diffusion._record_peak_probe(state, _Probe(2400), width = 1024, height = 1024, batch_size = 1)+ assert mp.lookup(family = "some-new-dit", quant = "q", speed_mode = "default") == 2400+++def test_failed_capture_video_peak_never_reaches_the_keyed_or_persisted_peaks(store, monkeypatch):+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)+ monkeypatch.setattr(vsr, "_PEAKS", {})+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")+ work = 1280 * 704 * 49+ captures = {"n": 0}+ monkeypatch.setattr(vsr, "_failed_captures", lambda: captures["n"])+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(cg, "failed_capture_count", lambda: captures["n"])++ def render(peak_mib):+ module = types.SimpleNamespace(+ _unsloth_video_pending = (work, 0, "cuda:0"),+ _unsloth_video_calibration_key = key,+ _unsloth_video_failed_captures = captures["n"],+ )++ class _Torch:+ class cuda: # noqa: N801+ @staticmethod+ def max_memory_allocated(dev):+ return peak_mib << 20++ return module, _Torch++ module, torch_stub = render(15000)+ captures["n"] += 1 # the capture failed during this render+ with monkeypatch.context() as m:+ m.setitem(__import__("sys").modules, "torch", torch_stub)+ vsr.record_request_peak(types.SimpleNamespace(transformer = module))+ module, torch_stub = render(3000)+ with monkeypatch.context() as m:+ m.setitem(__import__("sys").modules, "torch", torch_stub)+ vsr.record_request_peak(types.SimpleNamespace(transformer = module))+ assert mp.work_peaks(vsr._store_key(key)) == {work: 3000}+ assert vsr.measured_extra_mib(key, work) == 3000+++# ---------------------------------------------------------------------------------------------------- review round 9+++def test_graphs_left_by_an_unrecorded_render_stop_image_sampling(store, monkeypatch):+ import core.inference.diffusion_cuda_graph as cg++ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)+ monkeypatch.setattr(diffusion, "_peak_device_index", lambda state: 0)+ live = {0: 3, 1: 0}+ monkeypatch.setattr(cg, "live_graph_count", lambda index: live.get(index, 0))+ assert diffusion._start_peak_probe(state) is not None # the load's own graphs are the baseline+ live[1] = 5 # another card's graphs never count+ assert diffusion._start_peak_probe(state) is not None+ live[0] = 5 # a cache-served render recorded graphs on this card and was not sampled+ assert diffusion._start_peak_probe(state) is None+ live[0] = 0 # an OOM reset freed every graph: nothing unaccounted is left+ assert diffusion._start_peak_probe(state) is not None+++def test_graphs_a_recorded_render_captured_keep_sampling_open(store, monkeypatch):+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)+ monkeypatch.setattr(+ cg, "live_graph_count", lambda index: 4+ ) # recorded during this sampled render+ pipe = types.SimpleNamespace(+ _unsloth_peak_identity = ("some-new-dit", "q", "default"), _unsloth_peak_graphs = 0+ )+ diffusion._record_peak_probe(+ types.SimpleNamespace(pipe = pipe, speed_mode = "default"),+ _Probe(2400),+ width = 1024,+ height = 1024,+ batch_size = 1,+ )+ assert pipe._unsloth_peak_graphs == 4+++def test_graphs_left_by_a_discarded_video_request_stop_video_sampling(store, monkeypatch):+ torch = pytest.importorskip("torch")+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(vsr, "_PEAKS", {})+ live = {0: 1}+ monkeypatch.setattr(cg, "live_graph_count", lambda index: live.get(index, 0))+ module = types.SimpleNamespace(_unsloth_video_calibration_key = None)+ pipe = types.SimpleNamespace(transformer = module)+ vsr._open_window(module, torch.device("cuda", 0))+ live[0] += 1 # the request recorded a graph, then was cancelled+ vsr.discard_request(pipe)+ vsr._open_window(module, torch.device("cuda", 0))+ module._unsloth_video_pending = (1280 * 704 * 49, 0, torch.device("cuda", 0))++ class _Torch:+ class cuda: # noqa: N801+ @staticmethod+ def max_memory_allocated(dev):+ return 3000 << 20++ with monkeypatch.context() as m:+ m.setitem(__import__("sys").modules, "torch", _Torch)+ assert vsr.record_request_peak(pipe) is None+++def test_live_graph_count_is_per_device_and_drops_reset_graphs():+ import core.inference.diffusion_block_graph as bg+ import core.inference.diffusion_cuda_graph as cg++ class _Wrapper:+ def __init__(self, device, n):+ self._unsloth_capture_device = device+ self.cache = dict.fromkeys(range(n))++ a, b = _Wrapper(0, 2), _Wrapper(1, 3)+ blocks = _Wrapper(0, 4)+ cg._LIVE_WRAPPERS.add(a)+ cg._LIVE_WRAPPERS.add(b)+ bg._LIVE_BLOCKS.add(blocks)+ try:+ assert cg.live_graph_count(0) == 6 and cg.live_graph_count(1) == 3+ a.cache.clear() # reset_all+ assert cg.live_graph_count(0) == 4+ finally:+ cg._LIVE_WRAPPERS.discard(a)+ cg._LIVE_WRAPPERS.discard(b)+ bg._LIVE_BLOCKS.discard(blocks)+++def test_render_without_the_familys_cfg_is_not_probed(monkeypatch):+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)+ state.family = types.SimpleNamespace(uses_negative_prompt = True)+ state.pipe._unsloth_peak_cfg = True+ assert diffusion._start_peak_probe(state) is not None+ state.pipe._unsloth_peak_cfg = False # guidance off: half the denoiser batch+ assert diffusion._start_peak_probe(state) is None+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert "state.pipe._unsloth_peak_cfg = _negative_prompt_engaged(state.family, guidance)" in src+++def test_render_forced_eager_while_the_compile_is_pending_is_not_probed(monkeypatch):+ import core.inference.diffusion_bg_compile as bg++ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)+ assert diffusion._start_peak_probe(state) is not None+ token = bg._FORCE_EAGER.set(True)+ try:+ assert diffusion._start_peak_probe(state) is None+ finally:+ bg._FORCE_EAGER.reset(token)+++# ---------------------------------------------------------------------------------------------------- review round 10+++def test_render_with_lora_adapters_is_not_probed(monkeypatch):+ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)+ assert diffusion._start_peak_probe(state) is not None+ state.pipe._unsloth_loras = (("style", "/loras/style.safetensors", 0.8),)+ assert diffusion._start_peak_probe(state) is None+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert '"unknown" # adapters baked in at load' in src+++def test_learned_reserve_falls_back_to_the_flat_estimate_with_loras_or_a_failed_saver(klein_pipe):+ klein_pipe._unsloth_phase_seeds = (+ False # unseeded: the reserve came from the learned peak alone+ )+ klein_pipe._unsloth_peak_identity = ("some-new-dit", "torchao:Int8Tensor", "default")+ klein_pipe._unsloth_measured_reserve = (+ 3072,+ "some-new-dit",+ "default",+ None,+ None,+ 2600,+ 1 << 20,+ False,+ )+ flat = dm.estimate_image_runtime_mib(+ width = 1024, height = 1024, batch_size = 1, family = "some-new-dit"+ )+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0+ klein_pipe._unsloth_loras = (("style", "/loras/style.safetensors", 0.8),)+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == flat - 3072+ klein_pipe._unsloth_loras = ()+ klein_pipe._unsloth_peak_static = True # a planned VAE saver failed to engage+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == flat - 3072+++def test_store_written_under_other_runtimes_is_ignored(store, monkeypatch):+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default")+ assert mp.record(**kw, width = 1024, height = 1024, peak_mib = 2000)+ assert mp.lookup(**kw) == 2000+ monkeypatch.setattr(mp, "_runtime", ["torch=9.9|diffusers=9.9|cuda=99"]) # an upgraded runtime+ mp._cache.update(path = None, mtime = None, data = None)+ assert mp.lookup(**kw) is None+++def test_quadratic_peaks_scale_linearly_across_the_batch(store, klein_pipe):+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default")+ assert not mp.record(**kw, width = 1024, height = 1024, batch_size = 2, peak_mib = 3000, quadratic = True)+ assert mp.record(**kw, width = 1024, height = 1024, peak_mib = 1000, quadratic = True)+ assert mp.lookup(**kw, width = 1024, height = 1024, batch_size = 2) == 2000 # not 4000+ assert mp.lookup(**kw, width = 1448, height = 1448) >= 4000 # pixels still square+ klein_pipe._unsloth_peak_identity = ("flux.2-klein", "torchao:Int8Tensor", "default")+ klein_pipe._unsloth_measured_reserve = (+ 4608,+ "flux.2-klein",+ "default",+ None,+ None,+ 4000,+ 1 << 20,+ True,+ )+ # batch 2 at 1 MP: 4000 x 2 x 1.15 = 9200 -> 9216, not 4000 x 4+ assert (+ dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024, batch_size = 2)+ == 9216 - 4608+ )+++# ---------------------------------------------------------------------------------------------------- review round 11+++def test_resident_text_encoders_add_their_streaming_window(monkeypatch):+ torch = pytest.importorskip("torch")+ import core.inference.diffusion_offload_prefetch as prefetch++ monkeypatch.setattr(prefetch, "prefetch_depth", lambda: 1)+ dit, encoder = torch.nn.Linear(4, 4), torch.nn.Linear(1024, 1024, bias = False) # 4 MiB fp32 leaf+ streamed = types.SimpleNamespace(_unsloth_resident = False)+ resident = types.SimpleNamespace(_unsloth_resident = True)+ groups = {id(dit): [streamed], id(encoder): [resident]} # streamed denoiser, resident encoder+ monkeypatch.setattr(dm, "_offload_groups", lambda module: groups.get(id(module), []))+ pipe = types.SimpleNamespace(components = {"transformer": dit, "text_encoder": encoder})+ assert dm.stream_window_pad_mib(pipe) == 8 # (depth + 1) x the largest encoder leaf+ groups[id(encoder)] = [streamed] # streamed during the render: its window is in the peak+ assert dm.stream_window_pad_mib(pipe) == 0+ groups[id(encoder)] = [] # no hooks, weights on the host: placement unknown, record nothing+ assert dm.stream_window_pad_mib(pipe) is None+++def test_video_peaks_are_persisted_on_the_streamed_baseline(store, monkeypatch):+ monkeypatch.setattr(vsr, "_PEAKS", {})+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")+ work = 1280 * 704 * 49++ class _Torch:+ class cuda: # noqa: N801+ @staticmethod+ def max_memory_allocated(dev):+ return 3000 << 20++ def render(pad):+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: pad)+ module = types.SimpleNamespace(+ _unsloth_video_pending = (work, 0, "cuda:0"), _unsloth_video_calibration_key = key+ )+ with monkeypatch.context() as m:+ m.setitem(__import__("sys").modules, "torch", _Torch)+ vsr.record_request_peak(types.SimpleNamespace(transformer = module))++ render(None) # placement unknown: nothing persisted+ assert mp.work_peaks(vsr._store_key(key)) == {}+ render(700) # every group resident: the prefetch window a streamed load holds is added+ assert mp.work_peaks(vsr._store_key(key)) == {work: 3700}+ assert vsr.measured_extra_mib(key, work) == 3700+++# --------------------------------------------------------------------------------------------------- review round 12+++def test_a_graph_replaced_in_a_full_cache_stops_image_sampling(store, monkeypatch):+ import core.inference.diffusion_cuda_graph as cg++ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)+ monkeypatch.setattr(diffusion, "_peak_device_index", lambda state: 0)+ monkeypatch.setattr(cg, "live_graph_count", lambda index: 4)+ gen = {0: 7, 1: 0}+ monkeypatch.setattr(cg, "capture_generation", lambda index: gen.get(index, 0))+ assert diffusion._start_peak_probe(state) is not None # the load's graphs are the baseline+ gen[1] = 3 # another card's captures never count+ assert diffusion._start_peak_probe(state) is not None+ gen[0] = 8 # a full cache evicted one graph and captured its replacement: same count, new graph+ assert diffusion._start_peak_probe(state) is None+++def test_capture_generation_counts_per_device(monkeypatch):+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(cg, "_CAPTURES", {})+ cg.note_capture(0)+ cg.note_capture(0)+ cg.note_capture(1)+ assert (+ cg.capture_generation(0) == 2+ and cg.capture_generation(1) == 1+ and cg.capture_generation(2) == 0+ )+++def test_runtime_stamp_names_installed_kernel_packages(monkeypatch):+ import importlib.metadata as md++ real = md.version++ def fake(dist):+ if dist == "sageattention":+ return "2.2.0"+ if dist in mp._KERNEL_DISTS:+ raise md.PackageNotFoundError(dist)+ return real(dist)++ monkeypatch.setattr(md, "version", fake)+ monkeypatch.setattr(mp, "_runtime", [])+ stamp = mp.runtime_stamp()+ assert "sageattention=2.2.0" in stamp and "xformers" not in stamp+ monkeypatch.setattr(mp, "_runtime", [])+ monkeypatch.setattr(md, "version", lambda d: "2.3.0" if d == "sageattention" else fake(d))+ assert mp.runtime_stamp() != stamp # a kernel upgrade drops the stored peaks+++def test_attention_scaling_unknown_render_is_not_recorded(store, monkeypatch):+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)+ monkeypatch.setattr(+ diffusion, "_live_peak_identity", lambda state: ("some-new-dit", "q", "default")+ )+ monkeypatch.setattr(diffusion, "_conditioning_cache_hits", lambda pipe: 0)+ answers = {"math": False, "fused": False}+ monkeypatch.setattr(diffusion, "sdpa_math_only", lambda target: answers["math"])+ monkeypatch.setattr(diffusion, "sdpa_subquadratic_confirmed", lambda target: answers["fused"])+ target = types.SimpleNamespace(device = "cuda:0")+ assert not diffusion._attention_scaling_known(target, "native") # the probe answered nothing+ assert not diffusion._attention_scaling_known(None, None)+ assert diffusion._attention_scaling_known(+ None, "sage"+ ) # an engaged fused backend is evidence on its own+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default", device = "NVIDIA L4")++ def render():+ pipe = types.SimpleNamespace(_unsloth_peak_identity = ("some-new-dit", "q", "default"))+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default", attention_backend = "native")+ diffusion._record_peak_probe(+ state, _Probe(2400), width = 1024, height = 1024, batch_size = 1, target = target+ )+ return pipe++ pipe = render()+ assert mp.lookup(**kw, width = 1024, height = 1024) is None+ assert (+ getattr(pipe, "_unsloth_peak_graphs", None) is None+ ) # an unrecorded render moves no graph baseline+ answers["fused"] = True+ render()+ assert mp.lookup(**kw, width = 1024, height = 1024) is not None+++def test_partially_resident_denoiser_pads_the_window_difference(monkeypatch):+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: 900)+ monkeypatch.setattr(dm, "_encoder_stream_window_mib", lambda pipe: 0)+ pipe = types.SimpleNamespace(components = {"transformer": object()})+ groups = [+ types.SimpleNamespace(_unsloth_resident = True),+ types.SimpleNamespace(_unsloth_resident = False),+ ]+ monkeypatch.setattr(dm, "_offload_groups", lambda module: groups)+ monkeypatch.setattr(dm, "_streamed_group_window_mib", lambda pipe: 300)+ assert (+ dm.stream_window_pad_mib(pipe) == 600+ ) # a fully streamed load holds the largest group, not the streamed one+ monkeypatch.setattr(dm, "_streamed_group_window_mib", lambda pipe: 900)+ assert dm.stream_window_pad_mib(pipe) == 0+ monkeypatch.setattr(dm, "_streamed_group_window_mib", lambda pipe: -1)+ assert dm.stream_window_pad_mib(pipe) is None # unsized: not recorded+ groups[0]._unsloth_resident = False # fully streamed: already the baseline+ assert dm.stream_window_pad_mib(pipe) == 0+++def test_streamed_group_window_counts_only_streamed_groups(monkeypatch):+ torch = pytest.importorskip("torch")+ import core.inference.diffusion_offload_prefetch as pf++ monkeypatch.setattr(pf, "prefetch_depth", lambda: 1)+ big, small = torch.nn.Linear(1024, 1024, bias = False), torch.nn.Linear(512, 512, bias = False)+ groups = [+ types.SimpleNamespace(_unsloth_resident = True, modules = [big], parameters = None, buffers = None),+ types.SimpleNamespace(+ _unsloth_resident = False, modules = [small], parameters = None, buffers = None+ ),+ ]+ monkeypatch.setattr(dm, "_offload_groups", lambda module: groups)+ pipe = types.SimpleNamespace(components = {"transformer": torch.nn.Module()})+ assert dm._streamed_group_window_mib(pipe) == 2 # 1 MiB (512 x 512 fp32) x (depth 1 + 1)+++def test_a_graph_replaced_in_a_full_cache_stops_video_sampling(monkeypatch):+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(vsr, "_live_graphs", lambda dev: 4)+ gen = [7]+ monkeypatch.setattr(vsr, "_capture_generation", lambda dev: gen[0])+ monkeypatch.setattr("core.inference.diffusion_measured_peaks.open_window", lambda dev: 1)+ monkeypatch.setattr(+ "core.inference.diffusion_measured_peaks.close_window", lambda dev, token: True+ )+ module = types.SimpleNamespace()+ dev = torch.device("cuda", 0)+ vsr._open_window(module, dev)+ assert module._unsloth_video_graphs_clean # the load's graphs are the baseline+ vsr._open_window(module, dev)+ assert module._unsloth_video_graphs_clean+ gen[0] = (+ 8 # a discarded request evicted one graph and captured its replacement: same count, new graph+ )+ vsr._open_window(module, dev)+ assert not module._unsloth_video_graphs_clean+++# --------------------------------------------------------------------------------------------------- review round 14+++def test_unseeded_reserve_scales_without_the_store(store, klein_pipe):+ klein_pipe._unsloth_phase_seeds = False+ klein_pipe._unsloth_peak_identity = ("some-new-dit", "torchao:Int8Tensor", "default")+ klein_pipe._unsloth_measured_reserve = (+ 3072,+ "some-new-dit",+ "default",+ None,+ None,+ 2600,+ 1 << 20,+ False,+ )+ assert (+ mp.lookup(family = "some-new-dit", quant = "torchao:Int8Tensor", speed_mode = "default") is None+ )+ # the store holds nothing (evicted, deleted, unwritable): a larger request still scales the reserved peak+ assert dm.measured_request_extra_mib(klein_pipe, width = 2048, height = 2048) > 0+ assert dm.measured_request_extra_mib(klein_pipe, width = 1024, height = 1024) == 0+++def test_another_pipelines_reset_keeps_this_pipes_capture_dirty(monkeypatch):+ import core.inference.diffusion_cuda_graph as cg++ diffusion, state = _probe_state(monkeypatch, "q|vae:full|attn:native")+ monkeypatch.setattr(diffusion, "stream_window_pad_mib", lambda pipe: 0)+ monkeypatch.setattr(diffusion, "_peak_device_index", lambda state: 0)+ live, gen = [6], [10]+ monkeypatch.setattr(cg, "live_graph_count", lambda index: live[0])+ monkeypatch.setattr(cg, "capture_generation", lambda index: gen[0])+ assert diffusion._start_peak_probe(state) is not None # baseline: 6 graphs, generation 10+ live[0], gen[0] = 7, 11 # this pipe captured one graph in a render that was not recorded+ assert diffusion._start_peak_probe(state) is None+ live[0] = (+ 3 # a video pipeline on the same card reset its 4 graphs; this pipe's capture is still alive+ )+ assert diffusion._start_peak_probe(state) is None+ live[0] = 0 # nothing alive on the card: nothing unaccounted can remain+ assert diffusion._start_peak_probe(state) is not None+++# --------------------------------------------------------------------------------------------------- review round 15+++def test_record_reports_whether_the_store_holds_the_sample(store):+ kw = dict(family = "some-new-dit", quant = "q", speed_mode = "default", width = 1024, height = 1024)+ held: list = []+ assert mp.record(**kw, peak_mib = 2000, held = held) and held == [True]+ held = []+ assert not mp.record(**kw, peak_mib = 1500, held = held) and held == [True] # already covered+ held = []+ assert not mp.record_work(("k",), 10, 0, held = held) and held == []+ assert mp.record_work(("k",), 10, 100, held = held) and held == [True]+++def test_unpersisted_image_sample_keeps_its_graphs_dirty(store, monkeypatch):+ import core.inference.diffusion as diffusion+ import core.inference.diffusion_cuda_graph as cg++ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA L4")+ monkeypatch.setattr(cg, "failed_capture_count", lambda: 0)+ monkeypatch.setattr(cg, "live_graph_count", lambda index: 4)+ monkeypatch.setattr(cg, "capture_generation", lambda index: 9)+ pipe = types.SimpleNamespace(+ _unsloth_peak_identity = ("some-new-dit", "q", "default"), _unsloth_peak_graphs = 0+ )+ state = types.SimpleNamespace(pipe = pipe, speed_mode = "default")+ monkeypatch.setattr(mp, "record", lambda **kw: False) # the store is unavailable+ diffusion._record_peak_probe(state, _Probe(2400), width = 1024, height = 1024, batch_size = 1)+ assert pipe._unsloth_peak_graphs == 0+ monkeypatch.setattr(mp, "record", lambda **kw: kw["held"].append(True) or True)+ diffusion._record_peak_probe(state, _Probe(2400), width = 1024, height = 1024, batch_size = 1)+ assert pipe._unsloth_peak_graphs == 4 and pipe._unsloth_peak_graph_gen == 9+++def test_unknown_local_video_build_persists_nothing():+ known = vsr.calibration_key(+ "wan", "/models/wan", "group", "cuda:0", revision = "/models/wan@local:ab:cd"+ )+ unknown = vsr.calibration_key("wan", "/models/wan", "group", "cuda:0", revision = "unknown")+ unread = vsr.calibration_key("wan", "/models/wan", "group", "cuda:0", revision = None)+ assert vsr._store_key(known) is not None+ assert vsr._store_key(unknown) is None and vsr._store_key(unread) is None+ src = (__import__("pathlib").Path(dm.__file__).parent / "video.py").read_text(encoding = "utf-8")+ assert "revision = _peak_model_identity(" in src+++def test_local_weight_shards_change_the_video_revision(tmp_path, monkeypatch):+ import core.inference.diffusion as diffusion++ (tmp_path / "model_index.json").write_text("{}")+ (tmp_path / "transformer").mkdir()+ (tmp_path / "transformer" / "config.json").write_text("{}")+ shard = tmp_path / "transformer" / "diffusion_pytorch_model.safetensors"+ shard.write_bytes(b"a" * 8)+ before = diffusion._peak_model_identity(str(tmp_path), None, "pipeline", None)+ shard.write_bytes(b"b" * 16) # same manifests, new weights+ after = diffusion._peak_model_identity(str(tmp_path), None, "pipeline", None)+ assert before != "unknown" and before != after+++def test_learned_peak_label_carries_the_graph_mode(klein_pipe, monkeypatch):+ import core.inference.diffusion as diffusion++ label = dm.learned_peak_label(klein_pipe, "native", "m", "none", "dense", "off", "on")+ assert label.endswith("|graphs:on")+ assert dm.learned_peak_label(klein_pipe, "native", "m", "none", "dense", "off", "off") != label+ state = types.SimpleNamespace(resolved = None, speed_optims = ("compiled",), cuda_graphs = ())+ assert diffusion._live_graph_mode(state) == "off"+ state.speed_optims = ("compiled", "cuda_graph")+ assert diffusion._live_graph_mode(state) == "step"+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert 'label = with_label_part(label, "graphs", _live_graph_mode(state))' in src+ assert "graphs = peak_graphs," in src+++# --------------------------------------------------------------------------------------------------- review round 16+++def test_whole_step_and_block_graphs_record_under_distinct_keys(store):+ import core.inference.diffusion as diffusion++ state = types.SimpleNamespace(+ resolved = None,+ speed_optims = ("compiled", "cuda_graph"),+ cuda_graphs = (),+ pipe = types.SimpleNamespace(_unsloth_cuda_graph_mode = "step"),+ )+ assert diffusion._live_graph_mode(state) == "step"+ state.pipe._unsloth_cuda_graph_mode = "blocks"+ assert diffusion._live_graph_mode(state) == "blocks"+ assert diffusion._engaged_graph_mode(state.pipe, {"cuda_graph": False}) == "off"+ # a load places before it knows the mode: it reads the larger of both modes' peaks+ base = dict(family = "flux.2-klein", speed_mode = "default", width = 1024, height = 1024)+ assert mp.record(**base, quant = "torchao:Int8Tensor|graphs:step", peak_mib = 3000)+ assert mp.lookup(**base, quant = "torchao:Int8Tensor|graphs:on") == 3000+ assert mp.record(**base, quant = "torchao:Int8Tensor|graphs:blocks", peak_mib = 4200)+ assert mp.lookup(**base, quant = "torchao:Int8Tensor|graphs:on") == 4200+ assert mp.lookup(**base, quant = "torchao:Int8Tensor|graphs:step") == 3000+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert "_engaged_graph_mode(pipe, speed_applied)" in src+++def test_torchao_variants_of_one_weight_class_do_not_share_a_label(monkeypatch):+ torch = pytest.importorskip("torch")+ import core.inference.diffusion_compile_cache as cc++ monkeypatch.setattr(dm, "_torchao_weight_classes", lambda module: {"Int8Tensor"})+ pipe = types.SimpleNamespace(transformer = torch.nn.Linear(4, 4))+ variant = {"weights": {"Linear/Int8Tensor": 1}}+ monkeypatch.setattr(cc, "graph_variant", lambda dit: variant)+ plain = dm.denoiser_quant_label(pipe)+ assert plain.startswith("torchao:Int8Tensor#")+ variant = {"weights": {"Linear/Int8Tensor/convrot256": 1}}+ assert dm.denoiser_quant_label(pipe) != plain # ConvRot: same class, other kernels+ variant = {"weights": {"Linear/Int8Tensor/act_int8": 1}}+ assert dm.denoiser_quant_label(pipe) != plain # W8A8 vs weight-only+ variant = {"weights": "unavailable: RuntimeError"}+ assert not mp.identified(dm.denoiser_quant_label(pipe)) # unreadable: neither read nor recorded+++def test_partially_resident_text_encoder_pads_the_window_difference(monkeypatch):+ torch = pytest.importorskip("torch")+ import core.inference.diffusion_offload_prefetch as pf++ monkeypatch.setattr(pf, "prefetch_depth", lambda: 1)+ big, small = torch.nn.Linear(1024, 1024, bias = False), torch.nn.Linear(512, 512, bias = False)+ te = torch.nn.Sequential(big, small)+ groups = [+ types.SimpleNamespace(_unsloth_resident = True, modules = [big], parameters = None, buffers = None),+ types.SimpleNamespace(+ _unsloth_resident = False, modules = [small], parameters = None, buffers = None+ ),+ ]+ monkeypatch.setattr(dm, "_offload_groups", lambda module: groups)+ pipe = types.SimpleNamespace(components = {"text_encoder": te})+ # largest leaf 4 MiB x 2 = 8 MiB when fully streamed; the render streamed only the 1 MiB leaf (2 MiB in flight)+ assert dm._encoder_stream_window_mib(pipe) == 6+ groups[0]._unsloth_resident = False # fully streamed: already the baseline+ assert dm._encoder_stream_window_mib(pipe) == 0+++# --------------------------------------------------------------------------------------------------- review round 17+++def test_graph_mode_envelope_covers_either_mode(store):+ base = dict(family = "flux.2-klein", speed_mode = "default")+ # a 3000 MiB step peak covering 4 MP, a 2900 MiB block peak covering 1 MP+ assert mp.record(**base, quant = "q|graphs:step", width = 2048, height = 2048, peak_mib = 3000)+ assert mp.record(**base, quant = "q|graphs:blocks", width = 1024, height = 1024, peak_mib = 2900)+ blocks = mp.lookup_bucket(**base, quant = "q|graphs:blocks", width = 1024, height = 1024)+ peak, pixels, quadratic = mp.lookup_bucket(**base, quant = "q|graphs:on", width = 1024, height = 1024)+ assert peak == 3000 and pixels == blocks[1] # the larger peak over the smaller covered bucket+ assert not quadratic+ # a later 2 MP request scaled from the envelope covers what the block entry alone asks for+ want = mp.resolution_bucket(1448, 1448, 1)+ assert peak * want / pixels >= mp.lookup(+ **base, quant = "q|graphs:blocks", width = 1448, height = 1448+ )+++def test_in_process_video_peak_is_on_the_streamed_baseline(monkeypatch):+ monkeypatch.setenv(mp.LEARNED_PEAKS_ENV, "0") # persistence off: the keyed table alone answers+ monkeypatch.setattr(vsr, "_PEAKS", {})+ key = vsr.calibration_key("wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_mode = "default")+ work = 1280 * 704 * 49++ class _Torch:+ class cuda: # noqa: N801+ @staticmethod+ def max_memory_allocated(dev):+ return (3000 + 100) << 20++ def run(pad):+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: pad)+ module = types.SimpleNamespace(_unsloth_video_pending = (work, 100 << 20, "cuda:0"))+ module._unsloth_video_calibration_key = key+ with monkeypatch.context() as m:+ m.setitem(__import__("sys").modules, "torch", _Torch)+ assert vsr.record_request_peak(types.SimpleNamespace(transformer = module)) == 3000+ return module++ run(None) # unknown placement: nothing enters the keyed table+ assert vsr.measured_extra_mib(key, work) is None+ module = run(600) # groups kept resident: the reused peak includes the prefetch window+ assert vsr.measured_extra_mib(key, work) == 3600+ assert module._unsloth_video_peaks[work] == 3000 # the module's own diagnostic stays raw+++# --------------------------------------------------------------------------------------------------- review round 18+++def test_capture_generation_increments_are_serialized(monkeypatch):+ import core.inference.diffusion_cuda_graph as dcg++ seen: list = []++ class _Counts(dict):+ def get(self, *a):+ lock = getattr(dcg, "_CAPTURES_LOCK", None)+ seen.append(lock is not None and lock.locked())+ return super().get(*a)++ monkeypatch.setattr(dcg, "_CAPTURES", _Counts())+ dcg.note_capture(0)+ dcg.note_capture(0)+ assert dcg.capture_generation(0) == 2+ assert seen and all(seen[:2]) # each read-modify-write held the lock+++def test_poisoned_graph_load_records_no_video_calibration(monkeypatch):+ import core.inference.diffusion_cuda_graph as dcg++ monkeypatch.setenv(mp.LEARNED_PEAKS_ENV, "0")+ monkeypatch.setattr(vsr, "_PEAKS", {})+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)+ key = vsr.calibration_key(+ "wan2.2-ti2v-5b", "Wan-AI/x", "group", "cuda:0", speed_optims = ("compiled", "cuda_graph")+ )+ work = 1280 * 704 * 49+ handle = types.SimpleNamespace(poisoned = False)+ monkeypatch.setattr(+ dcg,+ "never_engaged",+ lambda handles: "capture failed" if all(h.poisoned for h in handles) else None,+ )++ class _Torch:+ class cuda: # noqa: N801+ @staticmethod+ def max_memory_allocated(dev):+ return (3000 + 100) << 20++ def run():+ module = types.SimpleNamespace(_unsloth_video_pending = (work, 100 << 20, "cuda:0"))+ module._unsloth_video_calibration_key = key+ pipe = types.SimpleNamespace(transformer = module, _unsloth_cuda_graphs = (handle,))+ with monkeypatch.context() as m:+ m.setitem(__import__("sys").modules, "torch", _Torch)+ vsr.record_request_peak(pipe)++ handle.poisoned = True # a failed capture poisoned the wrapper: later requests run eager+ run()+ assert vsr.measured_extra_mib(key, work) is None+ handle.poisoned = False # graphs live: the request records+ run()+ assert vsr.measured_extra_mib(key, work) == 3000+++# --------------------------------------------------------------------------------------------------- review round 19+++def test_partially_quantized_text_encoders_neither_read_nor_record(klein_pipe):+ import core.inference.diffusion as diffusion+ from core.inference.diffusion_precision import TEQuantOutcome++ full = TEQuantOutcome("int8", None)+ partial = full._replace(partial = True)+ assert diffusion._peak_te_label("int8", full) == "int8"+ assert diffusion._peak_te_label(None, None) == "dense"+ te = diffusion._peak_te_label("int8", partial)+ label = dm.learned_peak_label(klein_pipe, "native", "m", "none", te, "off", "off")+ assert not mp.identified(label) # a mixed build never reuses the fully quantised build's peak+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert "peak_te = _peak_te_label(te_quant, te_outcome)" in srcdiff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.pyindex f8a0271beb5..1b81a742128 100644--- a/studio/backend/tests/test_diffusion_measured_placement.py+++ b/studio/backend/tests/test_diffusion_measured_placement.py@@ -109,11 +109,9 @@ def test_24gb_keeps_transformer_and_text_encoder_resident(q21_pipe):assert new.offload_policy == dm.OFFLOAD_GROUPassert new.stream_text_encoders and not new.stream_transformerassert new.resident_transformer_mib is None- assert (- new.resident_text_encoder_mib- == 21432 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644 - 6922- >= 8959- )+ # the room (21432 - 2304 - overhead - 644 - 6922) covers the whole encoder; the plan records the encoder's size+ assert 21432 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644 - 6922 >= 8959+ assert new.resident_text_encoder_mib == 8959assert new.estimates["measured_runtime_headroom_mib"] == 2304# never under-reserved: loaded weights + measured peak x margin + base overhead within the safe budgetassert 6922 + 8959 + 644 + 2304 + dm.DEFAULT_BASE_OVERHEAD_MIB <= 21432@@ -402,7 +400,9 @@ def test_only_measured_family_moves(q21_pipe, speed, budget, total):plan = _flat_plan(budget, total)for family in names:new = _refine(q21_pipe, plan, family = family, speed = speed)- if family in ("qwen-image-2.1", "flux.1", "z-image") and speed in ("default", "max"):+ # seeded: the peak table, or the per-phase calibration for a family the peak table lacks+ seeded = set(dm._MEASURED_IMAGE_PEAK_MIB) | set(dm._MEASURED_IMAGE_ACTIVATION_MIB)+ if family in seeded and speed in ("default", "max"):continueassert new is plan, family