Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: price the compiled tier's requests from its own learned peak (offloaded bf16 keeps its residency after compile) (#15198)

Daniel Hanc5922a0d2114
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 5c57d89add7..62e6045777d 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -229,6 +229,8 @@ from .diffusion_memory import (
learned_peak_label,
CFG_OFF_PART,
cfg_off_peaks_enabled,
+ TIER_MOVED_ATTR,
+ tier_peaks_enabled,
without_cfg_off,
live_vae_saver_tag,
planned_vae_saver_tag,
@@ -3042,6 +3044,7 @@ def _attention_scaling_known(target: Any, engaged_backend: Optional[str] = None)
def _start_peak_probe(state: Any) -> Any:
"""Arm the learned-peak measurement for one render (CUDA loads that carry a peak identity), else None."""
pipe = getattr(state, "pipe", None)
+ _follow_live_graph_mode(state)
identity = getattr(pipe, "_unsloth_peak_identity", None)
if identity is None:
return None
@@ -3115,6 +3118,142 @@ def _graphs_settled_eager(state: Any) -> bool:
return True
+def _learned_peak_keys(
+ state: Any,
+ label: str,
+ speed: Optional[str],
+ *,
+ eager_stand_in: bool = True,
+) -> list:
+ """Learned-peak keys, most specific first. ``eager_stand_in=False``: live key only (a dropped graph can peak above
+ the capturing render)."""
+ pipe = getattr(state, "pipe", None)
+ cfg_off = CFG_OFF_PART in str(label).split("|")
+ keys = [(label, speed)]
+ if not eager_stand_in:
+ return keys if speed is not None else []
+ # a deferred compile tier records nothing: its eager peak (not exceeded when compiled) stands in
+ keys.append((label, SPEED_OFF))
+ loaded = getattr(pipe, "_unsloth_peak_identity", None)
+ if isinstance(loaded, tuple) and len(loaded) == 3:
+ loaded_label = str(loaded[1])
+ if cfg_off:
+ loaded_label = f"{without_cfg_off(loaded_label)}|{CFG_OFF_PART}"
+ keys += [(loaded_label, tier) for tier in (loaded[2], SPEED_OFF) if tier is not None]
+ tier_info = getattr(pipe, WHOLE_TIER_ATTR, None)
+ eager_label = tier_info.get("eager_peak_label") if isinstance(tier_info, dict) else None
+ if eager_label:
+ if cfg_off:
+ eager_label = f"{eager_label}|{CFG_OFF_PART}"
+ keys.append((eager_label, SPEED_OFF))
+ return list(dict.fromkeys(keys))
+
+
+def _learned_peak_for_keys(
+ family: Any,
+ keys: Sequence[tuple],
+ *,
+ width: Optional[int],
+ height: Optional[int],
+ batch_size: int,
+ workflow: Optional[str] = None,
+ phase: Optional[str] = None,
+) -> Optional[int]:
+ for key_label, tier in keys:
+ found = measured_image_runtime_mib(
+ family,
+ tier,
+ width = width,
+ height = height,
+ batch_size = batch_size,
+ quant = with_workflow(key_label, workflow),
+ static_seed = False,
+ phase = phase,
+ )
+ if found is not None:
+ return found
+ return None
+
+
+def _follow_live_graph_mode(state: Any) -> None:
+ """Keep a deferred profile's identity on the CUDA graph mode actually running (armed, then possibly dropped)."""
+ pipe = getattr(state, "pipe", None)
+ identity = getattr(pipe, "_unsloth_peak_identity", None)
+ if not getattr(pipe, TIER_MOVED_ATTR, False) or not isinstance(identity, tuple):
+ return
+ if len(identity) != 3 or "|graphs:" not in str(identity[1]):
+ return
+ try:
+ live = with_label_part(str(identity[1]), "graphs", _tier_graph_mode(state))
+ if live != identity[1]:
+ pipe._unsloth_peak_identity = (identity[0], live, identity[2])
+ except Exception: # noqa: BLE001 - recording is optional
+ pass
+
+
+def _tier_request_peak(
+ state: Any,
+ *,
+ width: Optional[int],
+ height: Optional[int],
+ batch_size: int,
+ cfg: Optional[bool],
+ workflow: Optional[str] = None,
+) -> Optional[int]:
+ """Learned peak of a deferred profile's engaged tier; None until that tier has its own measurement."""
+ pipe = getattr(state, "pipe", None)
+ if not getattr(pipe, TIER_MOVED_ATTR, False):
+ return None
+ try:
+ family, label, speed = _live_peak_identity(state)
+ except Exception: # noqa: BLE001
+ return None
+ if speed is None:
+ return None
+ label = without_cfg_off(str(label))
+ if (
+ cfg is False
+ and getattr(pipe, "_unsloth_peak_expects_cfg", False)
+ and cfg_off_peaks_enabled()
+ ):
+ label = f"{label}|{CFG_OFF_PART}"
+ return _learned_peak_for_keys(
+ family,
+ _learned_peak_keys(state, label, speed, eager_stand_in = False),
+ width = width,
+ height = height,
+ batch_size = batch_size,
+ workflow = workflow,
+ )
+
+
+def _hold_captured_release(state: Any, extra_mib: int) -> int:
+ """At least the release a live CUDA graph was captured over: graphs key on weight addresses, so re-pinning
+ groups would re-capture."""
+ pipe = getattr(state, "pipe", None)
+ held = getattr(pipe, "_unsloth_captured_release_mib", None)
+ if held is None:
+ return extra_mib
+ try:
+ if not cuda_graph.live_graph_count(_peak_device_index(state)):
+ pipe._unsloth_captured_release_mib = None
+ return extra_mib
+ except Exception: # noqa: BLE001 - unknown: keep the placement the graph was captured over
+ pass
+ return max(int(extra_mib), int(held))
+
+
+def _note_captured_release(state: Any, extra_mib: int, generation_before: Optional[int]) -> None:
+ pipe = getattr(state, "pipe", None)
+ if generation_before is None or not getattr(pipe, TIER_MOVED_ATTR, False):
+ return
+ try:
+ if cuda_graph.capture_generation(_peak_device_index(state)) > generation_before:
+ pipe._unsloth_captured_release_mib = int(max(0, extra_mib))
+ except Exception: # noqa: BLE001
+ pass
+
+
def _whole_tier_for_request(
state: Any,
*,
@@ -3164,33 +3303,15 @@ def _whole_tier_for_request(
return None
if speed is None:
return None
- # a deferred compile tier records nothing: its eager peak (not exceeded when compiled) stands in
- keys = [(label, speed), (label, SPEED_OFF)]
- loaded = getattr(pipe, "_unsloth_peak_identity", None)
- if isinstance(loaded, tuple) and len(loaded) == 3:
- loaded_label = str(loaded[1])
- if CFG_OFF_PART in str(label).split("|"):
- loaded_label = f"{without_cfg_off(loaded_label)}|{CFG_OFF_PART}"
- keys += [(loaded_label, tier) for tier in (loaded[2], SPEED_OFF) if tier is not None]
- eager_label = tier_info.get("eager_peak_label") if isinstance(tier_info, dict) else None
- if eager_label:
- if CFG_OFF_PART in str(label).split("|"):
- eager_label = f"{eager_label}|{CFG_OFF_PART}"
- keys.append((eager_label, SPEED_OFF))
- activation = None
- for key_label, tier in dict.fromkeys(keys):
- activation = measured_image_runtime_mib(
- family,
- tier,
- width = width,
- height = height,
- batch_size = batch_size,
- quant = with_workflow(key_label, workflow),
- static_seed = False,
- phase = "pre_decode",
- )
- if activation is not None:
- break
+ activation = _learned_peak_for_keys(
+ family,
+ _learned_peak_keys(state, label, speed),
+ width = width,
+ height = height,
+ batch_size = batch_size,
+ workflow = workflow,
+ phase = "pre_decode",
+ )
if activation is None:
if logger is not None:
logger.info(
@@ -3225,7 +3346,9 @@ def _live_peak_identity(state: Any) -> tuple:
)
label = with_label_part(label, "vae", live_vae_saver_tag(pipe))
label = with_label_part(label, "attn", getattr(state, "attention_backend", None) or "native")
- if "|graphs:" in label:
+ if "|graphs:" in label and getattr(pipe, TIER_MOVED_ATTR, False):
+ label = with_label_part(label, "graphs", _tier_graph_mode(state))
+ elif "|graphs:" in label:
label = with_label_part(label, "graphs", _live_graph_mode(state))
speed = getattr(state, "speed_mode", None)
if "compile_fallback_eager" in tuple(getattr(state, "speed_optims", ()) or ()):
@@ -3274,6 +3397,34 @@ def _live_graph_mode(state: Any) -> str:
return "unknown"
+TIER_FALLBACK_GRAPH_MODE = "step_blocks"
+
+
+def _tier_graph_mode(state: Any) -> str:
+ """``_live_graph_mode``, but the armed mode before the first capture."""
+ try:
+ if "cuda_graph" not in tuple(getattr(state, "speed_optims", ()) or ()):
+ return "off"
+ pipe = getattr(state, "pipe", None)
+ handles = tuple(getattr(state, "cuda_graphs", ()) or ()) or tuple(
+ getattr(pipe, "_unsloth_cuda_graphs", ()) or ()
+ )
+ armed = _engaged_graph_mode(pipe, {"cuda_graph": True})
+ if not handles:
+ return "off"
+ if any(getattr(h, "fallback_handle", None) not in (None, h) for h in handles):
+ # per-block fallback (GraphedForward._engage_fallback) is invisible to stats(): its own peak mode
+ return TIER_FALLBACK_GRAPH_MODE
+ totals = cuda_graph.stats(handles)
+ if totals.get("graphs"):
+ return armed
+ if totals.get("captures") or totals.get("poisoned") or _graphs_settled_eager(state):
+ return "off"
+ return armed
+ except Exception: # noqa: BLE001
+ return "unknown"
+
+
def _peak_te_label(te_quant: Any, outcome: Any) -> str:
"""The text encoders' precision as the learned-peak key names it. A partial cast (some encoders quantised, others
left dense) is no build the key can name: ``unknown``, so it neither reads nor records a learned peak."""
@@ -13124,9 +13275,11 @@ class DiffusionBackend:
)
except Exception: # noqa: BLE001 - recording is optional
pass
- # the reserve was placed for the load's tier: until a reload re-places it, requests size from the flat
- # estimate
- state.pipe._unsloth_peak_static = True
+ # reserve was placed for the load's tier: flat estimate until this tier records its own peak
+ if tier_peaks_enabled():
+ setattr(state.pipe, TIER_MOVED_ATTR, True)
+ else:
+ state.pipe._unsloth_peak_static = True
# Keep the badge in step with the state it describes: the top-level field moved, so leaving `resolved` on the
# load-time value would make the panel report an attention backend that is no longer the one running.
if isinstance(state.resolved, dict) and "attention_backend" in state.resolved:
@@ -13223,6 +13376,7 @@ class DiffusionBackend:
object.__setattr__(
state, "cuda_graphs", tuple(getattr(state.pipe, "_unsloth_cuda_graphs", ()) or ())
)
+ _follow_live_graph_mode(state)
entry = (state.resolved or {}).get("speed_mode")
if isinstance(entry, dict):
entry["value"] = SPEED_DEFAULT
@@ -13883,6 +14037,7 @@ class DiffusionBackend:
),
)
guard_width = guard_height = guard_target = None
+ guard_release_mib, guard_capture_gen = 0, None
guard_batch = 1
guard_condition_pixels = 0
@@ -13935,6 +14090,7 @@ class DiffusionBackend:
and ref_resolution is None
and not is_flux3_family(fam)
)
+ _follow_live_graph_mode(state)
guard_cfg, extra_mib = split_aware_request_extra(
state.pipe,
guard_cfg,
@@ -13948,9 +14104,21 @@ class DiffusionBackend:
cfg = cfg,
unpriced_references = guard_unpriced_refs,
workflow = _peak_workflow(fam, workflow),
+ tier_peak_mib = _tier_request_peak(
+ state,
+ width = guard_width,
+ height = guard_height,
+ batch_size = guard_batch,
+ cfg = cfg,
+ workflow = _peak_workflow(fam, workflow),
+ ),
),
step_cache = auto_step_cache_on,
)
+ if getattr(state.pipe, TIER_MOVED_ATTR, False):
+ extra_mib = _hold_captured_release(state, extra_mib)
+ guard_release_mib = extra_mib
+ guard_capture_gen = cuda_graph.capture_generation(_peak_device_index(state))
if extra_mib > 0:
releasable_mib = resident_group_mib(state.pipe)
if (
@@ -14469,6 +14637,7 @@ class DiffusionBackend:
target = guard_target,
workflow = workflow,
)
+ _note_captured_release(state, guard_release_mib, guard_capture_gen)
if static_skip:
logger.debug("diffusion.step_skip: %s", static_skip_stats(state.pipe))
# Keep progress ACTIVE through the post-denoise work: the route persists the image after this returns,
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 583389fcfa9..e481c58bbf7 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -3644,6 +3644,15 @@ def cfg_off_peaks_enabled() -> bool:
return not _env_off(LEARNED_PEAKS_CFG_OFF_ENV)
+# Deferred profiles record and price their own tier's learned peak; 0 = flat estimate for compiled renders.
+TIER_PEAKS_ENV = "UNSLOTH_DIFFUSION_TIER_PEAKS"
+TIER_MOVED_ATTR = "_unsloth_peak_tier_moved"
+
+
+def tier_peaks_enabled() -> bool:
+ return not _env_off(TIER_PEAKS_ENV)
+
+
def without_cfg_off(label: str) -> str:
return "|".join(part for part in str(label).split("|") if part != CFG_OFF_PART)
@@ -5536,8 +5545,10 @@ def measured_request_extra_mib(
cfg: Optional[bool] = None,
unpriced_references: bool = False,
workflow: Optional[str] = None,
+ tier_peak_mib: Optional[int] = None,
) -> int:
- """MiB a request needs beyond what the measured placement reserved (0 when it fits or none was made). ``workflow``:
+ """MiB a request needs beyond what the measured placement reserved (0 when it fits or none was made). ``tier_peak_mib``:
+ a deferred profile's tier peak (``TIER_MOVED_ATTR``), raising eager pricing; None keeps the flat estimate. ``workflow``:
a workflow other than text-to-image also reads the peak recorded under its own key (``with_workflow``). ``cfg``:
whether the request runs classifier-free guidance (None: unknown, treated as running it). ``unpriced_references``:
the request carries reference images whose latents join the denoiser's sequence but are not in
@@ -5573,11 +5584,14 @@ def measured_request_extra_mib(
(dense_mib is not None and sized["dense_tier_seeds"])
or (dense_mib is None and sized["phase_seeds"])
)
+ tier_moved = bool(getattr(pipe, TIER_MOVED_ATTR, False))
+ tier_peak = int(tier_peak_mib) if tier_moved and tier_peak_mib else None
if (unpriced_references and tier_seeded) or (
# the bf16 seed is eager-only like a learned peak: same invalidation
(learned or need is None or sized["dense_bf16"])
and (
getattr(pipe, "_unsloth_peak_static", False)
+ or (tier_moved and tier_peak is None)
or getattr(pipe, "_unsloth_loras", None)
or cfg_unmeasured
)
@@ -5624,6 +5638,8 @@ def measured_request_extra_mib(
need = max(need, stored)
if need is None and isinstance(identity, tuple) and len(identity) == 3:
need = measured_image_runtime_mib(family, speed_mode, quant = identity[1], **sized)
+ if tier_peak is not None:
+ need = tier_peak if need is None else max(int(need), tier_peak)
cond = max(0, int(condition_pixels or 0)) * max(1, int(batch_size or 1))
allowance = int(
8192 * cond / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT) * _MEASURED_PEAK_MARGIN
diff --git a/studio/backend/tests/test_diffusion_tier_peaks.py b/studio/backend/tests/test_diffusion_tier_peaks.py
new file mode 100644
index 00000000000..7b66445b3c5
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_tier_peaks.py
@@ -0,0 +1,322 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Deferred profile's compiled tier priced from its own learned peak, else flat. FLUX.1-schnell bf16 L4 numbers."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion as dif # noqa: E402
+import core.inference.diffusion_memory as dm # noqa: E402
+
+_REAL_TIER_GRAPH_MODE = dif._tier_graph_mode
+
+EAGER = "dense:bfloat16|model:m@1|te:dense|cache:off|vae:sliced|attn:native|graphs:off"
+COMPILED = "dense:bfloat16|model:m@1|te:dense|cache:off|vae:sliced|attn:_native_cudnn|graphs:off"
+HEADROOM = 4352 # the dense bf16 seed the L4 load reserved
+
+
+def store(tmp_path, monkeypatch):
+ import core.inference.diffusion_measured_peaks as mp
+
+ for name in (
+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION",
+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE",
+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16",
+ dm.TIER_PEAKS_ENV,
+ ):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(tmp_path / "peaks.json"))
+ monkeypatch.delenv(mp.LEARNED_PEAKS_ENV, raising = False)
+ mp._cache.update(path = None, mtime = None, data = None)
+ monkeypatch.setattr(mp, "current_device_name", lambda: "NVIDIA L4", raising = False)
+ return mp
+
+
+def _pipe(**kw):
+ pipe = types.SimpleNamespace(
+ _unsloth_dense_tier_seeds = True,
+ _unsloth_peak_identity = ("flux.1", COMPILED, "default"),
+ _unsloth_measured_reserve = (
+ HEADROOM,
+ "flux.1",
+ "off",
+ 22681,
+ None,
+ None,
+ None,
+ False,
+ True,
+ False,
+ True,
+ ),
+ )
+ setattr(pipe, dm.TIER_MOVED_ATTR, True)
+ for key, value in kw.items():
+ setattr(pipe, key, value)
+ return pipe
+
+
+def _flat_extra():
+ flat = dm.estimate_image_runtime_mib(width = 1024, height = 1024, batch_size = 1, family = "flux.1")
+ return max(0, int(flat) - HEADROOM)
+
+
+def test_the_first_compiled_render_keeps_the_flat_estimate(store):
+ """No measurement of the compiled tier yet: the flat estimate stands in (the L4's 3840 MiB release)."""
+ pipe = _pipe()
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == _flat_extra() == 3840
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, tier_peak_mib = None) == 3840
+
+
+def test_a_measured_compiled_peak_inside_the_reserve_releases_nothing(store):
+ pipe = _pipe()
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, tier_peak_mib = 4096) == 0
+
+
+def test_a_compiled_peak_above_the_reserve_still_streams_the_difference(store):
+ """Compiled buffers can exceed eager: the guard releases exactly what the measured compiled peak needs."""
+ pipe = _pipe()
+ assert (
+ dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, tier_peak_mib = 9984)
+ == 9984 - HEADROOM
+ )
+
+
+def test_a_compiled_peak_below_the_eager_pricing_never_lowers_it(store):
+ """The eager seed at 2 MP (8448 MiB) stands when the compiled tier measured less there."""
+ pipe = _pipe()
+ eager = dm.measured_request_extra_mib(
+ types.SimpleNamespace(**{k: v for k, v in vars(pipe).items() if k != dm.TIER_MOVED_ATTR}),
+ width = 2048,
+ height = 1024,
+ )
+ assert eager == 8448 - HEADROOM
+ assert dm.measured_request_extra_mib(pipe, width = 2048, height = 1024, tier_peak_mib = 5120) == eager
+
+
+ "extra",
+ [{"_unsloth_peak_static": True}, {"_unsloth_loras": ["a"]}],
+)
+def test_other_invalidations_keep_the_flat_estimate(store, extra):
+ """A VAE saver that did not engage, or runtime LoRA adapters: no tier peak describes the pipeline."""
+ pipe = _pipe(**extra)
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, tier_peak_mib = 4096) == 3840
+
+
+def test_a_tier_peak_without_a_moved_tier_is_ignored(store):
+ pipe = _pipe()
+ delattr(pipe, dm.TIER_MOVED_ATTR)
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, tier_peak_mib = 99999) == 0
+
+
+def test_kill_switch(store, monkeypatch):
+ assert dm.tier_peaks_enabled()
+ monkeypatch.setenv(dm.TIER_PEAKS_ENV, "0")
+ assert not dm.tier_peaks_enabled()
+
+
+def _state(
+ pipe,
+ speed = "default",
+ graph_mode = "off",
+):
+ return types.SimpleNamespace(
+ pipe = pipe,
+ speed_mode = speed,
+ speed_optims = ("compiled",),
+ transformer_cache = None,
+ attention_backend = "_native_cudnn",
+ family = types.SimpleNamespace(uses_negative_prompt = False),
+ device = "cuda:0",
+ _graph_mode = graph_mode,
+ )
+
+
+def live(monkeypatch):
+ monkeypatch.setattr(dif, "live_vae_saver_tag", lambda pipe: "sliced")
+ monkeypatch.setattr(dif, "_live_graph_mode", lambda state: state._graph_mode)
+ monkeypatch.setattr(dif, "_tier_graph_mode", lambda state: state._graph_mode)
+ monkeypatch.setattr(dif, "_peak_device_index", lambda state: 0)
+
+
+def _record(store, label, speed, peak):
+ assert store.record(
+ family = "flux.1",
+ quant = label,
+ speed_mode = speed,
+ width = 1024,
+ height = 1024,
+ peak_mib = peak,
+ device_name = "NVIDIA L4",
+ )
+
+
+def _tier_peak(state, **kw):
+ return dif._tier_request_peak(
+ state, width = 1024, height = 1024, batch_size = 1, cfg = kw.pop("cfg", None), **kw
+ )
+
+
+def test_the_eager_peak_never_stands_in_for_the_compiled_tier(store, live):
+ """The whole-module path (#15163) lets the eager key stand in; the request guard reads only the compiled tier."""
+ _record(store, EAGER, "off", 3581)
+ _record(store, COMPILED, "off", 3581)
+ state = _state(_pipe())
+ assert _tier_peak(state) is None
+ keys = dif._learned_peak_keys(state, COMPILED, "default", eager_stand_in = False)
+ assert all(tier == "default" for _, tier in keys)
+ assert (COMPILED, "off") in dif._learned_peak_keys(state, COMPILED, "default")
+
+
+def test_the_compiled_tier_reads_its_own_recorded_peak(store, live):
+ state = _state(_pipe())
+ _record(store, COMPILED, "default", 3900)
+ # 3900 x 1.15 margin, rounded up to 256 MiB
+ assert _tier_peak(state) == 4608
+
+
+def test_each_graph_mode_answers_only_for_itself(store, live):
+ """A capturing render's peak does not cover the renders after its graph is dropped (measured: 1447 MiB with the
+ graph, 2546 MiB after the drop): a dropped graph prices flat until the eager-compiled renders record."""
+ step = COMPILED.replace("graphs:off", "graphs:step")
+ _record(store, step, "default", 1447)
+ assert _tier_peak(_state(_pipe(), graph_mode = "off")) is None
+ assert _tier_peak(_state(_pipe(), graph_mode = "step")) == 1792
+ _record(store, COMPILED, "default", 2546)
+ assert _tier_peak(_state(_pipe(), graph_mode = "off")) == 3072
+
+
+def test_no_tier_peak_while_the_render_runs_eager_or_unkeyed(store, live, monkeypatch):
+ _record(store, COMPILED, "default", 3900)
+ assert _tier_peak(_state(_pipe(_unsloth_loras = ["x"]))) is None
+ monkeypatch.setattr("core.inference.diffusion_bg_compile.eager_forced", lambda: True)
+ assert _tier_peak(_state(_pipe())) is None
+
+
+def test_no_tier_peak_before_the_profile_engaged(store, live):
+ pipe = _pipe()
+ delattr(pipe, dm.TIER_MOVED_ATTR)
+ _record(store, COMPILED, "default", 3900)
+ assert _tier_peak(_state(pipe)) is None
+
+
+def test_the_identity_follows_the_graph_mode_the_session_keeps(live):
+ blocks = COMPILED.replace("graphs:off", "graphs:blocks")
+ pipe = _pipe(_unsloth_peak_identity = ("flux.1", COMPILED, "default"))
+ dif._follow_live_graph_mode(_state(pipe, graph_mode = "blocks"))
+ assert pipe._unsloth_peak_identity == ("flux.1", blocks, "default")
+ dif._follow_live_graph_mode(_state(pipe, graph_mode = "off"))
+ assert pipe._unsloth_peak_identity == ("flux.1", COMPILED, "default")
+ eager = _pipe(_unsloth_peak_identity = ("flux.1", EAGER, "off"))
+ delattr(eager, dm.TIER_MOVED_ATTR)
+ dif._follow_live_graph_mode(_state(eager, graph_mode = "blocks"))
+ assert eager._unsloth_peak_identity == ("flux.1", EAGER, "off")
+
+
+def test_a_live_graph_keeps_the_release_it_was_captured_over(live, monkeypatch):
+ """Placement a graph was captured over is held until no graph is live."""
+ graphs = {"live": 0, "gen": 0}
+ monkeypatch.setattr(dif.cuda_graph, "live_graph_count", lambda index: graphs["live"])
+ monkeypatch.setattr(dif.cuda_graph, "capture_generation", lambda index: graphs["gen"])
+ state = _state(_pipe())
+ assert dif._hold_captured_release(state, 0) == 0
+ before = graphs["gen"]
+ graphs.update(live = 1, gen = 1) # the render captured a graph over a 3840 MiB release
+ dif._note_captured_release(state, 3840, before)
+ assert dif._hold_captured_release(state, 0) == 3840
+ assert dif._hold_captured_release(state, 5000) == 5000
+ dif._note_captured_release(state, 0, graphs["gen"])
+ assert dif._hold_captured_release(state, 0) == 3840
+ graphs["live"] = 0 # dropped
+ assert dif._hold_captured_release(state, 0) == 0
+ assert state.pipe._unsloth_captured_release_mib is None
+
+
+def test_nothing_is_held_before_the_profile_engaged(live, monkeypatch):
+ monkeypatch.setattr(dif.cuda_graph, "capture_generation", lambda index: 5)
+ pipe = _pipe()
+ delattr(pipe, dm.TIER_MOVED_ATTR)
+ state = _state(pipe)
+ dif._note_captured_release(state, 3840, 0)
+ assert getattr(pipe, "_unsloth_captured_release_mib", None) is None
+
+
+class _Handle:
+ def __init__(
+ self,
+ cache = 0,
+ captures = 0,
+ poisoned = False,
+ declined = False,
+ ):
+ self.cache = [object()] * cache
+ self.stats = {"captures": captures}
+ self.poisoned = poisoned
+ self.capture_error = None
+ self.fallback_handle = None
+ self.fallback = None
+ self._declined = declined
+
+ def _placement_declined(self):
+ return self._declined
+
+
+ "handle, mode",
+ [
+ (
+ _Handle(),
+ "step",
+ ), # armed, before its first capture: the render that captures keeps the key
+ (_Handle(cache = 1, captures = 1), "step"), # kept
+ (_Handle(cache = 0, captures = 1), "off"), # captured, then dropped
+ (_Handle(poisoned = True), "off"),
+ (_Handle(declined = True), "off"),
+ ],
+)
+def test_the_tier_graph_mode_is_stable_across_the_first_capture(monkeypatch, handle, mode):
+ monkeypatch.setattr(dif.cuda_graph, "live_graph_count", lambda index: len(handle.cache))
+ monkeypatch.setattr(dif, "_peak_device_index", lambda state: 0)
+ pipe = _pipe(_unsloth_cuda_graphs = (handle,))
+ state = _state(pipe)
+ state.speed_optims = ("compiled", "cuda_graph")
+ state.cuda_graphs = (handle,)
+ assert dif._tier_graph_mode(state) == mode
+ state.speed_optims = ("compiled",)
+ assert dif._tier_graph_mode(state) == "off"
+
+
+def test_an_engaged_per_block_fallback_keys_its_own_graph_mode(store, live, monkeypatch):
+ """A dropped whole-step graph whose calls now run per-block graphs (``fallback_handle``) is neither ``step`` nor
+ ``off``: it reads and records under its own mode, so a step or off peak never prices it."""
+ monkeypatch.setattr(dif.cuda_graph, "live_graph_count", lambda index: 0)
+ monkeypatch.setattr(dif, "_tier_graph_mode", _REAL_TIER_GRAPH_MODE)
+ whole = _Handle(cache = 0, captures = 1) # its own graph dropped
+ whole.fallback_handle = _Handle(cache = 2, captures = 2) # per-block graphs live
+ pipe = _pipe(_unsloth_cuda_graphs = (whole,))
+ state = _state(pipe)
+ state.speed_optims = ("compiled", "cuda_graph")
+ state.cuda_graphs = (whole,)
+ assert dif._tier_graph_mode(state) == dif.TIER_FALLBACK_GRAPH_MODE
+ _record(store, COMPILED, "default", 2546)
+ _record(store, COMPILED.replace("graphs:off", "graphs:step"), "default", 1447)
+ assert _tier_peak(state) is None # neither the off nor the step peak prices it
+ dif._follow_live_graph_mode(state)
+ assert pipe._unsloth_peak_identity[1].endswith("graphs:" + dif.TIER_FALLBACK_GRAPH_MODE)
+ _record(store, pipe._unsloth_peak_identity[1], "default", 3000)
+ assert _tier_peak(state) == 3584 # its own entry: 3000 x 1.15, rounded up to 256 MiB
+ # not engaged yet (the fallback is only armed): the whole step's own mode
+ armed = _Handle()
+ armed.fallback_handle = None
+ state.cuda_graphs = (armed,)
+ pipe._unsloth_cuda_graphs = (armed,)
+ assert dif._tier_graph_mode(state) == "step"