Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: keep a streamed denoiser's blocks resident for the denoise phase, not the whole render (A100 8 GiB Qwen-Image-2.1 -18%, FLUX.3 -21 to -24%) (#17207)
Daniel Han0bfc894fed07
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 66830614a0d..dbd52741476 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -267,6 +267,9 @@ from .diffusion_memory import (applied_plan,release_resident_groups,release_for_decode,+ chained_restore,+ defer_resident_restore,+ take_deferred_restore,resident_group_mib,hook_resident_denoiser,measured_request_extra_mib,@@ -11128,6 +11131,8 @@ class DiffusionBackend:plan.estimates.get("measured_dense_bf16", False),plan.estimates.get("measured_small_host_int8", False),bool(streamed_headroom),+ # MEASURED_RESERVE_DENOISE_PHASE_INDEX: the blocks sized from the denoise phase+ plan.estimates.get("measured_denoise_phase_headroom_mib"),)setattr(pipe, DECODE_LEND_ATTR, decode_lend_from_plan(plan))@@ -14944,6 +14949,8 @@ class DiffusionBackend:extra_mib = _hold_captured_release(state, extra_mib)guard_release_mib = extra_mibguard_capture_gen = cuda_graph.capture_generation(_peak_device_index(state))+ # groups an earlier oversized request streamed out stay out when this one streams them all too+ carried_restore, extra_mib = take_deferred_restore(state.pipe, extra_mib)if extra_mib > 0:releasable_mib = resident_group_mib(state.pipe)if (@@ -14953,6 +14960,12 @@ class DiffusionBackend:# transformer so this request can stream part of it.hook_resident_denoiser(state.pipe, guard_target.torch_device, logger)restore_resident = release_resident_groups(state.pipe, extra_mib, logger)+ if carried_restore is not None:+ restore_resident = (+ carried_restore+ if restore_resident is None+ else chained_restore(carried_restore, restore_resident)+ )guard_kwargs = dict(# Not the settled snapshot: its empty_cache() is wrong per generation (re-mallocs activations).device_memory = reclaimable_snapshot_device_memory(guard_target),@@ -15619,7 +15632,8 @@ class DiffusionBackend:end_cfg_batch_render(getattr(state, "pipe", None))if restore_resident is not None:try:- restore_resident()+ if kick_bg or not defer_resident_restore(state.pipe, restore_resident):+ restore_resident() # a background compile kicked below needs it back firstexcept Exception as exc: # noqa: BLE001 - the groups keep streaminglogger.warning("diffusion.memory: re-residency failed: %s", exc)if kick_bg and bg is not None:diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex a1348c25fc5..f3bbdd688e8 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -4124,6 +4124,111 @@ def _streamed_dit_room_mib(memory: Any, headroom_mib: int, other_mib: int, windo)+# Size a streamed denoiser's resident blocks from the denoise phase (encode/decode release around it); "0" = whole-render peak.+DENOISE_PHASE_RESIDENCY_ENV = "UNSLOTH_DIFFUSION_DENOISE_PHASE_RESIDENCY"+_PHASE_DENOISE_COLUMN = 1+_PHASE_UNTILED_DECODE_COLUMN = 2+_PHASE_TILED_DECODE_COLUMN = 3+++def denoise_phase_residency_enabled() -> bool:+ return not _env_off(DENOISE_PHASE_RESIDENCY_ENV)+++def _top_level_group_mib(pipe: Any) -> Optional[int]:+ """MiB of the largest denoiser's weights outside its block stacks (``ModuleList`` / ``Sequential`` children):+ block-level group offload keeps them in one top-level group that is onloaded for the whole forward when it is not+ resident, beside the block window. None when unreadable."""+ try:+ import torch++ largest = 0+ for name, module in (getattr(pipe, "components", {}) or {}).items():+ if str(name) not in ("transformer", "transformer_2", "unconditional_transformer"):+ continue+ if not isinstance(module, torch.nn.Module):+ continue+ blocks: set[int] = set()+ for child in module.children():+ if isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):+ blocks.update(id(t) for t in (*child.parameters(), *child.buffers()))+ seen: set[int] = set()+ nbytes = 0+ for t in (*module.parameters(), *module.buffers()):+ if id(t) in seen or id(t) in blocks:+ continue+ seen.add(id(t))+ nbytes += sum(_storage_nbytes(t))+ largest = max(largest, -(-nbytes // (1024 * 1024)))+ return largest+ except Exception: # noqa: BLE001+ return None+++def _denoise_phase_reserve(+ pipe: Any,+ *,+ family: Optional[str],+ speed_mode: Optional[str],+ quant: Optional[str],+ static_seed: bool,+ window_mib: int,+ tiled: bool,+) -> Optional[tuple[int, tuple[int, int, bool]]]:+ """``(denoise MiB, (decode peak MiB, bucket pixels, quadratic))`` for a 1024x1024 render, or None.++ The denoise MiB is what the steps allocate above the resident weights besides the prefetch window (margin+ included): the larger of the static denoise seed (``_MEASURED_IMAGE_ACTIVATION_MIB``, window excluded) and this+ build's learned pre-decode peak less the part of the window it is sure to hold (learned peaks are on the streamed+ scale, ``stream_window_pad_mib``, at the prefetch depth they were recorded with), so no window is charged twice. The decode peak (no margin) is the larger of the+ static decode seed for the VAE savers in force and the learned unpadded decode phase. None when neither source+ covers both phases, when a learned whole-render peak has no consistent phase split (its phases are unknown), or+ when the switch is off."""+ if not denoise_phase_residency_enabled() or _env_off(MEASURED_ACTIVATION_ENV):+ return None+ try:+ from .diffusion_measured_peaks import (+ PHASE_DECODE_UNPADDED,+ PHASE_PRE_DECODE,+ lookup_bucket,+ )++ name = str(family or "").lower()+ denoise = decode = None+ pixels, quadratic = DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT, False+ phases = _MEASURED_IMAGE_ACTIVATION_MIB.get(name)+ if (+ static_seed+ and phases is not None+ and bool(getattr(pipe, "_unsloth_phase_seeds", False))+ and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES+ ):+ row = phases[1 if speed_mode == "max" else 0]+ denoise = int(math.ceil(row[_PHASE_DENOISE_COLUMN] * _MEASURED_PEAK_MARGIN))+ decode = int(row[_PHASE_TILED_DECODE_COLUMN if tiled else _PHASE_UNTILED_DECODE_COLUMN])+ if quant is not None:+ whole = lookup_bucket(family = family, quant = quant, speed_mode = speed_mode)+ if whole is not None:+ split = learned_phase_split(family, quant, speed_mode)+ if split is None:+ return None+ from .diffusion_offload_prefetch import prefetch_depth++ # the sample holds the window of the depth it was recorded at, which the key does not name: credit+ # only the smallest one (depth 1, two blocks), so a later deeper prefetch is never double-credited+ held = max(0, int(window_mib)) * 2 // (prefetch_depth() + 1)+ learned_pre = int(math.ceil(split[PHASE_PRE_DECODE] * _MEASURED_PEAK_MARGIN)) - held+ denoise = max(int(denoise or 0), learned_pre, 0)+ decode = max(int(decode or 0), int(split[PHASE_DECODE_UNPADDED]))+ pixels, quadratic = int(split["pixels"]), bool(split["quadratic"])+ if denoise is None or not decode:+ return None+ step = _MEASURED_PEAK_ROUND_MIB+ return int(-(-int(denoise) // step) * step), (int(decode), int(pixels), bool(quadratic))+ except Exception: # noqa: BLE001 - unreadable: the whole-render placement+ return None++def _denoiser_compute_bytes(pipe: Any) -> Optional[int]:"""Denoiser compute element size for the dense table: 2 (fp16), 4 (fp32); None for bf16 / unreadable."""try:@@ -4789,6 +4894,34 @@ def _refine_plan_from_loaded_weights(streamed_room = min(_streamed_dit_room_mib(memory, streamed_headroom, other, window), dit)+ phase_reserve = None+ if window >= 0:+ phase_reserve = _denoise_phase_reserve(+ pipe,+ family = family,+ speed_mode = speed_mode,+ quant = read_quant,+ static_seed = static_seed and not small_host_int8,+ window_mib = window,+ tiled = "tiled" in planned_vae_saver_tag(pipe, plan),+ )+ top = _top_level_group_mib(pipe) if phase_reserve is not None else None+ if top is None:+ phase_reserve = None+ if phase_reserve is not None:+ phase_room = min(+ _streamed_dit_room_mib(memory, phase_reserve[0], other, window), dit+ )+ if (+ phase_reserve[0] >= streamed_headroom+ or phase_room <= streamed_room+ # the top-level group goes resident first; one that misses is onloaded whole for every+ # forward beside the block window, which the denoise seed (measured resident) omits+ or phase_room < top+ ):+ phase_reserve = None+ else:+ streamed_room = phase_roomif streamed_room > dit_room:encode_room = int(dit_room)dit_room, room, wide_dit = streamed_room, streamed_room, True@@ -4797,6 +4930,16 @@ def _refine_plan_from_loaded_weights(estimates["measured_streamed_headroom_mib"] = streamed_headroom# seed covers denoise only; flat encode room on top overran at 16 GiBencode_room = 0+ if phase_reserve is not None:+ decode_peak, decode_pixels, decode_quadratic = phase_reserve[1]+ # the encode keeps the flat room, the decode streams the shortfall again+ estimates.update(+ measured_denoise_phase_headroom_mib = phase_reserve[0],+ decode_lend_transformer_mib = int(streamed_room),+ measured_decode_peak_mib = decode_peak,+ measured_decode_peak_pixels = decode_pixels,+ measured_decode_peak_quadratic = decode_quadratic,+ )te_room = (min(max(0, room - dit_room), encoders) if stream_te and policy == OFFLOAD_GROUP else 0)@@ -4863,8 +5006,14 @@ def _refine_plan_from_loaded_weights(f" {encode_room} MiB of it while the encoders run)"if whole_ditelse f" (free memory less a {_resident_dit_slack_mib(memory)} MiB slack and a "- f"{estimates.get('stream_window_mib')} MiB prefetch window; {encode_room} MiB of it while the "- "encoders run)"+ f"{estimates.get('stream_window_mib')} MiB prefetch window"+ + (+ f", sized from the {estimates['measured_denoise_phase_headroom_mib']} MiB denoise phase"+ " (the decode may stream part of it again)"+ if estimates.get("measured_denoise_phase_headroom_mib") is not None+ else ""+ )+ + f"; {encode_room} MiB of it while the encoders run)"if wide_ditelse ""),@@ -5233,7 +5382,7 @@ def release_resident_groups((int(need_mib) * 1024 * 1024 - max(left, 0)) >> 20,reason,)-+ restore.released_mib = (int(need_mib) * 1024 * 1024 - left) >> 20 # take_deferred_restorereturn restoreexcept Exception as exc: # noqa: BLE001 - the guard still refuses what cannot fitif logger is not None:@@ -5242,17 +5391,102 @@ def release_resident_groups(return restore if released else None+# Restoring after every oversized request costs a host-to-device round trip per render; wait for one that needs them.+DEFERRED_RESTORE_ATTR = "_unsloth_deferred_resident_restore"+++def _phase_placement(pipe: Any) -> bool:+ reserve = getattr(pipe, "_unsloth_measured_reserve", None) or ()+ return bool(+ len(reserve) > MEASURED_RESERVE_DENOISE_PHASE_INDEX+ and reserve[MEASURED_RESERVE_DENOISE_PHASE_INDEX]+ and denoise_phase_residency_enabled()+ )+++def chained_restore(first: Callable[[], None], second: Callable[[], None]) -> Callable[[], None]:+ def restore() -> None:+ try:+ first()+ finally:+ second()++ restore.released_mib = int(getattr(first, "released_mib", 0) or 0) + int(+ getattr(second, "released_mib", 0) or 0+ )+ return restore+++def defer_resident_restore(pipe: Any, restore: Optional[Callable[[], None]]) -> bool:+ """Hold ``restore`` (an oversized request's ``release_resident_groups``) on ``pipe`` instead of running it, for a+ placement sized from the denoise phase. True when held; False: the caller restores now."""+ if restore is None or not _phase_placement(pipe):+ return False+ try:+ held = getattr(pipe, DEFERRED_RESTORE_ATTR, None)+ setattr(+ pipe, DEFERRED_RESTORE_ATTR, restore if held is None else chained_restore(held, restore)+ )+ except Exception: # noqa: BLE001 - unheld: restore now+ return False+ return True+++# A held restore carries on while the next request's own release is at least this close to what it streamed out.+_DEFER_TOLERANCE_MIB = 256+++def _unused_room_mib(pipe: Any) -> int:+ """MiB of the placement's resident rooms its whole groups left unused (free memory a request may count on)."""+ room = placed = 0+ for module in (getattr(pipe, "components", {}) or {}).values():+ if not getattr(module, "_unsloth_resident_room", None):+ continue+ room += int(module._unsloth_resident_room)+ placed += sum(+ int(getattr(g, "_unsloth_resident_bytes", 0) or 0)+ for g in (_offload_groups(module) or ())+ )+ return max(0, room - (placed >> 20))+++def take_deferred_restore(pipe: Any, extra_mib: int) -> tuple[Optional[Callable[[], None]], int]:+ """At a request's guard: ``(restore to carry on, MiB the request must still release)``. For a placement sized from+ the denoise phase the request's MiB (priced against its room) first counts the room its whole groups left unused.+ A request releasing about as much as a held restore streamed out (or more) keeps those groups streamed and+ releases only the rest; any other request (one the placement was sized for, or a much smaller release) pins them+ back first."""+ extra = int(extra_mib or 0)+ if extra > 0 and _phase_placement(pipe):+ extra = max(0, extra - _unused_room_mib(pipe))+ held = getattr(pipe, DEFERRED_RESTORE_ATTR, None)+ if held is None:+ return None, extra+ try:+ setattr(pipe, DEFERRED_RESTORE_ATTR, None)+ except Exception: # noqa: BLE001+ pass+ streamed = int(getattr(held, "released_mib", 0) or 0)+ if extra > 0 and extra + _DEFER_TOLERANCE_MIB >= streamed:+ return held, max(0, extra - streamed)+ held()+ return None, extra++DECODE_LEND_ATTR = "_unsloth_decode_lend"# Kept free beside the decode's learned peak x margin when deciding whether lent encoder groups must stream again._DECODE_SLACK_MIB = 256def decode_lend_from_plan(plan: Any) -> Optional[tuple[int, int, int, bool]]:- """``(lent MiB, decode peak MiB, bucket pixels, quadratic)`` when ``plan`` lent encoder residency to the decode."""+ """``(lent MiB, decode peak MiB, bucket pixels, quadratic)`` when ``plan`` lent encoder residency, or denoiser+ residency sized from the denoise phase, to the decode."""estimates = getattr(plan, "estimates", None) or {}lend = estimates.get("decode_lend_text_encoder_mib")if not lend or not getattr(plan, "resident_text_encoder_mib", None):- return None+ lend = estimates.get("decode_lend_transformer_mib")+ if not lend or not getattr(plan, "resident_transformer_mib", None):+ return Nonereturn (int(lend),int(estimates.get("measured_decode_peak_mib") or 0),@@ -5813,6 +6047,9 @@ def install_encode_release(return 0+MEASURED_RESERVE_DENOISE_PHASE_INDEX = 13++def measured_request_extra_mib(pipe: Any,*,@@ -5831,10 +6068,21 @@ def measured_request_extra_mib(whether the request runs classifier-free guidance (None: unknown, treated as running it). ``unpriced_references``:the request carries reference images whose latents join the denoiser's sequence but are not in``condition_pixels`` (FLUX.2 Klein's primary reference): a placement made from a dense default / max tier seed,- measured on text-to-image, then sizes the request from the flat estimate."""+ measured on text-to-image, then sizes the request from the flat estimate.++ A placement that sized its resident blocks from the denoise phase (``MEASURED_RESERVE_DENOISE_PHASE_INDEX``) holds+ only that phase's reserve while the steps run: a request it was sized for (text-to-image, at most the 1024x1024+ bucket, no condition images or tier peak, within the whole-render reserve) releases nothing, its encode and decode+ priced by their own hooks; any other request is priced against the denoise reserve, which first streams the blocks+ the phase sizing added."""reserve = getattr(pipe, "_unsloth_measured_reserve", None)if not reserve:return 0+ phase = (+ reserve[MEASURED_RESERVE_DENOISE_PHASE_INDEX]+ if len(reserve) > MEASURED_RESERVE_DENOISE_PHASE_INDEX+ else None+ )headroom, family, speed_mode = reserve[:3]dense_mib = reserve[3] if len(reserve) > 3 else Nonecompute_bytes = reserve[4] if len(reserve) > 4 and reserve[4] else 2@@ -5885,7 +6133,7 @@ def measured_request_extra_mib(family = family,condition_pixels = condition_pixels,)- return max(0, int(flat) - int(headroom))+ return max(0, int(flat) - int(phase or headroom))if learned:# the learned peak the load reserved, scaled here (not reread from the store, which another worker may evict# or a failed write may leave empty while this pipeline stays loaded)@@ -5926,9 +6174,11 @@ def measured_request_extra_mib()if need is not None:need += allowance+ sized_for = True # the request is the one the placement was sized for (``phase`` pricing)if isinstance(identity, tuple) and len(identity) == 3:keyed = with_workflow(identity[1], workflow)if keyed != identity[1]:+ sized_for = False # another workflow: its denoise phase was never measured against the placementown = Noneif (cfg is False@@ -5946,6 +6196,20 @@ def measured_request_extra_mib(need = own if need is None else max(need, own)if need is None:return 0+ if phase:+ pixels = (+ max(64, int(width or DEFAULT_IMAGE_WIDTH))+ * max(64, int(height or DEFAULT_IMAGE_HEIGHT))+ * max(1, int(batch_size or 1))+ )+ sized_for = (+ sized_for+ and allowance == 0+ and tier_peak is None+ and pixels <= DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT+ and int(need) <= int(headroom)+ )+ return 0 if sized_for else max(0, int(need) - int(phase))return max(0, int(need) - int(headroom))diff --git a/studio/backend/tests/test_diffusion_denoise_phase_residency.py b/studio/backend/tests/test_diffusion_denoise_phase_residency.pynew file mode 100644index 00000000000..10de6fd4f0e--- /dev/null+++ b/studio/backend/tests/test_diffusion_denoise_phase_residency.py@@ -0,0 +1,405 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A partially streamed denoiser sizes its resident blocks from the denoise phase; CPU only.++Numbers from a Qwen-Image-2.1 int8 load on an A100 under an 8 GiB budget: 7482 MiB free, 6922 MiB DiT, 644 MiB VAE,+625 MiB prefetch window, tiled + sliced VAE."""++from __future__ import annotations++import types++import pytest++import core.inference.diffusion_measured_peaks as mp+import core.inference.diffusion_memory as dm++LOADED = {+ "transformer": (6922, "dit"),+ "text_encoder": (8917, "text_encoder"),+ "vae": (644, "other"),+}+WINDOW = 625+FREE_8G = 7482+QUANT = "torchao:Int8Tensor"+++def _memory(budget_mib: int, total_mib: int) -> "dm.DeviceMemory":+ reserve = dm._reserve_mib("discrete_vram", total_mib)+ return dm.DeviceMemory("cuda", "cuda", "discrete_vram", budget_mib + reserve, total_mib)+++def _flat_plan(free_mib: int, total_mib: int):+ reserve = dm._reserve_mib("discrete_vram", total_mib)+ plan = dm.plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = _memory(free_mib - reserve, total_mib),+ runtime_headroom_mib = dm.estimate_image_runtime_mib(+ width = None, height = None, family = "qwen-image-2.1"+ ),+ requested_mode = None,+ model_dense_mib = 19630,+ companion_dense_mib = 12182,+ text_encoder_dense_mib = 10847,+ )+ if plan.offload_policy == dm.OFFLOAD_MODEL:+ plan = dm.torchao_streaming_plan(plan)+ return plan+++def store(tmp_path, monkeypatch):+ path = tmp_path / "peaks.json"+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(path))+ for env in (+ mp.LEARNED_PEAKS_ENV,+ mp.PHASE_PEAKS_ENV,+ dm.MEASURED_ACTIVATION_ENV,+ dm.PARTIAL_RESIDENT_ENV,+ dm.DENOISE_PHASE_RESIDENCY_ENV,+ dm.STREAMED_RESIDENCY_ENV,+ dm.RESIDENT_DIT_ENV,+ ):+ monkeypatch.delenv(env, raising = False)+ mp._cache.update(path = None, mtime = None, data = None)+ monkeypatch.setattr(mp, "current_device_name", lambda: "NVIDIA A100-SXM4-40GB")+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA A100-SXM4-40GB")+ return path+++def pipe(monkeypatch, store):+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(LOADED))+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ monkeypatch.setattr(dm, "denoiser_quant_label", lambda pipe: QUANT)+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: WINDOW)+ monkeypatch.setattr(dm, "planned_vae_saver_tag", lambda pipe, plan: "tiled+sliced")+ return types.SimpleNamespace(_unsloth_phase_seeds = True)+++def _refine(+ pipe,+ plan,+ family = "qwen-image-2.1",+ speed = "default",+):+ return dm.refine_plan_from_loaded_weights(+ pipe, plan, family = family, speed_mode = speed, attention = None+ )+++def _record(peak, phases = None):+ mp.record(+ family = "qwen-image-2.1",+ quant = QUANT,+ speed_mode = "default",+ width = 1024,+ height = 1024,+ batch_size = 1,+ peak_mib = peak,+ device_name = "NVIDIA A100-SXM4-40GB",+ phases = phases or {},+ )+++def _old_room() -> int:+ """The whole-render placement: the 1849 MiB seed x margin (2304) instead of the denoise phase."""+ return FREE_8G - 1024 - 644 - 2304 - WINDOW+++def test_8gib_sizes_resident_blocks_from_the_denoise_phase(pipe):+ plan = _flat_plan(FREE_8G, 8192)+ assert plan.offload_policy == dm.OFFLOAD_STREAMING+ new = _refine(pipe, plan)+ # denoise seed 666 MiB x 1.15, rounded up to 256 MiB; the window is charged once beside it+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 768+ assert new.resident_transformer_mib == FREE_8G - 1024 - 644 - 768 - WINDOW == 4421+ assert new.resident_transformer_mib > _old_room() == 2885+ # the whole-render reserve is unchanged (request pricing, encode room)+ assert new.estimates["measured_runtime_headroom_mib"] == 2304+ assert (+ new.estimates["encode_resident_transformer_mib"]+ == plan.estimates["safe_device_budget_mib"] - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644+ )+ # the decode borrows the blocks back: tiled decode seed, 1 MP bucket, linear+ assert dm.decode_lend_from_plan(new) == (4421, 1697, 1024 * 1024, False)+ assert "denoise phase" in new.reasons[-1]++[email protected]("top, resident", [(3000, 4421), (4421, 4421), (4422, None)])+def test_a_streamed_top_level_group_is_charged_beside_the_window(pipe, monkeypatch, top, resident):+ """The top-level group goes resident first; one larger than the room is onloaded whole every forward."""+ monkeypatch.setattr(dm, "_top_level_group_mib", lambda pipe: top)+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))+ if resident is None:+ assert "measured_denoise_phase_headroom_mib" not in new.estimates+ assert new.resident_transformer_mib == _old_room()+ else:+ assert new.resident_transformer_mib == resident+++def test_kill_switch_restores_the_whole_render_placement(pipe, monkeypatch):+ monkeypatch.setenv(dm.DENOISE_PHASE_RESIDENCY_ENV, "0")+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))+ assert new.resident_transformer_mib == _old_room()+ assert "measured_denoise_phase_headroom_mib" not in new.estimates+ assert "decode_lend_transformer_mib" not in new.estimates+ assert dm.decode_lend_from_plan(new) is None+++def test_untiled_decode_reads_the_untiled_seed(pipe, monkeypatch):+ monkeypatch.setattr(dm, "planned_vae_saver_tag", lambda pipe, plan: "full")+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))+ assert dm.decode_lend_from_plan(new)[1] == 7648+++def test_learned_phases_charge_the_window_once(pipe):+ # what the 8 GiB render recorded: pre-decode 1189 (window included), decode 1739+ _record(+ 1739,+ {+ mp.PHASE_PRE_DECODE: 1189,+ mp.PHASE_DECODE: 1739,+ mp.PHASE_DECODE_UNPADDED: 1739,+ mp.PHASE_UNPADDED: 1739,+ },+ )+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))+ # 1189 x 1.15 = 1368 less the depth-1 window the sample surely holds (2 of 3 blocks: 416) = 952 -> 1024+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 1024+ assert new.resident_transformer_mib == FREE_8G - 1024 - 644 - 1024 - WINDOW == 4165+ assert dm.decode_lend_from_plan(new) == (4165, 1739, 1024 * 1024, False)+++def test_a_deeper_prefetch_never_credits_more_window_than_the_sample_held(pipe, monkeypatch):+ """A sample recorded at depth 2 read under depth 8: only the depth-1 share of the current window is credited."""+ import core.inference.diffusion_offload_prefetch as pf++ _record(+ 1739,+ {+ mp.PHASE_PRE_DECODE: 1189,+ mp.PHASE_DECODE: 1739,+ mp.PHASE_DECODE_UNPADDED: 1739,+ mp.PHASE_UNPADDED: 1739,+ },+ )+ window = 208 * 9 # the same blocks at depth 8+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: window)+ monkeypatch.setenv(pf.PREFETCH_DEPTH_ENV, "8")+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))+ # credited 1872 x 2 // 9 = 416 MiB, not the whole 1872 MiB window+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 1024+ assert new.resident_transformer_mib == FREE_8G - 1024 - 644 - 1024 - window+++def test_a_larger_learned_pre_decode_peak_raises_the_reserve(pipe):+ _record(+ 3000,+ {+ mp.PHASE_PRE_DECODE: 2400,+ mp.PHASE_DECODE: 3000,+ mp.PHASE_DECODE_UNPADDED: 3000,+ mp.PHASE_UNPADDED: 3000,+ },+ )+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))+ # ceil(2400 x 1.15) - 416 = 2344, rounded up to 2560, against a 3584 MiB whole-render reserve (3000 x 1.15)+ assert new.estimates["measured_runtime_headroom_mib"] == 3584+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 2560+ assert new.resident_transformer_mib == FREE_8G - 1024 - 644 - 2560 - WINDOW+++def test_learned_peak_without_phases_keeps_the_whole_render_placement(pipe):+ _record(1739)+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))+ assert "measured_denoise_phase_headroom_mib" not in new.estimates+ assert new.resident_transformer_mib == _old_room()+++def test_unseeded_target_keeps_the_whole_render_placement(pipe):+ pipe._unsloth_phase_seeds = False+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))+ assert "measured_denoise_phase_headroom_mib" not in new.estimates+++def test_unmeasured_family_keeps_its_placement(pipe):+ plan = _flat_plan(FREE_8G, 8192)+ new = _refine(pipe, plan, family = "qwen-image")+ assert "measured_denoise_phase_headroom_mib" not in new.estimates+++def test_12gib_whole_transformer_is_untouched(pipe):+ new = _refine(pipe, _flat_plan(11578, 12288))+ assert new.resident_transformer_mib == 6922+ assert "measured_denoise_phase_headroom_mib" not in new.estimates+ assert dm.decode_lend_from_plan(new) is None+++def _reserved_pipe(+ phase,+ learned = None,+ cfg_off = False,+):+ reserve = (+ 2304,+ "qwen-image-2.1",+ "default",+ None,+ None,+ learned,+ None,+ None,+ True,+ cfg_off,+ False,+ False,+ False,+ phase,+ )+ return types.SimpleNamespace(+ _unsloth_measured_reserve = reserve,+ _unsloth_phase_seeds = True,+ _unsloth_peak_identity = None,+ )++[email protected]("phase", [None, 768])+def test_a_request_within_the_reserve_releases_nothing(store, phase):+ assert dm.measured_request_extra_mib(_reserved_pipe(phase), width = 1024, height = 1024) == 0+++def test_a_larger_request_first_returns_the_phase_blocks(store):+ old = dm.measured_request_extra_mib(_reserved_pipe(None), width = 2048, height = 2048)+ assert old == 8704 - 2304+ new = dm.measured_request_extra_mib(_reserved_pipe(768), width = 2048, height = 2048)+ assert new == old + (2304 - 768)+ # a reserve tuple written by an older load prices as before+ short = _reserved_pipe(None)+ short._unsloth_measured_reserve = short._unsloth_measured_reserve[:13]+ assert dm.measured_request_extra_mib(short, width = 2048, height = 2048) == old+++def test_a_conditioned_request_returns_the_phase_blocks(store):+ old = dm.measured_request_extra_mib(+ _reserved_pipe(None), width = 1024, height = 1024, condition_pixels = 1024 * 1024+ )+ new = dm.measured_request_extra_mib(+ _reserved_pipe(768), width = 1024, height = 1024, condition_pixels = 1024 * 1024+ )+ assert old > 0 and new == old + 2304 - 768+++def test_a_smaller_canvas_releases_nothing(store):+ assert dm.measured_request_extra_mib(_reserved_pipe(768), width = 512, height = 512) == 0+++def test_another_workflow_is_priced_against_the_denoise_reserve(store):+ pipe = _reserved_pipe(768)+ pipe._unsloth_peak_identity = ("qwen-image-2.1", QUANT, "default")+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, workflow = "txt2img") == 0+ assert (+ dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, workflow = "img2img")+ == 2304 - 768+ )+ pipe = _reserved_pipe(None)+ pipe._unsloth_peak_identity = ("qwen-image-2.1", QUANT, "default")+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, workflow = "img2img") == 0+++def test_the_flat_fallback_is_priced_against_the_denoise_reserve(store):+ """Placed from a CFG-off sample, a CFG request takes the flat estimate: it must clear the denoise reserve, not the+ whole-render one the phase sizing no longer holds during the steps."""+ flat = dm.estimate_image_runtime_mib(width = 1024, height = 1024, family = "qwen-image-2.1")+ old = dm.measured_request_extra_mib(+ _reserved_pipe(None, learned = 1739, cfg_off = True), width = 1024, height = 1024, cfg = True+ )+ new = dm.measured_request_extra_mib(+ _reserved_pipe(768, learned = 1739, cfg_off = True), width = 1024, height = 1024, cfg = True+ )+ assert old == max(0, flat - 2304)+ assert new == max(0, flat - 768) > old+++def _deferring_pipe(phase):+ return _reserved_pipe(phase)+++def _held(calls, name, released_mib):+ def restore():+ calls.append(name)++ restore.released_mib = released_mib+ return restore+++def test_an_oversized_request_holds_its_restore_until_a_request_needs_the_blocks(store):+ calls = []+ pipe = _deferring_pipe(768)+ assert dm.defer_resident_restore(pipe, _held(calls, "edit", 4631))+ assert calls == []+ # the next edit releases about as much: the groups stay streamed, nothing re-pinned or released+ carried, extra = dm.take_deferred_restore(pipe, 4550)+ assert carried is not None and extra == 0 and calls == []+ assert getattr(pipe, dm.DEFERRED_RESTORE_ATTR) is None+ assert dm.defer_resident_restore(pipe, carried)+ # a text-to-image request the placement was sized for pins them back before it runs+ carried, extra = dm.take_deferred_restore(pipe, 0)+ assert carried is None and extra == 0 and calls == ["edit"]+ assert dm.take_deferred_restore(pipe, 0) == (None, 0) and calls == ["edit"]+++def test_a_larger_release_carries_the_held_groups_and_releases_the_rest(store):+ calls = []+ pipe = _deferring_pipe(768)+ dm.defer_resident_restore(pipe, _held(calls, "held", 3000))+ carried, extra = dm.take_deferred_restore(pipe, 4000)+ assert carried is not None and extra == 1000 and calls == []+++def test_a_much_smaller_release_pins_the_held_groups_back_first(store):+ calls = []+ pipe = _deferring_pipe(768)+ dm.defer_resident_restore(pipe, _held(calls, "held", 4631))+ assert dm.take_deferred_restore(pipe, 1000) == (None, 1000)+ assert calls == ["held"]+++def test_held_restores_chain_and_both_run(store):+ calls = []+ pipe = _deferring_pipe(768)+ dm.defer_resident_restore(pipe, _held(calls, "a", 1000))+ dm.defer_resident_restore(pipe, _held(calls, "b", 2000))+ assert getattr(pipe, dm.DEFERRED_RESTORE_ATTR).released_mib == 3000+ dm.take_deferred_restore(pipe, 0)+ assert calls == ["a", "b"]++[email protected]("phase, env", [(None, None), (768, "0")])+def test_whole_render_placements_restore_at_once(store, monkeypatch, phase, env):+ if env is not None:+ monkeypatch.setenv(dm.DENOISE_PHASE_RESIDENCY_ENV, env)+ pipe = _deferring_pipe(phase)+ assert not dm.defer_resident_restore(pipe, _held([], "x", 4631))+ assert dm.take_deferred_restore(pipe, 4631) == (None, 4631)+++def test_a_request_counts_the_room_whole_groups_left_unused(store, monkeypatch):+ """The L4 8 GiB edit: room 4718 MiB, 24 groups kept 4631 MiB. Priced at 4550 MiB against the room, the request+ needs 4463 MiB of groups streamed out, which leaves the top-level group resident (as the whole-render placement+ did), not 4550, which reached into it."""+ mib = 1024 * 1024+ groups = [types.SimpleNamespace(_unsloth_resident_bytes = 131 * mib)] + [+ types.SimpleNamespace(_unsloth_resident_bytes = 4500 * mib // 23) for _ in range(23)+ ]+ dit = types.SimpleNamespace(_unsloth_resident_room = 4718)+ monkeypatch.setattr(dm, "_offload_groups", lambda m: groups if m is dit else [])+ pipe = _reserved_pipe(768)+ pipe.components = {"transformer": dit}+ assert dm._unused_room_mib(pipe) == 4718 - (131 + 4499)+ assert dm.take_deferred_restore(pipe, 4550) == (None, 4550 - 88)+ # the whole-render placement prices as before+ pipe = _reserved_pipe(None)+ pipe.components = {"transformer": dit}+ assert dm.take_deferred_restore(pipe, 4550) == (None, 4550)