Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: keep a streamed denoiser's blocks resident for the denoise phase, not the whole render (A100 8 GiB Qwen-Image-2.1 -18%, FLUX.3 -21 to -24%) (#17207)

Daniel Han0bfc894fed07
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 66830614a0d..dbd52741476 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -267,6 +267,9 @@ from .diffusion_memory import (
applied_plan,
release_resident_groups,
release_for_decode,
+ chained_restore,
+ defer_resident_restore,
+ take_deferred_restore,
resident_group_mib,
hook_resident_denoiser,
measured_request_extra_mib,
@@ -11128,6 +11131,8 @@ class DiffusionBackend:
plan.estimates.get("measured_dense_bf16", False),
plan.estimates.get("measured_small_host_int8", False),
bool(streamed_headroom),
+ # MEASURED_RESERVE_DENOISE_PHASE_INDEX: the blocks sized from the denoise phase
+ plan.estimates.get("measured_denoise_phase_headroom_mib"),
)
setattr(pipe, DECODE_LEND_ATTR, decode_lend_from_plan(plan))
@@ -14944,6 +14949,8 @@ class DiffusionBackend:
extra_mib = _hold_captured_release(state, extra_mib)
guard_release_mib = extra_mib
guard_capture_gen = cuda_graph.capture_generation(_peak_device_index(state))
+ # groups an earlier oversized request streamed out stay out when this one streams them all too
+ carried_restore, extra_mib = take_deferred_restore(state.pipe, extra_mib)
if extra_mib > 0:
releasable_mib = resident_group_mib(state.pipe)
if (
@@ -14953,6 +14960,12 @@ class DiffusionBackend:
# transformer so this request can stream part of it.
hook_resident_denoiser(state.pipe, guard_target.torch_device, logger)
restore_resident = release_resident_groups(state.pipe, extra_mib, logger)
+ if carried_restore is not None:
+ restore_resident = (
+ carried_restore
+ if restore_resident is None
+ else chained_restore(carried_restore, restore_resident)
+ )
guard_kwargs = dict(
# Not the settled snapshot: its empty_cache() is wrong per generation (re-mallocs activations).
device_memory = reclaimable_snapshot_device_memory(guard_target),
@@ -15619,7 +15632,8 @@ class DiffusionBackend:
end_cfg_batch_render(getattr(state, "pipe", None))
if restore_resident is not None:
try:
- restore_resident()
+ if kick_bg or not defer_resident_restore(state.pipe, restore_resident):
+ restore_resident() # a background compile kicked below needs it back first
except Exception as exc: # noqa: BLE001 - the groups keep streaming
logger.warning("diffusion.memory: re-residency failed: %s", exc)
if kick_bg and bg is not None:
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index a1348c25fc5..f3bbdd688e8 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -4124,6 +4124,111 @@ def _streamed_dit_room_mib(memory: Any, headroom_mib: int, other_mib: int, windo
)
+# Size a streamed denoiser's resident blocks from the denoise phase (encode/decode release around it); "0" = whole-render peak.
+DENOISE_PHASE_RESIDENCY_ENV = "UNSLOTH_DIFFUSION_DENOISE_PHASE_RESIDENCY"
+_PHASE_DENOISE_COLUMN = 1
+_PHASE_UNTILED_DECODE_COLUMN = 2
+_PHASE_TILED_DECODE_COLUMN = 3
+
+
+def denoise_phase_residency_enabled() -> bool:
+ return not _env_off(DENOISE_PHASE_RESIDENCY_ENV)
+
+
+def _top_level_group_mib(pipe: Any) -> Optional[int]:
+ """MiB of the largest denoiser's weights outside its block stacks (``ModuleList`` / ``Sequential`` children):
+ block-level group offload keeps them in one top-level group that is onloaded for the whole forward when it is not
+ resident, beside the block window. None when unreadable."""
+ try:
+ import torch
+
+ largest = 0
+ for name, module in (getattr(pipe, "components", {}) or {}).items():
+ if str(name) not in ("transformer", "transformer_2", "unconditional_transformer"):
+ continue
+ if not isinstance(module, torch.nn.Module):
+ continue
+ blocks: set[int] = set()
+ for child in module.children():
+ if isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):
+ blocks.update(id(t) for t in (*child.parameters(), *child.buffers()))
+ seen: set[int] = set()
+ nbytes = 0
+ for t in (*module.parameters(), *module.buffers()):
+ if id(t) in seen or id(t) in blocks:
+ continue
+ seen.add(id(t))
+ nbytes += sum(_storage_nbytes(t))
+ largest = max(largest, -(-nbytes // (1024 * 1024)))
+ return largest
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _denoise_phase_reserve(
+ pipe: Any,
+ *,
+ family: Optional[str],
+ speed_mode: Optional[str],
+ quant: Optional[str],
+ static_seed: bool,
+ window_mib: int,
+ tiled: bool,
+) -> Optional[tuple[int, tuple[int, int, bool]]]:
+ """``(denoise MiB, (decode peak MiB, bucket pixels, quadratic))`` for a 1024x1024 render, or None.
+
+ The denoise MiB is what the steps allocate above the resident weights besides the prefetch window (margin
+ included): the larger of the static denoise seed (``_MEASURED_IMAGE_ACTIVATION_MIB``, window excluded) and this
+ build's learned pre-decode peak less the part of the window it is sure to hold (learned peaks are on the streamed
+ scale, ``stream_window_pad_mib``, at the prefetch depth they were recorded with), so no window is charged twice. The decode peak (no margin) is the larger of the
+ static decode seed for the VAE savers in force and the learned unpadded decode phase. None when neither source
+ covers both phases, when a learned whole-render peak has no consistent phase split (its phases are unknown), or
+ when the switch is off."""
+ if not denoise_phase_residency_enabled() or _env_off(MEASURED_ACTIVATION_ENV):
+ return None
+ try:
+ from .diffusion_measured_peaks import (
+ PHASE_DECODE_UNPADDED,
+ PHASE_PRE_DECODE,
+ lookup_bucket,
+ )
+
+ name = str(family or "").lower()
+ denoise = decode = None
+ pixels, quadratic = DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT, False
+ phases = _MEASURED_IMAGE_ACTIVATION_MIB.get(name)
+ if (
+ static_seed
+ and phases is not None
+ and bool(getattr(pipe, "_unsloth_phase_seeds", False))
+ and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES
+ ):
+ row = phases[1 if speed_mode == "max" else 0]
+ denoise = int(math.ceil(row[_PHASE_DENOISE_COLUMN] * _MEASURED_PEAK_MARGIN))
+ decode = int(row[_PHASE_TILED_DECODE_COLUMN if tiled else _PHASE_UNTILED_DECODE_COLUMN])
+ if quant is not None:
+ whole = lookup_bucket(family = family, quant = quant, speed_mode = speed_mode)
+ if whole is not None:
+ split = learned_phase_split(family, quant, speed_mode)
+ if split is None:
+ return None
+ from .diffusion_offload_prefetch import prefetch_depth
+
+ # the sample holds the window of the depth it was recorded at, which the key does not name: credit
+ # only the smallest one (depth 1, two blocks), so a later deeper prefetch is never double-credited
+ held = max(0, int(window_mib)) * 2 // (prefetch_depth() + 1)
+ learned_pre = int(math.ceil(split[PHASE_PRE_DECODE] * _MEASURED_PEAK_MARGIN)) - held
+ denoise = max(int(denoise or 0), learned_pre, 0)
+ decode = max(int(decode or 0), int(split[PHASE_DECODE_UNPADDED]))
+ pixels, quadratic = int(split["pixels"]), bool(split["quadratic"])
+ if denoise is None or not decode:
+ return None
+ step = _MEASURED_PEAK_ROUND_MIB
+ return int(-(-int(denoise) // step) * step), (int(decode), int(pixels), bool(quadratic))
+ except Exception: # noqa: BLE001 - unreadable: the whole-render placement
+ return None
+
+
def _denoiser_compute_bytes(pipe: Any) -> Optional[int]:
"""Denoiser compute element size for the dense table: 2 (fp16), 4 (fp32); None for bf16 / unreadable."""
try:
@@ -4789,6 +4894,34 @@ def _refine_plan_from_loaded_weights(
streamed_room = min(
_streamed_dit_room_mib(memory, streamed_headroom, other, window), dit
)
+ phase_reserve = None
+ if window >= 0:
+ phase_reserve = _denoise_phase_reserve(
+ pipe,
+ family = family,
+ speed_mode = speed_mode,
+ quant = read_quant,
+ static_seed = static_seed and not small_host_int8,
+ window_mib = window,
+ tiled = "tiled" in planned_vae_saver_tag(pipe, plan),
+ )
+ top = _top_level_group_mib(pipe) if phase_reserve is not None else None
+ if top is None:
+ phase_reserve = None
+ if phase_reserve is not None:
+ phase_room = min(
+ _streamed_dit_room_mib(memory, phase_reserve[0], other, window), dit
+ )
+ if (
+ phase_reserve[0] >= streamed_headroom
+ or phase_room <= streamed_room
+ # the top-level group goes resident first; one that misses is onloaded whole for every
+ # forward beside the block window, which the denoise seed (measured resident) omits
+ or phase_room < top
+ ):
+ phase_reserve = None
+ else:
+ streamed_room = phase_room
if streamed_room > dit_room:
encode_room = int(dit_room)
dit_room, room, wide_dit = streamed_room, streamed_room, True
@@ -4797,6 +4930,16 @@ def _refine_plan_from_loaded_weights(
estimates["measured_streamed_headroom_mib"] = streamed_headroom
# seed covers denoise only; flat encode room on top overran at 16 GiB
encode_room = 0
+ if phase_reserve is not None:
+ decode_peak, decode_pixels, decode_quadratic = phase_reserve[1]
+ # the encode keeps the flat room, the decode streams the shortfall again
+ estimates.update(
+ measured_denoise_phase_headroom_mib = phase_reserve[0],
+ decode_lend_transformer_mib = int(streamed_room),
+ measured_decode_peak_mib = decode_peak,
+ measured_decode_peak_pixels = decode_pixels,
+ measured_decode_peak_quadratic = decode_quadratic,
+ )
te_room = (
min(max(0, room - dit_room), encoders) if stream_te and policy == OFFLOAD_GROUP else 0
)
@@ -4863,8 +5006,14 @@ def _refine_plan_from_loaded_weights(
f" {encode_room} MiB of it while the encoders run)"
if whole_dit
else f" (free memory less a {_resident_dit_slack_mib(memory)} MiB slack and a "
- f"{estimates.get('stream_window_mib')} MiB prefetch window; {encode_room} MiB of it while the "
- "encoders run)"
+ f"{estimates.get('stream_window_mib')} MiB prefetch window"
+ + (
+ f", sized from the {estimates['measured_denoise_phase_headroom_mib']} MiB denoise phase"
+ " (the decode may stream part of it again)"
+ if estimates.get("measured_denoise_phase_headroom_mib") is not None
+ else ""
+ )
+ + f"; {encode_room} MiB of it while the encoders run)"
if wide_dit
else ""
),
@@ -5233,7 +5382,7 @@ def release_resident_groups(
(int(need_mib) * 1024 * 1024 - max(left, 0)) >> 20,
reason,
)
-
+ restore.released_mib = (int(need_mib) * 1024 * 1024 - left) >> 20 # take_deferred_restore
return restore
except Exception as exc: # noqa: BLE001 - the guard still refuses what cannot fit
if logger is not None:
@@ -5242,17 +5391,102 @@ def release_resident_groups(
return restore if released else None
+# Restoring after every oversized request costs a host-to-device round trip per render; wait for one that needs them.
+DEFERRED_RESTORE_ATTR = "_unsloth_deferred_resident_restore"
+
+
+def _phase_placement(pipe: Any) -> bool:
+ reserve = getattr(pipe, "_unsloth_measured_reserve", None) or ()
+ return bool(
+ len(reserve) > MEASURED_RESERVE_DENOISE_PHASE_INDEX
+ and reserve[MEASURED_RESERVE_DENOISE_PHASE_INDEX]
+ and denoise_phase_residency_enabled()
+ )
+
+
+def chained_restore(first: Callable[[], None], second: Callable[[], None]) -> Callable[[], None]:
+ def restore() -> None:
+ try:
+ first()
+ finally:
+ second()
+
+ restore.released_mib = int(getattr(first, "released_mib", 0) or 0) + int(
+ getattr(second, "released_mib", 0) or 0
+ )
+ return restore
+
+
+def defer_resident_restore(pipe: Any, restore: Optional[Callable[[], None]]) -> bool:
+ """Hold ``restore`` (an oversized request's ``release_resident_groups``) on ``pipe`` instead of running it, for a
+ placement sized from the denoise phase. True when held; False: the caller restores now."""
+ if restore is None or not _phase_placement(pipe):
+ return False
+ try:
+ held = getattr(pipe, DEFERRED_RESTORE_ATTR, None)
+ setattr(
+ pipe, DEFERRED_RESTORE_ATTR, restore if held is None else chained_restore(held, restore)
+ )
+ except Exception: # noqa: BLE001 - unheld: restore now
+ return False
+ return True
+
+
+# A held restore carries on while the next request's own release is at least this close to what it streamed out.
+_DEFER_TOLERANCE_MIB = 256
+
+
+def _unused_room_mib(pipe: Any) -> int:
+ """MiB of the placement's resident rooms its whole groups left unused (free memory a request may count on)."""
+ room = placed = 0
+ for module in (getattr(pipe, "components", {}) or {}).values():
+ if not getattr(module, "_unsloth_resident_room", None):
+ continue
+ room += int(module._unsloth_resident_room)
+ placed += sum(
+ int(getattr(g, "_unsloth_resident_bytes", 0) or 0)
+ for g in (_offload_groups(module) or ())
+ )
+ return max(0, room - (placed >> 20))
+
+
+def take_deferred_restore(pipe: Any, extra_mib: int) -> tuple[Optional[Callable[[], None]], int]:
+ """At a request's guard: ``(restore to carry on, MiB the request must still release)``. For a placement sized from
+ the denoise phase the request's MiB (priced against its room) first counts the room its whole groups left unused.
+ A request releasing about as much as a held restore streamed out (or more) keeps those groups streamed and
+ releases only the rest; any other request (one the placement was sized for, or a much smaller release) pins them
+ back first."""
+ extra = int(extra_mib or 0)
+ if extra > 0 and _phase_placement(pipe):
+ extra = max(0, extra - _unused_room_mib(pipe))
+ held = getattr(pipe, DEFERRED_RESTORE_ATTR, None)
+ if held is None:
+ return None, extra
+ try:
+ setattr(pipe, DEFERRED_RESTORE_ATTR, None)
+ except Exception: # noqa: BLE001
+ pass
+ streamed = int(getattr(held, "released_mib", 0) or 0)
+ if extra > 0 and extra + _DEFER_TOLERANCE_MIB >= streamed:
+ return held, max(0, extra - streamed)
+ held()
+ return None, extra
+
+
DECODE_LEND_ATTR = "_unsloth_decode_lend"
# Kept free beside the decode's learned peak x margin when deciding whether lent encoder groups must stream again.
_DECODE_SLACK_MIB = 256
def decode_lend_from_plan(plan: Any) -> Optional[tuple[int, int, int, bool]]:
- """``(lent MiB, decode peak MiB, bucket pixels, quadratic)`` when ``plan`` lent encoder residency to the decode."""
+ """``(lent MiB, decode peak MiB, bucket pixels, quadratic)`` when ``plan`` lent encoder residency, or denoiser
+ residency sized from the denoise phase, to the decode."""
estimates = getattr(plan, "estimates", None) or {}
lend = estimates.get("decode_lend_text_encoder_mib")
if not lend or not getattr(plan, "resident_text_encoder_mib", None):
- return None
+ lend = estimates.get("decode_lend_transformer_mib")
+ if not lend or not getattr(plan, "resident_transformer_mib", None):
+ return None
return (
int(lend),
int(estimates.get("measured_decode_peak_mib") or 0),
@@ -5813,6 +6047,9 @@ def install_encode_release(
return 0
+MEASURED_RESERVE_DENOISE_PHASE_INDEX = 13
+
+
def measured_request_extra_mib(
pipe: Any,
*,
@@ -5831,10 +6068,21 @@ def measured_request_extra_mib(
whether the request runs classifier-free guidance (None: unknown, treated as running it). ``unpriced_references``:
the request carries reference images whose latents join the denoiser's sequence but are not in
``condition_pixels`` (FLUX.2 Klein's primary reference): a placement made from a dense default / max tier seed,
- measured on text-to-image, then sizes the request from the flat estimate."""
+ measured on text-to-image, then sizes the request from the flat estimate.
+
+ A placement that sized its resident blocks from the denoise phase (``MEASURED_RESERVE_DENOISE_PHASE_INDEX``) holds
+ only that phase's reserve while the steps run: a request it was sized for (text-to-image, at most the 1024x1024
+ bucket, no condition images or tier peak, within the whole-render reserve) releases nothing, its encode and decode
+ priced by their own hooks; any other request is priced against the denoise reserve, which first streams the blocks
+ the phase sizing added."""
reserve = getattr(pipe, "_unsloth_measured_reserve", None)
if not reserve:
return 0
+ phase = (
+ reserve[MEASURED_RESERVE_DENOISE_PHASE_INDEX]
+ if len(reserve) > MEASURED_RESERVE_DENOISE_PHASE_INDEX
+ else None
+ )
headroom, family, speed_mode = reserve[:3]
dense_mib = reserve[3] if len(reserve) > 3 else None
compute_bytes = reserve[4] if len(reserve) > 4 and reserve[4] else 2
@@ -5885,7 +6133,7 @@ def measured_request_extra_mib(
family = family,
condition_pixels = condition_pixels,
)
- return max(0, int(flat) - int(headroom))
+ return max(0, int(flat) - int(phase or headroom))
if learned:
# the learned peak the load reserved, scaled here (not reread from the store, which another worker may evict
# or a failed write may leave empty while this pipeline stays loaded)
@@ -5926,9 +6174,11 @@ def measured_request_extra_mib(
)
if need is not None:
need += allowance
+ sized_for = True # the request is the one the placement was sized for (``phase`` pricing)
if isinstance(identity, tuple) and len(identity) == 3:
keyed = with_workflow(identity[1], workflow)
if keyed != identity[1]:
+ sized_for = False # another workflow: its denoise phase was never measured against the placement
own = None
if (
cfg is False
@@ -5946,6 +6196,20 @@ def measured_request_extra_mib(
need = own if need is None else max(need, own)
if need is None:
return 0
+ if phase:
+ pixels = (
+ max(64, int(width or DEFAULT_IMAGE_WIDTH))
+ * max(64, int(height or DEFAULT_IMAGE_HEIGHT))
+ * max(1, int(batch_size or 1))
+ )
+ sized_for = (
+ sized_for
+ and allowance == 0
+ and tier_peak is None
+ and pixels <= DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT
+ and int(need) <= int(headroom)
+ )
+ return 0 if sized_for else max(0, int(need) - int(phase))
return max(0, int(need) - int(headroom))
diff --git a/studio/backend/tests/test_diffusion_denoise_phase_residency.py b/studio/backend/tests/test_diffusion_denoise_phase_residency.py
new file mode 100644
index 00000000000..10de6fd4f0e
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_denoise_phase_residency.py
@@ -0,0 +1,405 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A partially streamed denoiser sizes its resident blocks from the denoise phase; CPU only.
+
+Numbers from a Qwen-Image-2.1 int8 load on an A100 under an 8 GiB budget: 7482 MiB free, 6922 MiB DiT, 644 MiB VAE,
+625 MiB prefetch window, tiled + sliced VAE."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+import core.inference.diffusion_measured_peaks as mp
+import core.inference.diffusion_memory as dm
+
+LOADED = {
+ "transformer": (6922, "dit"),
+ "text_encoder": (8917, "text_encoder"),
+ "vae": (644, "other"),
+}
+WINDOW = 625
+FREE_8G = 7482
+QUANT = "torchao:Int8Tensor"
+
+
+def _memory(budget_mib: int, total_mib: int) -> "dm.DeviceMemory":
+ reserve = dm._reserve_mib("discrete_vram", total_mib)
+ return dm.DeviceMemory("cuda", "cuda", "discrete_vram", budget_mib + reserve, total_mib)
+
+
+def _flat_plan(free_mib: int, total_mib: int):
+ reserve = dm._reserve_mib("discrete_vram", total_mib)
+ plan = dm.plan_diffusion_memory(
+ target = types.SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = _memory(free_mib - reserve, total_mib),
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(
+ width = None, height = None, family = "qwen-image-2.1"
+ ),
+ requested_mode = None,
+ model_dense_mib = 19630,
+ companion_dense_mib = 12182,
+ text_encoder_dense_mib = 10847,
+ )
+ if plan.offload_policy == dm.OFFLOAD_MODEL:
+ plan = dm.torchao_streaming_plan(plan)
+ return plan
+
+
+def store(tmp_path, monkeypatch):
+ path = tmp_path / "peaks.json"
+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(path))
+ for env in (
+ mp.LEARNED_PEAKS_ENV,
+ mp.PHASE_PEAKS_ENV,
+ dm.MEASURED_ACTIVATION_ENV,
+ dm.PARTIAL_RESIDENT_ENV,
+ dm.DENOISE_PHASE_RESIDENCY_ENV,
+ dm.STREAMED_RESIDENCY_ENV,
+ dm.RESIDENT_DIT_ENV,
+ ):
+ monkeypatch.delenv(env, raising = False)
+ mp._cache.update(path = None, mtime = None, data = None)
+ monkeypatch.setattr(mp, "current_device_name", lambda: "NVIDIA A100-SXM4-40GB")
+ monkeypatch.setattr(mp, "device_name", lambda dev: "NVIDIA A100-SXM4-40GB")
+ return path
+
+
+def pipe(monkeypatch, store):
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(LOADED))
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ monkeypatch.setattr(dm, "denoiser_quant_label", lambda pipe: QUANT)
+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: WINDOW)
+ monkeypatch.setattr(dm, "planned_vae_saver_tag", lambda pipe, plan: "tiled+sliced")
+ return types.SimpleNamespace(_unsloth_phase_seeds = True)
+
+
+def _refine(
+ pipe,
+ plan,
+ family = "qwen-image-2.1",
+ speed = "default",
+):
+ return dm.refine_plan_from_loaded_weights(
+ pipe, plan, family = family, speed_mode = speed, attention = None
+ )
+
+
+def _record(peak, phases = None):
+ mp.record(
+ family = "qwen-image-2.1",
+ quant = QUANT,
+ speed_mode = "default",
+ width = 1024,
+ height = 1024,
+ batch_size = 1,
+ peak_mib = peak,
+ device_name = "NVIDIA A100-SXM4-40GB",
+ phases = phases or {},
+ )
+
+
+def _old_room() -> int:
+ """The whole-render placement: the 1849 MiB seed x margin (2304) instead of the denoise phase."""
+ return FREE_8G - 1024 - 644 - 2304 - WINDOW
+
+
+def test_8gib_sizes_resident_blocks_from_the_denoise_phase(pipe):
+ plan = _flat_plan(FREE_8G, 8192)
+ assert plan.offload_policy == dm.OFFLOAD_STREAMING
+ new = _refine(pipe, plan)
+ # denoise seed 666 MiB x 1.15, rounded up to 256 MiB; the window is charged once beside it
+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 768
+ assert new.resident_transformer_mib == FREE_8G - 1024 - 644 - 768 - WINDOW == 4421
+ assert new.resident_transformer_mib > _old_room() == 2885
+ # the whole-render reserve is unchanged (request pricing, encode room)
+ assert new.estimates["measured_runtime_headroom_mib"] == 2304
+ assert (
+ new.estimates["encode_resident_transformer_mib"]
+ == plan.estimates["safe_device_budget_mib"] - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644
+ )
+ # the decode borrows the blocks back: tiled decode seed, 1 MP bucket, linear
+ assert dm.decode_lend_from_plan(new) == (4421, 1697, 1024 * 1024, False)
+ assert "denoise phase" in new.reasons[-1]
+
+
[email protected]("top, resident", [(3000, 4421), (4421, 4421), (4422, None)])
+def test_a_streamed_top_level_group_is_charged_beside_the_window(pipe, monkeypatch, top, resident):
+ """The top-level group goes resident first; one larger than the room is onloaded whole every forward."""
+ monkeypatch.setattr(dm, "_top_level_group_mib", lambda pipe: top)
+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))
+ if resident is None:
+ assert "measured_denoise_phase_headroom_mib" not in new.estimates
+ assert new.resident_transformer_mib == _old_room()
+ else:
+ assert new.resident_transformer_mib == resident
+
+
+def test_kill_switch_restores_the_whole_render_placement(pipe, monkeypatch):
+ monkeypatch.setenv(dm.DENOISE_PHASE_RESIDENCY_ENV, "0")
+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))
+ assert new.resident_transformer_mib == _old_room()
+ assert "measured_denoise_phase_headroom_mib" not in new.estimates
+ assert "decode_lend_transformer_mib" not in new.estimates
+ assert dm.decode_lend_from_plan(new) is None
+
+
+def test_untiled_decode_reads_the_untiled_seed(pipe, monkeypatch):
+ monkeypatch.setattr(dm, "planned_vae_saver_tag", lambda pipe, plan: "full")
+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))
+ assert dm.decode_lend_from_plan(new)[1] == 7648
+
+
+def test_learned_phases_charge_the_window_once(pipe):
+ # what the 8 GiB render recorded: pre-decode 1189 (window included), decode 1739
+ _record(
+ 1739,
+ {
+ mp.PHASE_PRE_DECODE: 1189,
+ mp.PHASE_DECODE: 1739,
+ mp.PHASE_DECODE_UNPADDED: 1739,
+ mp.PHASE_UNPADDED: 1739,
+ },
+ )
+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))
+ # 1189 x 1.15 = 1368 less the depth-1 window the sample surely holds (2 of 3 blocks: 416) = 952 -> 1024
+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 1024
+ assert new.resident_transformer_mib == FREE_8G - 1024 - 644 - 1024 - WINDOW == 4165
+ assert dm.decode_lend_from_plan(new) == (4165, 1739, 1024 * 1024, False)
+
+
+def test_a_deeper_prefetch_never_credits_more_window_than_the_sample_held(pipe, monkeypatch):
+ """A sample recorded at depth 2 read under depth 8: only the depth-1 share of the current window is credited."""
+ import core.inference.diffusion_offload_prefetch as pf
+
+ _record(
+ 1739,
+ {
+ mp.PHASE_PRE_DECODE: 1189,
+ mp.PHASE_DECODE: 1739,
+ mp.PHASE_DECODE_UNPADDED: 1739,
+ mp.PHASE_UNPADDED: 1739,
+ },
+ )
+ window = 208 * 9 # the same blocks at depth 8
+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: window)
+ monkeypatch.setenv(pf.PREFETCH_DEPTH_ENV, "8")
+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))
+ # credited 1872 x 2 // 9 = 416 MiB, not the whole 1872 MiB window
+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 1024
+ assert new.resident_transformer_mib == FREE_8G - 1024 - 644 - 1024 - window
+
+
+def test_a_larger_learned_pre_decode_peak_raises_the_reserve(pipe):
+ _record(
+ 3000,
+ {
+ mp.PHASE_PRE_DECODE: 2400,
+ mp.PHASE_DECODE: 3000,
+ mp.PHASE_DECODE_UNPADDED: 3000,
+ mp.PHASE_UNPADDED: 3000,
+ },
+ )
+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))
+ # ceil(2400 x 1.15) - 416 = 2344, rounded up to 2560, against a 3584 MiB whole-render reserve (3000 x 1.15)
+ assert new.estimates["measured_runtime_headroom_mib"] == 3584
+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 2560
+ assert new.resident_transformer_mib == FREE_8G - 1024 - 644 - 2560 - WINDOW
+
+
+def test_learned_peak_without_phases_keeps_the_whole_render_placement(pipe):
+ _record(1739)
+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))
+ assert "measured_denoise_phase_headroom_mib" not in new.estimates
+ assert new.resident_transformer_mib == _old_room()
+
+
+def test_unseeded_target_keeps_the_whole_render_placement(pipe):
+ pipe._unsloth_phase_seeds = False
+ new = _refine(pipe, _flat_plan(FREE_8G, 8192))
+ assert "measured_denoise_phase_headroom_mib" not in new.estimates
+
+
+def test_unmeasured_family_keeps_its_placement(pipe):
+ plan = _flat_plan(FREE_8G, 8192)
+ new = _refine(pipe, plan, family = "qwen-image")
+ assert "measured_denoise_phase_headroom_mib" not in new.estimates
+
+
+def test_12gib_whole_transformer_is_untouched(pipe):
+ new = _refine(pipe, _flat_plan(11578, 12288))
+ assert new.resident_transformer_mib == 6922
+ assert "measured_denoise_phase_headroom_mib" not in new.estimates
+ assert dm.decode_lend_from_plan(new) is None
+
+
+def _reserved_pipe(
+ phase,
+ learned = None,
+ cfg_off = False,
+):
+ reserve = (
+ 2304,
+ "qwen-image-2.1",
+ "default",
+ None,
+ None,
+ learned,
+ None,
+ None,
+ True,
+ cfg_off,
+ False,
+ False,
+ False,
+ phase,
+ )
+ return types.SimpleNamespace(
+ _unsloth_measured_reserve = reserve,
+ _unsloth_phase_seeds = True,
+ _unsloth_peak_identity = None,
+ )
+
+
[email protected]("phase", [None, 768])
+def test_a_request_within_the_reserve_releases_nothing(store, phase):
+ assert dm.measured_request_extra_mib(_reserved_pipe(phase), width = 1024, height = 1024) == 0
+
+
+def test_a_larger_request_first_returns_the_phase_blocks(store):
+ old = dm.measured_request_extra_mib(_reserved_pipe(None), width = 2048, height = 2048)
+ assert old == 8704 - 2304
+ new = dm.measured_request_extra_mib(_reserved_pipe(768), width = 2048, height = 2048)
+ assert new == old + (2304 - 768)
+ # a reserve tuple written by an older load prices as before
+ short = _reserved_pipe(None)
+ short._unsloth_measured_reserve = short._unsloth_measured_reserve[:13]
+ assert dm.measured_request_extra_mib(short, width = 2048, height = 2048) == old
+
+
+def test_a_conditioned_request_returns_the_phase_blocks(store):
+ old = dm.measured_request_extra_mib(
+ _reserved_pipe(None), width = 1024, height = 1024, condition_pixels = 1024 * 1024
+ )
+ new = dm.measured_request_extra_mib(
+ _reserved_pipe(768), width = 1024, height = 1024, condition_pixels = 1024 * 1024
+ )
+ assert old > 0 and new == old + 2304 - 768
+
+
+def test_a_smaller_canvas_releases_nothing(store):
+ assert dm.measured_request_extra_mib(_reserved_pipe(768), width = 512, height = 512) == 0
+
+
+def test_another_workflow_is_priced_against_the_denoise_reserve(store):
+ pipe = _reserved_pipe(768)
+ pipe._unsloth_peak_identity = ("qwen-image-2.1", QUANT, "default")
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, workflow = "txt2img") == 0
+ assert (
+ dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, workflow = "img2img")
+ == 2304 - 768
+ )
+ pipe = _reserved_pipe(None)
+ pipe._unsloth_peak_identity = ("qwen-image-2.1", QUANT, "default")
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, workflow = "img2img") == 0
+
+
+def test_the_flat_fallback_is_priced_against_the_denoise_reserve(store):
+ """Placed from a CFG-off sample, a CFG request takes the flat estimate: it must clear the denoise reserve, not the
+ whole-render one the phase sizing no longer holds during the steps."""
+ flat = dm.estimate_image_runtime_mib(width = 1024, height = 1024, family = "qwen-image-2.1")
+ old = dm.measured_request_extra_mib(
+ _reserved_pipe(None, learned = 1739, cfg_off = True), width = 1024, height = 1024, cfg = True
+ )
+ new = dm.measured_request_extra_mib(
+ _reserved_pipe(768, learned = 1739, cfg_off = True), width = 1024, height = 1024, cfg = True
+ )
+ assert old == max(0, flat - 2304)
+ assert new == max(0, flat - 768) > old
+
+
+def _deferring_pipe(phase):
+ return _reserved_pipe(phase)
+
+
+def _held(calls, name, released_mib):
+ def restore():
+ calls.append(name)
+
+ restore.released_mib = released_mib
+ return restore
+
+
+def test_an_oversized_request_holds_its_restore_until_a_request_needs_the_blocks(store):
+ calls = []
+ pipe = _deferring_pipe(768)
+ assert dm.defer_resident_restore(pipe, _held(calls, "edit", 4631))
+ assert calls == []
+ # the next edit releases about as much: the groups stay streamed, nothing re-pinned or released
+ carried, extra = dm.take_deferred_restore(pipe, 4550)
+ assert carried is not None and extra == 0 and calls == []
+ assert getattr(pipe, dm.DEFERRED_RESTORE_ATTR) is None
+ assert dm.defer_resident_restore(pipe, carried)
+ # a text-to-image request the placement was sized for pins them back before it runs
+ carried, extra = dm.take_deferred_restore(pipe, 0)
+ assert carried is None and extra == 0 and calls == ["edit"]
+ assert dm.take_deferred_restore(pipe, 0) == (None, 0) and calls == ["edit"]
+
+
+def test_a_larger_release_carries_the_held_groups_and_releases_the_rest(store):
+ calls = []
+ pipe = _deferring_pipe(768)
+ dm.defer_resident_restore(pipe, _held(calls, "held", 3000))
+ carried, extra = dm.take_deferred_restore(pipe, 4000)
+ assert carried is not None and extra == 1000 and calls == []
+
+
+def test_a_much_smaller_release_pins_the_held_groups_back_first(store):
+ calls = []
+ pipe = _deferring_pipe(768)
+ dm.defer_resident_restore(pipe, _held(calls, "held", 4631))
+ assert dm.take_deferred_restore(pipe, 1000) == (None, 1000)
+ assert calls == ["held"]
+
+
+def test_held_restores_chain_and_both_run(store):
+ calls = []
+ pipe = _deferring_pipe(768)
+ dm.defer_resident_restore(pipe, _held(calls, "a", 1000))
+ dm.defer_resident_restore(pipe, _held(calls, "b", 2000))
+ assert getattr(pipe, dm.DEFERRED_RESTORE_ATTR).released_mib == 3000
+ dm.take_deferred_restore(pipe, 0)
+ assert calls == ["a", "b"]
+
+
[email protected]("phase, env", [(None, None), (768, "0")])
+def test_whole_render_placements_restore_at_once(store, monkeypatch, phase, env):
+ if env is not None:
+ monkeypatch.setenv(dm.DENOISE_PHASE_RESIDENCY_ENV, env)
+ pipe = _deferring_pipe(phase)
+ assert not dm.defer_resident_restore(pipe, _held([], "x", 4631))
+ assert dm.take_deferred_restore(pipe, 4631) == (None, 4631)
+
+
+def test_a_request_counts_the_room_whole_groups_left_unused(store, monkeypatch):
+ """The L4 8 GiB edit: room 4718 MiB, 24 groups kept 4631 MiB. Priced at 4550 MiB against the room, the request
+ needs 4463 MiB of groups streamed out, which leaves the top-level group resident (as the whole-render placement
+ did), not 4550, which reached into it."""
+ mib = 1024 * 1024
+ groups = [types.SimpleNamespace(_unsloth_resident_bytes = 131 * mib)] + [
+ types.SimpleNamespace(_unsloth_resident_bytes = 4500 * mib // 23) for _ in range(23)
+ ]
+ dit = types.SimpleNamespace(_unsloth_resident_room = 4718)
+ monkeypatch.setattr(dm, "_offload_groups", lambda m: groups if m is dit else [])
+ pipe = _reserved_pipe(768)
+ pipe.components = {"transformer": dit}
+ assert dm._unused_room_mib(pipe) == 4718 - (131 + 4499)
+ assert dm.take_deferred_restore(pipe, 4550) == (None, 4550 - 88)
+ # the whole-render placement prices as before
+ pipe = _reserved_pipe(None)
+ pipe.components = {"transformer": dit}
+ assert dm.take_deferred_restore(pipe, 4550) == (None, 4550)