Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: choose whole-module vs streamed offload per request from its step count (#15163)

Daniel Han90ccf1200046
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 1f031033d9f..5c57d89add7 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -188,6 +188,7 @@ from .diffusion_memory import (
ACTIVATION_TILE,
DeviceMemory,
ImageActivationShortfallError,
+ OFFLOAD_GROUP,
OFFLOAD_MODEL,
OFFLOAD_NONE,
OFFLOAD_STREAMING,
@@ -221,6 +222,10 @@ from .diffusion_memory import (
refine_memory_plan_for_components,
refine_plan_from_loaded_weights,
prefer_whole_module_offload,
+ whole_tier_request_costs,
+ pin_denoiser_for_call,
+ measured_image_runtime_mib,
+ WHOLE_TIER_ATTR,
learned_peak_label,
CFG_OFF_PART,
cfg_off_peaks_enabled,
@@ -3084,6 +3089,121 @@ def _start_peak_probe(state: Any) -> Any:
return None
+def _graphs_settled_eager(state: Any) -> bool:
+ """No CUDA graph can still record over the denoiser placement (graphs key on weight addresses)."""
+ pipe = getattr(state, "pipe", None)
+ handles = tuple(getattr(pipe, "_unsloth_cuda_graphs", ()) or ()) or tuple(
+ getattr(state, "cuda_graphs", ()) or ()
+ )
+ try:
+ if cuda_graph.live_graph_count(_peak_device_index(state)):
+ return False
+ except Exception: # noqa: BLE001 - unreadable: something may be recorded
+ return False
+ for handle in handles:
+ if getattr(handle, "poisoned", False):
+ continue
+ declined = getattr(handle, "_placement_declined", None)
+ if (
+ callable(declined)
+ and declined()
+ and getattr(handle, "fallback_handle", None) is None
+ and getattr(handle, "fallback", None) is None
+ ):
+ continue
+ return False
+ return True
+
+
+def _whole_tier_for_request(
+ state: Any,
+ *,
+ steps: int,
+ width: Optional[int],
+ height: Optional[int],
+ batch_size: int,
+ cfg: Optional[bool],
+ condition_pixels: int,
+ workflow: Optional[str] = None,
+ logger: Any = None,
+) -> Optional[tuple[float, float, int]]:
+ """``(whole s, streamed s, activation MiB)`` if this request should run the denoiser whole, else None.
+ Blocks compile below their offload hooks, so compiled code never sees the pin; a pending background compile or a
+ CUDA graph that could still record keeps the streamed plan. Without a learned pre-decode peak the call streams."""
+ pipe = getattr(state, "pipe", None)
+ if getattr(state, "offload_policy", None) != OFFLOAD_GROUP or getattr(
+ state, "transformer_cache", None
+ ):
+ return None
+ if int(condition_pixels or 0) > 0:
+ # no learned peak covers condition images
+ return None
+ try:
+ family, label, speed = _live_peak_identity(state)
+ except Exception: # noqa: BLE001
+ return None
+ tier_info = getattr(pipe, WHOLE_TIER_ATTR, None)
+ if isinstance(tier_info, dict) and speed == SPEED_OFF:
+ # remembered: a deferred profile later rewrites the load's key, and post-capture renders record under none
+ tier_info.setdefault("eager_peak_label", without_cfg_off(str(label)))
+ costs = whole_tier_request_costs(
+ pipe,
+ steps = steps,
+ width = width,
+ height = height,
+ batch_size = batch_size,
+ cfg = cfg,
+ condition_pixels = condition_pixels,
+ )
+ if costs is None or costs[0] >= costs[1]:
+ return None
+ bg = getattr(state, "bg_compile", None)
+ if bg is not None and bg.pending():
+ return None
+ if not _graphs_settled_eager(state):
+ return None
+ if speed is None:
+ return None
+ # a deferred compile tier records nothing: its eager peak (not exceeded when compiled) stands in
+ keys = [(label, speed), (label, SPEED_OFF)]
+ loaded = getattr(pipe, "_unsloth_peak_identity", None)
+ if isinstance(loaded, tuple) and len(loaded) == 3:
+ loaded_label = str(loaded[1])
+ if CFG_OFF_PART in str(label).split("|"):
+ loaded_label = f"{without_cfg_off(loaded_label)}|{CFG_OFF_PART}"
+ keys += [(loaded_label, tier) for tier in (loaded[2], SPEED_OFF) if tier is not None]
+ eager_label = tier_info.get("eager_peak_label") if isinstance(tier_info, dict) else None
+ if eager_label:
+ if CFG_OFF_PART in str(label).split("|"):
+ eager_label = f"{eager_label}|{CFG_OFF_PART}"
+ keys.append((eager_label, SPEED_OFF))
+ activation = None
+ for key_label, tier in dict.fromkeys(keys):
+ activation = measured_image_runtime_mib(
+ family,
+ tier,
+ width = width,
+ height = height,
+ batch_size = batch_size,
+ quant = with_workflow(key_label, workflow),
+ static_seed = False,
+ phase = "pre_decode",
+ )
+ if activation is not None:
+ break
+ if activation is None:
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: a %d-step render prices the denoiser whole (%.2f s vs %.2f s streamed), but no "
+ "pre-decode peak is learned for its size yet: it streams",
+ int(steps),
+ costs[0],
+ costs[1],
+ )
+ return None
+ return costs[0], costs[1], int(activation)
+
+
def _peak_device_index(state: Any) -> Optional[int]:
"""CUDA ordinal the pipeline renders on (``state.device`` may be un-indexed: the thread's current card)."""
try:
@@ -10428,7 +10548,20 @@ class DiffusionBackend:
cache = peak_cache,
graphs = peak_graphs,
)
- plan = prefer_whole_module_offload(pipe, plan, logger = logger)
+ try:
+ plan_steps, _ = default_generation_params(
+ *((fam.name,) if is_flux3_family(fam) else ()),
+ gguf_filename,
+ content_variant_hint(repo_id, gguf_filename),
+ repo_id,
+ base,
+ fam.name,
+ )
+ except Exception: # noqa: BLE001 - unpriced: the unconditional switch
+ plan_steps = None
+ plan = prefer_whole_module_offload(
+ pipe, plan, logger = logger, steps = plan_steps
+ )
headroom = plan.estimates.get("measured_runtime_headroom_mib")
if headroom and (
plan.resident_transformer_mib or plan.resident_text_encoder_mib
@@ -13751,6 +13884,7 @@ class DiffusionBackend:
)
guard_width = guard_height = guard_target = None
guard_batch = 1
+ guard_condition_pixels = 0
try:
# The dimensions the forward runs at, not the sliders: img2img / inpaint / upscale / edit take
@@ -14064,7 +14198,40 @@ class DiffusionBackend:
state.pipe._unsloth_peak_cfg = _negative_prompt_engaged(state.family, guidance)
except Exception: # noqa: BLE001 - an unknown guidance mode records nothing
state.pipe._unsloth_peak_cfg = None
- peak_probe = _start_peak_probe(state)
+ whole_tier = None
+ if _hires_pass is None and not aux_pass and not bg_eager:
+ try:
+ whole_tier = _whole_tier_for_request(
+ state,
+ steps = denoise_steps,
+ width = guard_width,
+ height = guard_height,
+ # the largest forward this request runs, not the activation guard's singleton floor
+ batch_size = max((len(chunk) for chunk in chunks), default = 1),
+ cfg = getattr(state.pipe, "_unsloth_peak_cfg", None),
+ condition_pixels = guard_condition_pixels,
+ workflow = workflow,
+ logger = logger,
+ )
+ except Exception as exc: # noqa: BLE001 - the load's streamed plan
+ logger.debug(
+ "diffusion.memory: per-request whole denoiser skipped (%s)", exc
+ )
+ whole_tier = None
+ whole_release: list = []
+
+ def _release_whole_denoiser() -> None:
+ while whole_release:
+ try:
+ whole_release.pop()()
+ except Exception as release_exc: # noqa: BLE001 - the groups keep their state
+ logger.warning(
+ "diffusion.memory: streaming the denoiser again failed (%s)",
+ release_exc,
+ )
+
+ # a whole-denoiser call's peak is not the streamed placement's
+ peak_probe = None if whole_tier is not None else _start_peak_probe(state)
decode_restores: list = []
try:
previewer = _start_previewer()
@@ -14136,6 +14303,23 @@ class DiffusionBackend:
# The prompt is encoded: the encoders are idle until the next prompt, so free them.
if state.te_releaser is not None:
state.te_releaser.release()
+ if whole_tier is not None and not whole_release:
+ # pinned after the encode, released before the decode
+ release = pin_denoiser_for_call(
+ state.pipe,
+ getattr(guard_target, "torch_device", None) or state.device,
+ activation_mib = whole_tier[2],
+ logger = logger,
+ )
+ if release is not None:
+ whole_release.append(release)
+ logger.info(
+ "diffusion.memory: this %d-step render runs the denoiser whole (priced "
+ "%.2f s vs %.2f s streamed)",
+ int(denoise_steps),
+ whole_tier[0],
+ whole_tier[1],
+ )
chunk_ticker[0] = _CompletedStepTicker(progress_steps)
@@ -14145,6 +14329,7 @@ class DiffusionBackend:
to wait on (CPU, MPS) there is no queue to be ahead of, so flip now."""
if peak_probe is not None:
peak_probe.mark_decode()
+ _release_whole_denoiser()
# encoder residency lent to the decode streams again only if free memory is short now
restore = release_for_decode(
state.pipe,
@@ -14225,6 +14410,7 @@ class DiffusionBackend:
)
continue
finally:
+ _release_whole_denoiser()
while decode_restores:
try:
decode_restores.pop()()
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 3baec144685..34e5539ee6d 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -4023,6 +4023,113 @@ _WHOLE_MODULE_KEPT_REASON = (
"whole-module offload ran the {dit} MiB denoiser whole below the streamed tiers' {floor} MiB floor; it still "
"fits, and uploading it once per call beats streaming it every step"
)
+_WHOLE_MODULE_COST_REASON = (
+ " (measured {h2d:.0f} GiB/s host-to-device and {tflops:.0f} bf16 TFLOPS: {model:.2f} s vs {group:.2f} s "
+ "streamed for a {steps}-step {side}px render)"
+)
+_WHOLE_MODULE_DECLINED_REASON = (
+ "streaming kept over whole-module offload of the {dit} MiB denoiser: measured {h2d:.0f} GiB/s host-to-device "
+ "and {tflops:.0f} bf16 TFLOPS price a {steps}-step {side}px render at {model:.2f} s whole vs {group:.2f} s streamed"
+)
+
+# ``auto``: whole-module offload only where priced cheaper; ``1`` always (#15129); ``0`` never.
+WHOLE_MODULE_OFFLOAD_ENV = "UNSLOTH_DIFFUSION_WHOLE_MODULE_OFFLOAD"
+WHOLE_MODULE_PER_REQUEST_ENV = "UNSLOTH_DIFFUSION_WHOLE_MODULE_PER_REQUEST"
+WHOLE_TIER_ATTR = "_unsloth_whole_tier"
+
+
+def per_request_whole_enabled() -> bool:
+ return not _env_off(WHOLE_MODULE_PER_REQUEST_ENV)
+
+
+# Un-overlapped whole upload vs one streamed step's transfer: 1.47x B200, 1.27-1.58x RTX PRO 6000 (klein-4B bf16).
+_WHOLE_UPLOAD_FACTOR = 1.5
+# Resident step time / probe-priced FLOPs: 1.38 B200, 0.97 RTX PRO 6000; 1.0 picks right on both, 1.35 does not.
+_DIT_COMPUTE_EFFICIENCY = 1.0
+_COST_CANVAS_PX = 1024
+# 2x2 patches of an 8x VAE latent
+_DIT_PIXELS_PER_TOKEN = 16
+
+_OFFLOAD_COST_PROBES: dict = {}
+
+
+def _offload_cost_measurements() -> Optional[tuple[float, float]]:
+ """(pinned host-to-device GiB/s, bf16 GEMM TFLOPS) of the current CUDA device, cached per device; None off CUDA.
+ Best of a few runs, so a transient stall on a shared card is not read as the card."""
+ result = None
+ key = None
+ try:
+ import torch
+
+ if not torch.cuda.is_available() or is_rocm_torch(torch):
+ return None
+ key = torch.cuda.current_device()
+ if key in _OFFLOAD_COST_PROBES:
+ return _OFFLOAD_COST_PROBES[key]
+ device = torch.device("cuda", key)
+ with torch.cuda.device(device):
+ nbytes = 256 << 20
+ host = torch.empty(nbytes, dtype = torch.uint8, pin_memory = True)
+ dev = torch.empty(nbytes, dtype = torch.uint8, device = device)
+ a = torch.randn(4096, 3072, dtype = torch.bfloat16, device = device)
+ b = torch.randn(3072, 3072, dtype = torch.bfloat16, device = device)
+ start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)
+
+ def _best_ms(fn, reps: int, inner: int) -> float:
+ fn()
+ torch.cuda.synchronize()
+ best = float("inf")
+ for _ in range(reps):
+ start.record()
+ for _ in range(inner):
+ fn()
+ end.record()
+ end.synchronize()
+ best = min(best, start.elapsed_time(end) / inner)
+ return best
+
+ copy_ms = _best_ms(lambda: dev.copy_(host, non_blocking = True), 3, 2)
+ gemm_ms = _best_ms(lambda: torch.mm(a, b), 3, 10)
+ del host, dev, a, b
+ torch.cuda.empty_cache()
+ if copy_ms > 0 and gemm_ms > 0:
+ result = (
+ (nbytes / 2**30) / (copy_ms / 1000.0),
+ 2.0 * 4096 * 3072 * 3072 / (gemm_ms / 1000.0) / 1e12,
+ )
+ except Exception: # noqa: BLE001 - unmeasurable: the caller keeps its unpriced choice
+ result = None
+ if key is not None:
+ _OFFLOAD_COST_PROBES[key] = result
+ return result
+
+
+def _whole_module_offload_costs(
+ dit_mib: int,
+ *,
+ steps: int,
+ h2d_gib_s: float,
+ bf16_tflops: float,
+ side_px: int = _COST_CANVAS_PX,
+ tokens: Optional[int] = None,
+) -> tuple[float, float]:
+ """(whole-module, streamed) denoiser seconds for one ``steps``-step render.
+
+ streamed = ``steps * max(compute, transfer)``; transfer counts the whole denoiser even when partly resident (B200
+ klein step stayed 0.145-0.170 s at 0-6751 of 7393 MiB resident). whole = one un-overlapped upload + resident steps.
+ """
+ gib = max(0, int(dit_mib)) / 1024.0
+ transfer = gib / max(float(h2d_gib_s), 1e-6)
+ params = max(0, int(dit_mib)) * 2**20 / 2.0 # bf16
+ if tokens is None:
+ tokens = (int(side_px) // _DIT_PIXELS_PER_TOKEN) ** 2
+ compute = (
+ _DIT_COMPUTE_EFFICIENCY * 2.0 * params * tokens / (max(float(bf16_tflops), 1e-6) * 1e12)
+ )
+ n = max(1, int(steps))
+ whole = _WHOLE_UPLOAD_FACTOR * transfer + n * compute
+ streamed = n * max(compute, transfer)
+ return whole, streamed
def prefer_whole_module_offload(
@@ -4030,12 +4137,15 @@ def prefer_whole_module_offload(
refined: MemoryPlan,
*,
logger: Any = None,
+ steps: Optional[int] = None,
) -> MemoryPlan:
- """``refined`` (what ``refine_plan_from_loaded_weights`` returned), or whole-module offload in its place.
-
- Below the streamed tiers' floor whole-module offload holds the whole denoiser on the card while it runs; a few
- hundred MiB more free VRAM used to switch to streaming all of it every step, but uploading it once per call beats
- that (FLUX.2-klein-4B bf16, B200: 0.34 vs 0.60 s per image). Plain bf16 only: an fp16 T4 run pointed the other way."""
+ """``refined``, or whole-module offload in its place where it beats streaming. With ``steps`` the switch is
+ priced from this card's measured bandwidth and bf16 throughput (RTX PRO 6000 streams faster at 4 steps, B200 does
+ not); unmeasurable or without ``steps`` it is kept. Plain bf16 only: an fp16 T4 run pointed the other way.
+ """
+ switch = (os.environ.get(WHOLE_MODULE_OFFLOAD_ENV) or "auto").strip().lower()
+ if switch in ("0", "off", "false", "no", "never"):
+ return refined
try:
if (
getattr(refined, "requested_mode", None) != MEMORY_MODE_AUTO
@@ -4067,6 +4177,53 @@ def prefer_whole_module_offload(
if int(getattr(refined, "resident_transformer_mib", None) or 0) >= dit:
return refined
reason = _WHOLE_MODULE_KEPT_REASON.format(dit = transformer, floor = floor)
+ priced: dict = {}
+ if (
+ steps is not None
+ and switch not in ("1", "on", "true", "yes", "always")
+ and getattr(pipe, "transformer", None) is not None
+ ):
+ measured = _offload_cost_measurements()
+ if measured is not None:
+ h2d, tflops = measured
+ whole, streamed = _whole_module_offload_costs(
+ dit, steps = int(steps), h2d_gib_s = h2d, bf16_tflops = tflops
+ )
+ fields = dict(
+ dit = dit,
+ h2d = h2d,
+ tflops = tflops,
+ model = whole,
+ group = streamed,
+ steps = int(steps),
+ side = _COST_CANVAS_PX,
+ )
+ priced = {
+ "whole_module_cost_s": round(whole, 4),
+ "streamed_cost_s": round(streamed, 4),
+ "whole_module_cost_steps": int(steps),
+ "measured_h2d_gib_s": round(h2d, 2),
+ "measured_bf16_tflops": round(tflops, 1),
+ }
+ if whole >= streamed:
+ declined = _WHOLE_MODULE_DECLINED_REASON.format(**fields)
+ if logger is not None:
+ logger.info("diffusion.memory: %s", declined)
+ if per_request_whole_enabled():
+ try:
+ setattr(
+ pipe,
+ WHOLE_TIER_ATTR,
+ {"dit_mib": int(dit), "h2d_gib_s": h2d, "bf16_tflops": tflops},
+ )
+ except Exception: # noqa: BLE001 - the load-time plan then stays for every request
+ pass
+ return replace(
+ refined,
+ reasons = refined.reasons + (declined,),
+ estimates = {**refined.estimates, **priced},
+ )
+ reason += _WHOLE_MODULE_COST_REASON.format(**fields)
if logger is not None:
logger.info("diffusion.memory: %s", reason)
return replace(
@@ -4079,7 +4236,7 @@ def prefer_whole_module_offload(
resident_transformer_mib = None,
resident_text_encoder_mib = None,
reasons = refined.reasons + (reason,),
- estimates = {**refined.estimates, "whole_module_kept_below_floor_mib": floor},
+ estimates = {**refined.estimates, "whole_module_kept_below_floor_mib": floor, **priced},
)
except Exception as exc: # noqa: BLE001 - keep the streamed refinement
if logger is not None:
@@ -4087,6 +4244,144 @@ def prefer_whole_module_offload(
return refined
+def whole_tier_request_costs(
+ pipe: Any,
+ *,
+ steps: Optional[int],
+ width: Optional[int],
+ height: Optional[int],
+ batch_size: int = 1,
+ cfg: Optional[bool] = None,
+ condition_pixels: int = 0,
+) -> Optional[tuple[float, float]]:
+ """(whole, streamed) seconds for this request's own tokens (canvas, condition images, batch, CFG); None when the
+ load did not record :data:`WHOLE_TIER_ATTR` or the per-request choice is off."""
+ tier = getattr(pipe, WHOLE_TIER_ATTR, None)
+ if not isinstance(tier, dict) or steps is None or not per_request_whole_enabled():
+ return None
+ try:
+ w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
+ h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
+ per_image = (w // _DIT_PIXELS_PER_TOKEN) * (h // _DIT_PIXELS_PER_TOKEN) + max(
+ 0, int(condition_pixels or 0)
+ ) // (_DIT_PIXELS_PER_TOKEN**2)
+ tokens = per_image * max(1, int(batch_size or 1)) * (2 if cfg else 1)
+ return _whole_module_offload_costs(
+ int(tier["dit_mib"]),
+ steps = int(steps),
+ h2d_gib_s = float(tier["h2d_gib_s"]),
+ bf16_tflops = float(tier["bf16_tflops"]),
+ tokens = tokens,
+ )
+ except Exception: # noqa: BLE001 - unpriced: the load's streamed plan
+ return None
+
+
+_MISSING = object()
+
+
+def pin_denoiser_for_call(
+ pipe: Any,
+ device: Any,
+ *,
+ activation_mib: int,
+ logger: Any = None,
+) -> Optional[Callable[[], None]]:
+ """Pin every streamed offload group of ``pipe.transformer`` for one call; returns the release, or None.
+
+ The hooks stay and blocks compile below them, so compiled code never sees the pin. Pinned only if the groups plus
+ ``activation_mib`` fit free memory now (VAE reserve kept); all or nothing."""
+ transformer = getattr(pipe, "transformer", None)
+ try:
+ import torch
+
+ groups = _offload_groups(transformer) if transformer is not None else []
+ if not groups or any(getattr(g, "offload_to_disk_path", None) for g in groups):
+ return None
+ streamed = [g for g in groups if not getattr(g, "_unsloth_resident", False)]
+ if not streamed:
+ return None
+ onload = torch.device(device if device is not None else "cuda")
+ if onload.type != "cuda":
+ return None
+ if onload.index is None:
+ onload = torch.device("cuda", torch.cuda.current_device())
+ unique: dict = {}
+ for group in streamed:
+ for t in _resident_tensors(group):
+ unique.setdefault(id(t), t)
+ need = sum(sum(_storage_nbytes(t)) for t in unique.values())
+ free, total = (int(x) for x in torch.cuda.mem_get_info(onload))
+ free -= user_budget_reserve_bytes(onload, total)
+ cached = max(
+ 0,
+ int(torch.cuda.memory_reserved(onload))
+ - int(torch.cuda.memory_allocated(onload))
+ - _graph_held_cache_bytes(onload.index),
+ )
+ room = (free + cached) >> 20
+ if (need >> 20) + max(0, int(activation_mib)) > room:
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: the denoiser keeps streaming: running it whole needs %d MiB + %d MiB of "
+ "activations, %d MiB free",
+ need >> 20,
+ int(activation_mib),
+ room,
+ )
+ return None
+ every: dict = {}
+ for group in groups:
+ for t in _resident_tensors(group):
+ every.setdefault(id(t), t)
+ whole_mib = (sum(sum(_storage_nbytes(t)) for t in every.values()) >> 20) + 2
+ saved = {
+ name: transformer.__dict__.get(name, _MISSING)
+ for name in ("_unsloth_resident_room", "_unsloth_resident_device")
+ }
+ _keep_groups_resident(transformer, whole_mib, onload, logger)
+ pinned = [g for g in streamed if getattr(g, "_unsloth_resident", False)]
+ done = [False]
+
+ def release() -> None:
+ if done[0]:
+ return
+ done[0] = True
+ state = getattr(transformer, "_unsloth_stream_state", None)
+ if isinstance(state, dict):
+ state["streamed"] = (
+ 1 # a release that fails part way still waits on the copy stream
+ )
+ try:
+ for group in reversed(pinned):
+ if getattr(group, "_unsloth_resident", False):
+ _release_group(group)
+ _release_group_range(transformer, group)
+ finally:
+ if isinstance(state, dict):
+ state["streamed"] = _streamed_group_count(groups)
+ for name, value in saved.items():
+ try:
+ if value is _MISSING:
+ transformer.__dict__.pop(name, None)
+ else:
+ setattr(transformer, name, value)
+ except Exception: # noqa: BLE001
+ pass
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ synchronize_outside_capture(onload)
+
+ if len(pinned) != len(streamed):
+ release()
+ return None
+ return release
+ except Exception as exc: # noqa: BLE001 - streaming every group is the safe state
+ if logger is not None:
+ logger.warning("diffusion.memory: whole denoiser for this call skipped (%s)", exc)
+ return None
+
+
def _refine_plan_from_loaded_weights(
pipe: Any,
plan: MemoryPlan,
diff --git a/studio/backend/tests/test_diffusion_offload_tier_per_request.py b/studio/backend/tests/test_diffusion_offload_tier_per_request.py
new file mode 100644
index 00000000000..2f75d5169c1
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_offload_tier_per_request.py
@@ -0,0 +1,362 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Whole-module vs streamed offload chosen per request."""
+
+from __future__ import annotations
+
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+import core.inference.diffusion as dif
+import core.inference.diffusion_memory as dm
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_diffusion_placement_monotonic import ( # noqa: E402, F401 - the autouse env fixture
+ _card_probe,
+ _clean_env,
+ _klein_above_floor,
+ _priced,
+)
+
+KLEIN_DIT_MIB = 7393
+
+
[email protected](autouse = True)
+def _per_request_env(monkeypatch):
+ monkeypatch.delenv(dm.WHOLE_MODULE_OFFLOAD_ENV, raising = False)
+ monkeypatch.delenv(dm.WHOLE_MODULE_PER_REQUEST_ENV, raising = False)
+
+
+def _tier_pipe(card):
+ h2d, tflops = _card_probe(card)
+ pipe = types.SimpleNamespace()
+ setattr(
+ pipe,
+ dm.WHOLE_TIER_ATTR,
+ {"dit_mib": KLEIN_DIT_MIB, "h2d_gib_s": h2d, "bf16_tflops": tflops},
+ )
+ return pipe
+
+
+def _whole(pipe, steps, **kw):
+ costs = dm.whole_tier_request_costs(
+ pipe, steps = steps, width = kw.pop("width", 1024), height = kw.pop("height", 1024), **kw
+ )
+ return None if costs is None else costs[0] < costs[1]
+
+
+def test_streamed_load_records_the_probe_for_requests(monkeypatch):
+ """RTX PRO 6000 at 4 steps streams and keeps the probe; a card that priced whole keeps the #15129 plan."""
+ flat = _klein_above_floor(monkeypatch)
+ refined, plan = _priced(monkeypatch, 12544, flat, steps = 4, card = "rtx_pro_6000")
+ assert plan.offload_policy == dm.OFFLOAD_GROUP
+ import torch
+
+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = torch.bfloat16))
+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: _card_probe("rtx_pro_6000"))
+ dm.prefer_whole_module_offload(pipe, refined, steps = 4)
+ tier = getattr(pipe, dm.WHOLE_TIER_ATTR)
+ assert tier["dit_mib"] == KLEIN_DIT_MIB
+ assert (tier["h2d_gib_s"], tier["bf16_tflops"]) == _card_probe("rtx_pro_6000")
+
+ for card in ("b200", "a100"):
+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = torch.bfloat16))
+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda card = card: _card_probe(card))
+ assert (
+ dm.prefer_whole_module_offload(pipe, refined, steps = 4).offload_policy
+ == dm.OFFLOAD_MODEL
+ )
+ assert not hasattr(pipe, dm.WHOLE_TIER_ATTR)
+
+ monkeypatch.setenv(dm.WHOLE_MODULE_PER_REQUEST_ENV, "0")
+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = torch.bfloat16))
+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: _card_probe("rtx_pro_6000"))
+ assert dm.prefer_whole_module_offload(pipe, refined, steps = 4).offload_policy == dm.OFFLOAD_GROUP
+ assert not hasattr(pipe, dm.WHOLE_TIER_ATTR)
+
+
+def test_request_step_count_picks_the_tier():
+ """The measured RTX PRO 6000 calls: streamed at 2 and 4 steps, whole at 8."""
+ pipe = _tier_pipe("rtx_pro_6000")
+ assert [_whole(pipe, n) for n in (1, 2, 4, 8, 16)] == [False, False, False, True, True]
+
+
+def test_request_size_moves_the_crossover():
+ """More tokens per step (canvas, batch, CFG) let streaming win to more steps."""
+ pipe = _tier_pipe("rtx_pro_6000")
+ assert _whole(pipe, 8) is True
+ assert _whole(pipe, 8, batch_size = 2) is False
+ assert _whole(pipe, 8, cfg = True) is False
+ assert _whole(pipe, 8, width = 1536, height = 1536) is False
+ assert _whole(pipe, 4, width = 512, height = 512) is True
+
+
+def test_no_tier_or_kill_switch_prices_nothing(monkeypatch):
+ assert (
+ dm.whole_tier_request_costs(types.SimpleNamespace(), steps = 8, width = 1024, height = 1024)
+ is None
+ )
+ pipe = _tier_pipe("rtx_pro_6000")
+ assert dm.whole_tier_request_costs(pipe, steps = None, width = 1024, height = 1024) is None
+ monkeypatch.setenv(dm.WHOLE_MODULE_PER_REQUEST_ENV, "0")
+ assert dm.whole_tier_request_costs(pipe, steps = 8, width = 1024, height = 1024) is None
+
+
+class _Bg:
+ def __init__(self, pending):
+ self._pending = pending
+
+ def pending(self):
+ return self._pending
+
+
+class _Graph:
+ def __init__(
+ self,
+ declined,
+ poisoned = False,
+ ):
+ self._declined = declined
+ self.poisoned = poisoned
+ self.fallback_handle = None
+ self.fallback = None
+
+ def _placement_declined(self):
+ return self._declined
+
+
+def _state(**kw):
+ pipe = _tier_pipe("rtx_pro_6000")
+ pipe._unsloth_cuda_graphs = kw.pop("graphs", ())
+ base = dict(
+ pipe = pipe,
+ offload_policy = dm.OFFLOAD_GROUP,
+ transformer_cache = None,
+ bg_compile = None,
+ cuda_graphs = (),
+ device = "cuda:0",
+ )
+ base.update(kw)
+ return types.SimpleNamespace(**base)
+
+
+def _request(state, steps = 8):
+ return dif._whole_tier_for_request(
+ state, steps = steps, width = 1024, height = 1024, batch_size = 1, cfg = False, condition_pixels = 0
+ )
+
+
+LIVE = ("flux.2-klein", "dense:bfloat16|attn:_native_cudnn", "default")
+LOADED = ("flux.2-klein", "dense:bfloat16|attn:native", "default")
+RECORDED = ("dense:bfloat16|attn:native", "off") # the eager renders' key
+
+
+def _learned(monkeypatch):
+ seen = []
+
+ def fake_peak(family, speed, **kw):
+ seen.append((kw.get("quant"), speed, kw.get("phase")))
+ return 2560 if (kw.get("quant"), speed) == RECORDED else None
+
+ monkeypatch.setattr(dif, "_live_peak_identity", lambda state: LIVE)
+ monkeypatch.setattr(dif, "measured_image_runtime_mib", fake_peak)
+ monkeypatch.setattr(dif.cuda_graph, "live_graph_count", lambda index: 0)
+ monkeypatch.setattr(dif, "_peak_device_index", lambda state: 0)
+ return seen
+
+
+def test_request_gate_prices_and_reads_the_eager_peak(_learned):
+ """A deferred compile tier records no peak: the eager renders' pre-decode peak sizes the pinned call."""
+ state = _state()
+ state.pipe._unsloth_peak_identity = LOADED
+ whole, streamed, activation = _request(state)
+ assert whole < streamed and activation == 2560
+ assert _learned == [
+ (LIVE[1], "default", "pre_decode"),
+ (LIVE[1], "off", "pre_decode"),
+ (LOADED[1], "default", "pre_decode"),
+ (LOADED[1], "off", "pre_decode"),
+ ]
+ assert _request(state, steps = 4) is None
+
+
+def test_request_gate_reads_the_live_key_first(_learned, monkeypatch):
+ monkeypatch.setattr(dif, "_live_peak_identity", lambda state: ("flux.2-klein", *RECORDED))
+ state = _state()
+ state.pipe._unsloth_peak_identity = LOADED
+ assert _request(state)[2] == 2560
+ assert _learned == [(*RECORDED, "pre_decode")]
+
+
+ "kw",
+ [
+ dict(offload_policy = dm.OFFLOAD_MODEL),
+ dict(transformer_cache = "fbcache"),
+ dict(bg_compile = _Bg(True)),
+ dict(graphs = (_Graph(declined = False),)),
+ ],
+)
+def test_request_gate_keeps_the_load_plan(_learned, kw):
+ assert _request(_state(**kw)) is None
+
+
+def test_request_gate_after_graphs_settle(_learned, monkeypatch):
+ def state(**kw):
+ out = _state(**kw)
+ out.pipe._unsloth_peak_identity = LOADED
+ return out
+
+ assert _request(state(bg_compile = _Bg(False), graphs = (_Graph(declined = True),))) is not None
+ assert _request(state(graphs = (_Graph(declined = False, poisoned = True),))) is not None
+ monkeypatch.setattr(dif.cuda_graph, "live_graph_count", lambda index: 1)
+ assert _request(state(graphs = (_Graph(declined = True),))) is None
+
+
+def test_request_gate_without_a_learned_peak_streams(_learned, monkeypatch):
+ monkeypatch.setattr(dif, "measured_image_runtime_mib", lambda *a, **k: None)
+ assert _request(_state()) is None
+
+
+def test_request_gate_declines_condition_images(_learned):
+ """Condition-image requests stream."""
+ state = _state()
+ state.pipe._unsloth_peak_identity = LOADED
+ assert (
+ dif._whole_tier_for_request(
+ state,
+ steps = 8,
+ width = 1024,
+ height = 1024,
+ batch_size = 1,
+ cfg = False,
+ condition_pixels = 1 << 20,
+ )
+ is None
+ )
+ assert _learned == []
+
+
+def test_request_gate_reads_the_workflow_peak(_learned):
+ """An img2img request is sized from the peak recorded under its own workflow, never the text-to-image one."""
+ state = _state()
+ state.pipe._unsloth_peak_identity = LOADED
+ out = dif._whole_tier_for_request(
+ state,
+ steps = 8,
+ width = 1024,
+ height = 1024,
+ batch_size = 1,
+ cfg = False,
+ condition_pixels = 0,
+ workflow = "img2img",
+ )
+ assert out is None # only the text-to-image key holds a peak in the fixture
+ assert _learned and all(q.endswith("|wf:img2img") for q, _, _ in _learned)
+
+
+def test_the_request_gate_prices_the_largest_chunk():
+ """The tier is priced on the forward the request runs, not the activation guard's singleton floor."""
+ import inspect
+
+ src = inspect.getsource(dif)
+ call = src.split("whole_tier = _whole_tier_for_request(", 1)[1].split(")\n", 1)[0]
+ assert "batch_size = max((len(chunk) for chunk in chunks), default = 1)" in call
+ assert "guard_batch" not in call
+
+
+def _cuda_net():
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("CUDA-only: diffusers group offloading with a copy stream")
+ pytest.importorskip("diffusers.hooks")
+ from diffusers.hooks import apply_group_offloading
+
+ class Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(64, 1024)
+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(1024, 1024) for _ in range(6))
+ self.proj_out = torch.nn.Linear(1024, 64)
+
+ def forward(self, x):
+ x = self.proj_in(x)
+ for block in self.blocks:
+ x = torch.nn.functional.gelu(block(x))
+ return self.proj_out(x)
+
+ torch.manual_seed(0)
+ net = Net()
+ x = torch.randn(3, 64)
+ ref = net.to("cuda")(x.cuda()).cpu()
+ net.to("cpu")
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ record_stream = True,
+ non_blocking = True,
+ )
+ return torch, net, x, ref
+
+
+def test_pin_for_call_is_bit_identical_and_streams_again():
+ """Every group resident for the call, same output, then the exact pre-call placement (same host tensors)."""
+ torch, net, x, ref = _cuda_net()
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ hosts = {id(p): p.data for p in net.blocks.parameters()}
+ release = dm.pin_denoiser_for_call(pipe, "cuda", activation_mib = 0)
+ assert release is not None
+ assert all(p.device.type == "cuda" for p in net.parameters())
+ assert dm.denoisers_pinned_resident(pipe)
+ for _ in range(2):
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ release()
+ release() # idempotent
+ assert all(p.device.type == "cpu" for p in net.blocks.parameters())
+ assert all(p.data.data_ptr() == hosts[id(p)].data_ptr() for p in net.blocks.parameters())
+ assert not getattr(net, "_unsloth_resident_room", None)
+ assert not any(getattr(g, "_unsloth_resident", False) for g in dm._offload_groups(net))
+ assert dm.resident_group_mib(pipe) == 0
+ for _ in range(2):
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+
+
+def test_pin_for_call_needs_the_room(monkeypatch):
+ torch, net, x, ref = _cuda_net()
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ torch.cuda.empty_cache() # the room counts the allocator's reusable cache
+ free, total = torch.cuda.mem_get_info()
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1 << 20, total))
+ assert dm.pin_denoiser_for_call(pipe, "cuda", activation_mib = 0) is None
+ assert all(p.device.type == "cpu" for p in net.blocks.parameters())
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (free, total))
+ assert dm.pin_denoiser_for_call(pipe, "cuda", activation_mib = 1 << 30) is None
+ release = dm.pin_denoiser_for_call(pipe, "cuda", activation_mib = 64)
+ assert release is not None
+ release()
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+
+
+def test_request_gate_keeps_the_eager_key_once_the_profile_rewrites_the_load_key(
+ _learned, monkeypatch
+):
+ """The deferred profile rewrites the load's key and nothing records after capture: the eager key is remembered."""
+ live = {"id": ("flux.2-klein", *RECORDED)}
+ monkeypatch.setattr(dif, "_live_peak_identity", lambda state: live["id"])
+ state = _state()
+ state.pipe._unsloth_peak_identity = ("flux.2-klein", *RECORDED)
+ assert _request(state, steps = 2) is None # streamed, but the eager key is kept
+ live["id"] = LIVE
+ state.pipe._unsloth_peak_identity = LIVE
+ _learned.clear()
+ assert _request(state)[2] == 2560
+ assert _learned[-1] == (*RECORDED, "pre_decode")
diff --git a/studio/backend/tests/test_diffusion_placement_monotonic.py b/studio/backend/tests/test_diffusion_placement_monotonic.py
index 43a5951e903..87cc32ac055 100644
--- a/studio/backend/tests/test_diffusion_placement_monotonic.py
+++ b/studio/backend/tests/test_diffusion_placement_monotonic.py
@@ -379,3 +379,182 @@ def test_unmeasured_denoisers_keep_the_streamed_plan(monkeypatch, family, quant)
plan = _plan(12544, 24 * GIB, flat, family = family, torchao = torchao, compute = compute)
assert plan.offload_policy == dm.OFFLOAD_GROUP and plan.stream_transformer
assert "whole_module_kept_below_floor_mib" not in plan.estimates
+
+
+# (GiB/s, TFLOPS) probes reproducing measured klein-4B bf16 1024^2 per-step transfer / compute.
+KLEIN_DIT_MIB = 7393
+CARDS = {
+ "b200": (0.142, 0.0296),
+ "a100": (0.541, 0.150),
+ "rtx_pro_6000": (0.1415, 0.0985),
+}
+
+
+def _card_probe(card):
+ transfer, compute = CARDS[card]
+ flops = 2.0 * KLEIN_DIT_MIB * 2**20 / 2.0 * (1024 // 16) ** 2
+ return (KLEIN_DIT_MIB / 1024.0 / transfer, dm._DIT_COMPUTE_EFFICIENCY * flops / compute / 1e12)
+
+
+def _klein_above_floor(monkeypatch, *, compute = None):
+ q8, dense, encoders, vae = FAMILIES["flux.2-klein"]
+ te = sum(encoders.values())
+ loaded = {
+ "transformer": (KLEIN_DIT_MIB, "dit"),
+ "text_encoder": (te, "text_encoder"),
+ "vae": (vae, "other"),
+ }
+ _patch_weights(monkeypatch, loaded, torchao = False, compute = compute)
+ flat = dict(
+ model_dense_mib = KLEIN_DIT_MIB + te + vae,
+ companion_dense_mib = te + vae,
+ text_encoder_dense_mib = te,
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(
+ width = None, height = None, family = "flux.2-klein"
+ ),
+ )
+ return flat
+
+
+def _priced(
+ monkeypatch,
+ free_mib,
+ flat,
+ *,
+ steps,
+ card,
+ compute = None,
+):
+ import torch
+
+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: _card_probe(card))
+ plan = dm.plan_diffusion_memory(
+ target = types.SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = _discrete(free_mib, 24 * GIB),
+ **flat,
+ )
+ dtype = torch.float16 if compute == 2 else torch.bfloat16
+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = dtype))
+ refined = dm.refine_plan_from_loaded_weights(
+ pipe, plan, family = "flux.2-klein", speed_mode = "default"
+ )
+ return refined, dm.prefer_whole_module_offload(pipe, refined, steps = steps)
+
+
+ "card, steps, whole",
+ [
+ ("b200", 2, True),
+ ("b200", 4, True),
+ ("b200", 8, True),
+ ("a100", 4, True),
+ ("a100", 8, True),
+ ("rtx_pro_6000", 2, False),
+ ("rtx_pro_6000", 4, False),
+ ("rtx_pro_6000", 8, True),
+ ],
+)
+def test_whole_module_offload_is_kept_only_where_it_is_cheaper(monkeypatch, card, steps, whole):
+ """B200 0.336 vs 0.595 s and A100 2.336 vs 3.429 s at 4 steps (whole-module faster); RTX PRO 6000 0.762 vs 0.743 s
+ at 4 steps (streamed faster), 1.022 vs 1.181 s at 8, 0.561 vs 0.458 s at 2."""
+ flat = _klein_above_floor(monkeypatch)
+ refined, plan = _priced(monkeypatch, 12544, flat, steps = steps, card = card)
+ assert refined.offload_policy == dm.OFFLOAD_GROUP and refined.stream_transformer
+ model_s, group_s = plan.estimates["whole_module_cost_s"], plan.estimates["streamed_cost_s"]
+ assert (model_s < group_s) is whole
+ if whole:
+ assert plan.offload_policy == dm.OFFLOAD_MODEL
+ assert plan.estimates["whole_module_kept_below_floor_mib"] == 10400
+ else:
+ assert plan.offload_policy == dm.OFFLOAD_GROUP
+ assert "whole_module_kept_below_floor_mib" not in plan.estimates
+ assert replace_fields(plan) == replace_fields(refined)
+ assert any("streaming kept over whole-module offload" in r for r in plan.reasons)
+
+
+def replace_fields(plan):
+ return dm.replace(plan, reasons = (), estimates = {})
+
+
+def test_cost_model_fits_the_measured_crossovers():
+ """Per-card crossover of whole-module vs streamed (steps where they cost the same): RTX PRO 6000 measured 4.2-4.35."""
+
+ def crossover(card):
+ h2d, tflops = _card_probe(card)
+ (w1, s1), (w2, _) = (
+ dm._whole_module_offload_costs(
+ KLEIN_DIT_MIB, steps = n, h2d_gib_s = h2d, bf16_tflops = tflops
+ )
+ for n in (1, 2)
+ )
+ compute = w2 - w1
+ return (w1 - compute) / (s1 - compute)
+
+ assert crossover("b200") <= 2.0
+ assert crossover("a100") <= 4.0
+ assert 4.0 < crossover("rtx_pro_6000") < 5.0
+
+
+def test_unpriced_or_kill_switch_keep_their_meaning(monkeypatch):
+ """No steps or no measurement: the unconditional switch (#15129). ``1`` always switches, ``0`` never does."""
+ flat = _klein_above_floor(monkeypatch)
+ _, unpriced = _priced(monkeypatch, 12544, flat, steps = None, card = "rtx_pro_6000")
+ assert unpriced.offload_policy == dm.OFFLOAD_MODEL
+ assert "whole_module_cost_s" not in unpriced.estimates
+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: None)
+ q8, dense, encoders, vae = FAMILIES["flux.2-klein"]
+ import torch
+
+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = torch.bfloat16))
+ plan = dm.plan_diffusion_memory(
+ target = types.SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = _discrete(12544, 24 * GIB),
+ **flat,
+ )
+ refined = dm.refine_plan_from_loaded_weights(
+ pipe, plan, family = "flux.2-klein", speed_mode = "default"
+ )
+ assert dm.prefer_whole_module_offload(pipe, refined, steps = 4).offload_policy == dm.OFFLOAD_MODEL
+ monkeypatch.setenv(dm.WHOLE_MODULE_OFFLOAD_ENV, "1")
+ _, forced = _priced(monkeypatch, 12544, flat, steps = 4, card = "rtx_pro_6000")
+ assert (
+ forced.offload_policy == dm.OFFLOAD_MODEL and "whole_module_cost_s" not in forced.estimates
+ )
+ monkeypatch.setenv(dm.WHOLE_MODULE_OFFLOAD_ENV, "0")
+ refined, off = _priced(monkeypatch, 12544, flat, steps = 4, card = "b200")
+ assert off is refined
+
+
[email protected]("card", sorted(CARDS))
[email protected]("steps", (2, 4, 8))
+def test_plans_outside_the_window_are_unchanged_by_pricing(monkeypatch, card, steps):
+ """Below the floor, at the resident tiers and on an fp16 denoiser the priced call returns the same plan."""
+ for compute in (None, 2):
+ flat = _klein_above_floor(monkeypatch, compute = compute)
+ for free in range(4 * GIB, 24 * GIB + 1, STEP_MIB):
+ refined, priced = _priced(
+ monkeypatch, free, flat, steps = steps, card = card, compute = compute
+ )
+ import torch
+
+ dtype = torch.float16 if compute == 2 else torch.bfloat16
+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = dtype))
+ unpriced = dm.prefer_whole_module_offload(pipe, refined)
+ if "whole_module_kept_below_floor_mib" not in unpriced.estimates:
+ assert priced == unpriced == refined, (free, compute)
+
+
+ "probe, decisions",
+ [
+ # real probe readings: B200, and RTX PRO 6000 (measured streamed at 2 and 4 steps, whole at 8)
+ ((51.5, 1490.0), {2: True, 4: True, 8: True}),
+ ((52.97, 309.0), {2: False, 4: False, 8: True}),
+ ],
+)
+def test_real_probe_readings_make_the_measured_call(probe, decisions):
+ for steps, whole in decisions.items():
+ model_s, group_s = dm._whole_module_offload_costs(
+ 7438, steps = steps, h2d_gib_s = probe[0], bf16_tflops = probe[1]
+ )
+ assert (model_s < group_s) is whole, (probe, steps, model_s, group_s)