Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: choose whole-module vs streamed offload per request from its step count (#15163)
Daniel Han90ccf1200046
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 1f031033d9f..5c57d89add7 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -188,6 +188,7 @@ from .diffusion_memory import (ACTIVATION_TILE,DeviceMemory,ImageActivationShortfallError,+ OFFLOAD_GROUP,OFFLOAD_MODEL,OFFLOAD_NONE,OFFLOAD_STREAMING,@@ -221,6 +222,10 @@ from .diffusion_memory import (refine_memory_plan_for_components,refine_plan_from_loaded_weights,prefer_whole_module_offload,+ whole_tier_request_costs,+ pin_denoiser_for_call,+ measured_image_runtime_mib,+ WHOLE_TIER_ATTR,learned_peak_label,CFG_OFF_PART,cfg_off_peaks_enabled,@@ -3084,6 +3089,121 @@ def _start_peak_probe(state: Any) -> Any:return None+def _graphs_settled_eager(state: Any) -> bool:+ """No CUDA graph can still record over the denoiser placement (graphs key on weight addresses)."""+ pipe = getattr(state, "pipe", None)+ handles = tuple(getattr(pipe, "_unsloth_cuda_graphs", ()) or ()) or tuple(+ getattr(state, "cuda_graphs", ()) or ()+ )+ try:+ if cuda_graph.live_graph_count(_peak_device_index(state)):+ return False+ except Exception: # noqa: BLE001 - unreadable: something may be recorded+ return False+ for handle in handles:+ if getattr(handle, "poisoned", False):+ continue+ declined = getattr(handle, "_placement_declined", None)+ if (+ callable(declined)+ and declined()+ and getattr(handle, "fallback_handle", None) is None+ and getattr(handle, "fallback", None) is None+ ):+ continue+ return False+ return True+++def _whole_tier_for_request(+ state: Any,+ *,+ steps: int,+ width: Optional[int],+ height: Optional[int],+ batch_size: int,+ cfg: Optional[bool],+ condition_pixels: int,+ workflow: Optional[str] = None,+ logger: Any = None,+) -> Optional[tuple[float, float, int]]:+ """``(whole s, streamed s, activation MiB)`` if this request should run the denoiser whole, else None.+ Blocks compile below their offload hooks, so compiled code never sees the pin; a pending background compile or a+ CUDA graph that could still record keeps the streamed plan. Without a learned pre-decode peak the call streams."""+ pipe = getattr(state, "pipe", None)+ if getattr(state, "offload_policy", None) != OFFLOAD_GROUP or getattr(+ state, "transformer_cache", None+ ):+ return None+ if int(condition_pixels or 0) > 0:+ # no learned peak covers condition images+ return None+ try:+ family, label, speed = _live_peak_identity(state)+ except Exception: # noqa: BLE001+ return None+ tier_info = getattr(pipe, WHOLE_TIER_ATTR, None)+ if isinstance(tier_info, dict) and speed == SPEED_OFF:+ # remembered: a deferred profile later rewrites the load's key, and post-capture renders record under none+ tier_info.setdefault("eager_peak_label", without_cfg_off(str(label)))+ costs = whole_tier_request_costs(+ pipe,+ steps = steps,+ width = width,+ height = height,+ batch_size = batch_size,+ cfg = cfg,+ condition_pixels = condition_pixels,+ )+ if costs is None or costs[0] >= costs[1]:+ return None+ bg = getattr(state, "bg_compile", None)+ if bg is not None and bg.pending():+ return None+ if not _graphs_settled_eager(state):+ return None+ if speed is None:+ return None+ # a deferred compile tier records nothing: its eager peak (not exceeded when compiled) stands in+ keys = [(label, speed), (label, SPEED_OFF)]+ loaded = getattr(pipe, "_unsloth_peak_identity", None)+ if isinstance(loaded, tuple) and len(loaded) == 3:+ loaded_label = str(loaded[1])+ if CFG_OFF_PART in str(label).split("|"):+ loaded_label = f"{without_cfg_off(loaded_label)}|{CFG_OFF_PART}"+ keys += [(loaded_label, tier) for tier in (loaded[2], SPEED_OFF) if tier is not None]+ eager_label = tier_info.get("eager_peak_label") if isinstance(tier_info, dict) else None+ if eager_label:+ if CFG_OFF_PART in str(label).split("|"):+ eager_label = f"{eager_label}|{CFG_OFF_PART}"+ keys.append((eager_label, SPEED_OFF))+ activation = None+ for key_label, tier in dict.fromkeys(keys):+ activation = measured_image_runtime_mib(+ family,+ tier,+ width = width,+ height = height,+ batch_size = batch_size,+ quant = with_workflow(key_label, workflow),+ static_seed = False,+ phase = "pre_decode",+ )+ if activation is not None:+ break+ if activation is None:+ if logger is not None:+ logger.info(+ "diffusion.memory: a %d-step render prices the denoiser whole (%.2f s vs %.2f s streamed), but no "+ "pre-decode peak is learned for its size yet: it streams",+ int(steps),+ costs[0],+ costs[1],+ )+ return None+ return costs[0], costs[1], int(activation)++def _peak_device_index(state: Any) -> Optional[int]:"""CUDA ordinal the pipeline renders on (``state.device`` may be un-indexed: the thread's current card)."""try:@@ -10428,7 +10548,20 @@ class DiffusionBackend:cache = peak_cache,graphs = peak_graphs,)- plan = prefer_whole_module_offload(pipe, plan, logger = logger)+ try:+ plan_steps, _ = default_generation_params(+ *((fam.name,) if is_flux3_family(fam) else ()),+ gguf_filename,+ content_variant_hint(repo_id, gguf_filename),+ repo_id,+ base,+ fam.name,+ )+ except Exception: # noqa: BLE001 - unpriced: the unconditional switch+ plan_steps = None+ plan = prefer_whole_module_offload(+ pipe, plan, logger = logger, steps = plan_steps+ )headroom = plan.estimates.get("measured_runtime_headroom_mib")if headroom and (plan.resident_transformer_mib or plan.resident_text_encoder_mib@@ -13751,6 +13884,7 @@ class DiffusionBackend:)guard_width = guard_height = guard_target = Noneguard_batch = 1+ guard_condition_pixels = 0try:# The dimensions the forward runs at, not the sliders: img2img / inpaint / upscale / edit take@@ -14064,7 +14198,40 @@ class DiffusionBackend:state.pipe._unsloth_peak_cfg = _negative_prompt_engaged(state.family, guidance)except Exception: # noqa: BLE001 - an unknown guidance mode records nothingstate.pipe._unsloth_peak_cfg = None- peak_probe = _start_peak_probe(state)+ whole_tier = None+ if _hires_pass is None and not aux_pass and not bg_eager:+ try:+ whole_tier = _whole_tier_for_request(+ state,+ steps = denoise_steps,+ width = guard_width,+ height = guard_height,+ # the largest forward this request runs, not the activation guard's singleton floor+ batch_size = max((len(chunk) for chunk in chunks), default = 1),+ cfg = getattr(state.pipe, "_unsloth_peak_cfg", None),+ condition_pixels = guard_condition_pixels,+ workflow = workflow,+ logger = logger,+ )+ except Exception as exc: # noqa: BLE001 - the load's streamed plan+ logger.debug(+ "diffusion.memory: per-request whole denoiser skipped (%s)", exc+ )+ whole_tier = None+ whole_release: list = []++ def _release_whole_denoiser() -> None:+ while whole_release:+ try:+ whole_release.pop()()+ except Exception as release_exc: # noqa: BLE001 - the groups keep their state+ logger.warning(+ "diffusion.memory: streaming the denoiser again failed (%s)",+ release_exc,+ )++ # a whole-denoiser call's peak is not the streamed placement's+ peak_probe = None if whole_tier is not None else _start_peak_probe(state)decode_restores: list = []try:previewer = _start_previewer()@@ -14136,6 +14303,23 @@ class DiffusionBackend:# The prompt is encoded: the encoders are idle until the next prompt, so free them.if state.te_releaser is not None:state.te_releaser.release()+ if whole_tier is not None and not whole_release:+ # pinned after the encode, released before the decode+ release = pin_denoiser_for_call(+ state.pipe,+ getattr(guard_target, "torch_device", None) or state.device,+ activation_mib = whole_tier[2],+ logger = logger,+ )+ if release is not None:+ whole_release.append(release)+ logger.info(+ "diffusion.memory: this %d-step render runs the denoiser whole (priced "+ "%.2f s vs %.2f s streamed)",+ int(denoise_steps),+ whole_tier[0],+ whole_tier[1],+ )chunk_ticker[0] = _CompletedStepTicker(progress_steps)@@ -14145,6 +14329,7 @@ class DiffusionBackend:to wait on (CPU, MPS) there is no queue to be ahead of, so flip now."""if peak_probe is not None:peak_probe.mark_decode()+ _release_whole_denoiser()# encoder residency lent to the decode streams again only if free memory is short nowrestore = release_for_decode(state.pipe,@@ -14225,6 +14410,7 @@ class DiffusionBackend:)continuefinally:+ _release_whole_denoiser()while decode_restores:try:decode_restores.pop()()diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 3baec144685..34e5539ee6d 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -4023,6 +4023,113 @@ _WHOLE_MODULE_KEPT_REASON = ("whole-module offload ran the {dit} MiB denoiser whole below the streamed tiers' {floor} MiB floor; it still ""fits, and uploading it once per call beats streaming it every step")+_WHOLE_MODULE_COST_REASON = (+ " (measured {h2d:.0f} GiB/s host-to-device and {tflops:.0f} bf16 TFLOPS: {model:.2f} s vs {group:.2f} s "+ "streamed for a {steps}-step {side}px render)"+)+_WHOLE_MODULE_DECLINED_REASON = (+ "streaming kept over whole-module offload of the {dit} MiB denoiser: measured {h2d:.0f} GiB/s host-to-device "+ "and {tflops:.0f} bf16 TFLOPS price a {steps}-step {side}px render at {model:.2f} s whole vs {group:.2f} s streamed"+)++# ``auto``: whole-module offload only where priced cheaper; ``1`` always (#15129); ``0`` never.+WHOLE_MODULE_OFFLOAD_ENV = "UNSLOTH_DIFFUSION_WHOLE_MODULE_OFFLOAD"+WHOLE_MODULE_PER_REQUEST_ENV = "UNSLOTH_DIFFUSION_WHOLE_MODULE_PER_REQUEST"+WHOLE_TIER_ATTR = "_unsloth_whole_tier"+++def per_request_whole_enabled() -> bool:+ return not _env_off(WHOLE_MODULE_PER_REQUEST_ENV)+++# Un-overlapped whole upload vs one streamed step's transfer: 1.47x B200, 1.27-1.58x RTX PRO 6000 (klein-4B bf16).+_WHOLE_UPLOAD_FACTOR = 1.5+# Resident step time / probe-priced FLOPs: 1.38 B200, 0.97 RTX PRO 6000; 1.0 picks right on both, 1.35 does not.+_DIT_COMPUTE_EFFICIENCY = 1.0+_COST_CANVAS_PX = 1024+# 2x2 patches of an 8x VAE latent+_DIT_PIXELS_PER_TOKEN = 16++_OFFLOAD_COST_PROBES: dict = {}+++def _offload_cost_measurements() -> Optional[tuple[float, float]]:+ """(pinned host-to-device GiB/s, bf16 GEMM TFLOPS) of the current CUDA device, cached per device; None off CUDA.+ Best of a few runs, so a transient stall on a shared card is not read as the card."""+ result = None+ key = None+ try:+ import torch++ if not torch.cuda.is_available() or is_rocm_torch(torch):+ return None+ key = torch.cuda.current_device()+ if key in _OFFLOAD_COST_PROBES:+ return _OFFLOAD_COST_PROBES[key]+ device = torch.device("cuda", key)+ with torch.cuda.device(device):+ nbytes = 256 << 20+ host = torch.empty(nbytes, dtype = torch.uint8, pin_memory = True)+ dev = torch.empty(nbytes, dtype = torch.uint8, device = device)+ a = torch.randn(4096, 3072, dtype = torch.bfloat16, device = device)+ b = torch.randn(3072, 3072, dtype = torch.bfloat16, device = device)+ start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)++ def _best_ms(fn, reps: int, inner: int) -> float:+ fn()+ torch.cuda.synchronize()+ best = float("inf")+ for _ in range(reps):+ start.record()+ for _ in range(inner):+ fn()+ end.record()+ end.synchronize()+ best = min(best, start.elapsed_time(end) / inner)+ return best++ copy_ms = _best_ms(lambda: dev.copy_(host, non_blocking = True), 3, 2)+ gemm_ms = _best_ms(lambda: torch.mm(a, b), 3, 10)+ del host, dev, a, b+ torch.cuda.empty_cache()+ if copy_ms > 0 and gemm_ms > 0:+ result = (+ (nbytes / 2**30) / (copy_ms / 1000.0),+ 2.0 * 4096 * 3072 * 3072 / (gemm_ms / 1000.0) / 1e12,+ )+ except Exception: # noqa: BLE001 - unmeasurable: the caller keeps its unpriced choice+ result = None+ if key is not None:+ _OFFLOAD_COST_PROBES[key] = result+ return result+++def _whole_module_offload_costs(+ dit_mib: int,+ *,+ steps: int,+ h2d_gib_s: float,+ bf16_tflops: float,+ side_px: int = _COST_CANVAS_PX,+ tokens: Optional[int] = None,+) -> tuple[float, float]:+ """(whole-module, streamed) denoiser seconds for one ``steps``-step render.++ streamed = ``steps * max(compute, transfer)``; transfer counts the whole denoiser even when partly resident (B200+ klein step stayed 0.145-0.170 s at 0-6751 of 7393 MiB resident). whole = one un-overlapped upload + resident steps.+ """+ gib = max(0, int(dit_mib)) / 1024.0+ transfer = gib / max(float(h2d_gib_s), 1e-6)+ params = max(0, int(dit_mib)) * 2**20 / 2.0 # bf16+ if tokens is None:+ tokens = (int(side_px) // _DIT_PIXELS_PER_TOKEN) ** 2+ compute = (+ _DIT_COMPUTE_EFFICIENCY * 2.0 * params * tokens / (max(float(bf16_tflops), 1e-6) * 1e12)+ )+ n = max(1, int(steps))+ whole = _WHOLE_UPLOAD_FACTOR * transfer + n * compute+ streamed = n * max(compute, transfer)+ return whole, streameddef prefer_whole_module_offload(@@ -4030,12 +4137,15 @@ def prefer_whole_module_offload(refined: MemoryPlan,*,logger: Any = None,+ steps: Optional[int] = None,) -> MemoryPlan:- """``refined`` (what ``refine_plan_from_loaded_weights`` returned), or whole-module offload in its place.-- Below the streamed tiers' floor whole-module offload holds the whole denoiser on the card while it runs; a few- hundred MiB more free VRAM used to switch to streaming all of it every step, but uploading it once per call beats- that (FLUX.2-klein-4B bf16, B200: 0.34 vs 0.60 s per image). Plain bf16 only: an fp16 T4 run pointed the other way."""+ """``refined``, or whole-module offload in its place where it beats streaming. With ``steps`` the switch is+ priced from this card's measured bandwidth and bf16 throughput (RTX PRO 6000 streams faster at 4 steps, B200 does+ not); unmeasurable or without ``steps`` it is kept. Plain bf16 only: an fp16 T4 run pointed the other way.+ """+ switch = (os.environ.get(WHOLE_MODULE_OFFLOAD_ENV) or "auto").strip().lower()+ if switch in ("0", "off", "false", "no", "never"):+ return refinedtry:if (getattr(refined, "requested_mode", None) != MEMORY_MODE_AUTO@@ -4067,6 +4177,53 @@ def prefer_whole_module_offload(if int(getattr(refined, "resident_transformer_mib", None) or 0) >= dit:return refinedreason = _WHOLE_MODULE_KEPT_REASON.format(dit = transformer, floor = floor)+ priced: dict = {}+ if (+ steps is not None+ and switch not in ("1", "on", "true", "yes", "always")+ and getattr(pipe, "transformer", None) is not None+ ):+ measured = _offload_cost_measurements()+ if measured is not None:+ h2d, tflops = measured+ whole, streamed = _whole_module_offload_costs(+ dit, steps = int(steps), h2d_gib_s = h2d, bf16_tflops = tflops+ )+ fields = dict(+ dit = dit,+ h2d = h2d,+ tflops = tflops,+ model = whole,+ group = streamed,+ steps = int(steps),+ side = _COST_CANVAS_PX,+ )+ priced = {+ "whole_module_cost_s": round(whole, 4),+ "streamed_cost_s": round(streamed, 4),+ "whole_module_cost_steps": int(steps),+ "measured_h2d_gib_s": round(h2d, 2),+ "measured_bf16_tflops": round(tflops, 1),+ }+ if whole >= streamed:+ declined = _WHOLE_MODULE_DECLINED_REASON.format(**fields)+ if logger is not None:+ logger.info("diffusion.memory: %s", declined)+ if per_request_whole_enabled():+ try:+ setattr(+ pipe,+ WHOLE_TIER_ATTR,+ {"dit_mib": int(dit), "h2d_gib_s": h2d, "bf16_tflops": tflops},+ )+ except Exception: # noqa: BLE001 - the load-time plan then stays for every request+ pass+ return replace(+ refined,+ reasons = refined.reasons + (declined,),+ estimates = {**refined.estimates, **priced},+ )+ reason += _WHOLE_MODULE_COST_REASON.format(**fields)if logger is not None:logger.info("diffusion.memory: %s", reason)return replace(@@ -4079,7 +4236,7 @@ def prefer_whole_module_offload(resident_transformer_mib = None,resident_text_encoder_mib = None,reasons = refined.reasons + (reason,),- estimates = {**refined.estimates, "whole_module_kept_below_floor_mib": floor},+ estimates = {**refined.estimates, "whole_module_kept_below_floor_mib": floor, **priced},)except Exception as exc: # noqa: BLE001 - keep the streamed refinementif logger is not None:@@ -4087,6 +4244,144 @@ def prefer_whole_module_offload(return refined+def whole_tier_request_costs(+ pipe: Any,+ *,+ steps: Optional[int],+ width: Optional[int],+ height: Optional[int],+ batch_size: int = 1,+ cfg: Optional[bool] = None,+ condition_pixels: int = 0,+) -> Optional[tuple[float, float]]:+ """(whole, streamed) seconds for this request's own tokens (canvas, condition images, batch, CFG); None when the+ load did not record :data:`WHOLE_TIER_ATTR` or the per-request choice is off."""+ tier = getattr(pipe, WHOLE_TIER_ATTR, None)+ if not isinstance(tier, dict) or steps is None or not per_request_whole_enabled():+ return None+ try:+ w = max(64, int(width or DEFAULT_IMAGE_WIDTH))+ h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))+ per_image = (w // _DIT_PIXELS_PER_TOKEN) * (h // _DIT_PIXELS_PER_TOKEN) + max(+ 0, int(condition_pixels or 0)+ ) // (_DIT_PIXELS_PER_TOKEN**2)+ tokens = per_image * max(1, int(batch_size or 1)) * (2 if cfg else 1)+ return _whole_module_offload_costs(+ int(tier["dit_mib"]),+ steps = int(steps),+ h2d_gib_s = float(tier["h2d_gib_s"]),+ bf16_tflops = float(tier["bf16_tflops"]),+ tokens = tokens,+ )+ except Exception: # noqa: BLE001 - unpriced: the load's streamed plan+ return None+++_MISSING = object()+++def pin_denoiser_for_call(+ pipe: Any,+ device: Any,+ *,+ activation_mib: int,+ logger: Any = None,+) -> Optional[Callable[[], None]]:+ """Pin every streamed offload group of ``pipe.transformer`` for one call; returns the release, or None.++ The hooks stay and blocks compile below them, so compiled code never sees the pin. Pinned only if the groups plus+ ``activation_mib`` fit free memory now (VAE reserve kept); all or nothing."""+ transformer = getattr(pipe, "transformer", None)+ try:+ import torch++ groups = _offload_groups(transformer) if transformer is not None else []+ if not groups or any(getattr(g, "offload_to_disk_path", None) for g in groups):+ return None+ streamed = [g for g in groups if not getattr(g, "_unsloth_resident", False)]+ if not streamed:+ return None+ onload = torch.device(device if device is not None else "cuda")+ if onload.type != "cuda":+ return None+ if onload.index is None:+ onload = torch.device("cuda", torch.cuda.current_device())+ unique: dict = {}+ for group in streamed:+ for t in _resident_tensors(group):+ unique.setdefault(id(t), t)+ need = sum(sum(_storage_nbytes(t)) for t in unique.values())+ free, total = (int(x) for x in torch.cuda.mem_get_info(onload))+ free -= user_budget_reserve_bytes(onload, total)+ cached = max(+ 0,+ int(torch.cuda.memory_reserved(onload))+ - int(torch.cuda.memory_allocated(onload))+ - _graph_held_cache_bytes(onload.index),+ )+ room = (free + cached) >> 20+ if (need >> 20) + max(0, int(activation_mib)) > room:+ if logger is not None:+ logger.info(+ "diffusion.memory: the denoiser keeps streaming: running it whole needs %d MiB + %d MiB of "+ "activations, %d MiB free",+ need >> 20,+ int(activation_mib),+ room,+ )+ return None+ every: dict = {}+ for group in groups:+ for t in _resident_tensors(group):+ every.setdefault(id(t), t)+ whole_mib = (sum(sum(_storage_nbytes(t)) for t in every.values()) >> 20) + 2+ saved = {+ name: transformer.__dict__.get(name, _MISSING)+ for name in ("_unsloth_resident_room", "_unsloth_resident_device")+ }+ _keep_groups_resident(transformer, whole_mib, onload, logger)+ pinned = [g for g in streamed if getattr(g, "_unsloth_resident", False)]+ done = [False]++ def release() -> None:+ if done[0]:+ return+ done[0] = True+ state = getattr(transformer, "_unsloth_stream_state", None)+ if isinstance(state, dict):+ state["streamed"] = (+ 1 # a release that fails part way still waits on the copy stream+ )+ try:+ for group in reversed(pinned):+ if getattr(group, "_unsloth_resident", False):+ _release_group(group)+ _release_group_range(transformer, group)+ finally:+ if isinstance(state, dict):+ state["streamed"] = _streamed_group_count(groups)+ for name, value in saved.items():+ try:+ if value is _MISSING:+ transformer.__dict__.pop(name, None)+ else:+ setattr(transformer, name, value)+ except Exception: # noqa: BLE001+ pass+ from .diffusion_cuda_graph import synchronize_outside_capture++ synchronize_outside_capture(onload)++ if len(pinned) != len(streamed):+ release()+ return None+ return release+ except Exception as exc: # noqa: BLE001 - streaming every group is the safe state+ if logger is not None:+ logger.warning("diffusion.memory: whole denoiser for this call skipped (%s)", exc)+ return None++def _refine_plan_from_loaded_weights(pipe: Any,plan: MemoryPlan,diff --git a/studio/backend/tests/test_diffusion_offload_tier_per_request.py b/studio/backend/tests/test_diffusion_offload_tier_per_request.pynew file mode 100644index 00000000000..2f75d5169c1--- /dev/null+++ b/studio/backend/tests/test_diffusion_offload_tier_per_request.py@@ -0,0 +1,362 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Whole-module vs streamed offload chosen per request."""++from __future__ import annotations++import sys+import types+from pathlib import Path++import pytest++import core.inference.diffusion as dif+import core.inference.diffusion_memory as dm++sys.path.insert(0, str(Path(__file__).resolve().parent))+from test_diffusion_placement_monotonic import ( # noqa: E402, F401 - the autouse env fixture+ _card_probe,+ _clean_env,+ _klein_above_floor,+ _priced,+)++KLEIN_DIT_MIB = 7393++[email protected](autouse = True)+def _per_request_env(monkeypatch):+ monkeypatch.delenv(dm.WHOLE_MODULE_OFFLOAD_ENV, raising = False)+ monkeypatch.delenv(dm.WHOLE_MODULE_PER_REQUEST_ENV, raising = False)+++def _tier_pipe(card):+ h2d, tflops = _card_probe(card)+ pipe = types.SimpleNamespace()+ setattr(+ pipe,+ dm.WHOLE_TIER_ATTR,+ {"dit_mib": KLEIN_DIT_MIB, "h2d_gib_s": h2d, "bf16_tflops": tflops},+ )+ return pipe+++def _whole(pipe, steps, **kw):+ costs = dm.whole_tier_request_costs(+ pipe, steps = steps, width = kw.pop("width", 1024), height = kw.pop("height", 1024), **kw+ )+ return None if costs is None else costs[0] < costs[1]+++def test_streamed_load_records_the_probe_for_requests(monkeypatch):+ """RTX PRO 6000 at 4 steps streams and keeps the probe; a card that priced whole keeps the #15129 plan."""+ flat = _klein_above_floor(monkeypatch)+ refined, plan = _priced(monkeypatch, 12544, flat, steps = 4, card = "rtx_pro_6000")+ assert plan.offload_policy == dm.OFFLOAD_GROUP+ import torch++ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = torch.bfloat16))+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: _card_probe("rtx_pro_6000"))+ dm.prefer_whole_module_offload(pipe, refined, steps = 4)+ tier = getattr(pipe, dm.WHOLE_TIER_ATTR)+ assert tier["dit_mib"] == KLEIN_DIT_MIB+ assert (tier["h2d_gib_s"], tier["bf16_tflops"]) == _card_probe("rtx_pro_6000")++ for card in ("b200", "a100"):+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = torch.bfloat16))+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda card = card: _card_probe(card))+ assert (+ dm.prefer_whole_module_offload(pipe, refined, steps = 4).offload_policy+ == dm.OFFLOAD_MODEL+ )+ assert not hasattr(pipe, dm.WHOLE_TIER_ATTR)++ monkeypatch.setenv(dm.WHOLE_MODULE_PER_REQUEST_ENV, "0")+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = torch.bfloat16))+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: _card_probe("rtx_pro_6000"))+ assert dm.prefer_whole_module_offload(pipe, refined, steps = 4).offload_policy == dm.OFFLOAD_GROUP+ assert not hasattr(pipe, dm.WHOLE_TIER_ATTR)+++def test_request_step_count_picks_the_tier():+ """The measured RTX PRO 6000 calls: streamed at 2 and 4 steps, whole at 8."""+ pipe = _tier_pipe("rtx_pro_6000")+ assert [_whole(pipe, n) for n in (1, 2, 4, 8, 16)] == [False, False, False, True, True]+++def test_request_size_moves_the_crossover():+ """More tokens per step (canvas, batch, CFG) let streaming win to more steps."""+ pipe = _tier_pipe("rtx_pro_6000")+ assert _whole(pipe, 8) is True+ assert _whole(pipe, 8, batch_size = 2) is False+ assert _whole(pipe, 8, cfg = True) is False+ assert _whole(pipe, 8, width = 1536, height = 1536) is False+ assert _whole(pipe, 4, width = 512, height = 512) is True+++def test_no_tier_or_kill_switch_prices_nothing(monkeypatch):+ assert (+ dm.whole_tier_request_costs(types.SimpleNamespace(), steps = 8, width = 1024, height = 1024)+ is None+ )+ pipe = _tier_pipe("rtx_pro_6000")+ assert dm.whole_tier_request_costs(pipe, steps = None, width = 1024, height = 1024) is None+ monkeypatch.setenv(dm.WHOLE_MODULE_PER_REQUEST_ENV, "0")+ assert dm.whole_tier_request_costs(pipe, steps = 8, width = 1024, height = 1024) is None+++class _Bg:+ def __init__(self, pending):+ self._pending = pending++ def pending(self):+ return self._pending+++class _Graph:+ def __init__(+ self,+ declined,+ poisoned = False,+ ):+ self._declined = declined+ self.poisoned = poisoned+ self.fallback_handle = None+ self.fallback = None++ def _placement_declined(self):+ return self._declined+++def _state(**kw):+ pipe = _tier_pipe("rtx_pro_6000")+ pipe._unsloth_cuda_graphs = kw.pop("graphs", ())+ base = dict(+ pipe = pipe,+ offload_policy = dm.OFFLOAD_GROUP,+ transformer_cache = None,+ bg_compile = None,+ cuda_graphs = (),+ device = "cuda:0",+ )+ base.update(kw)+ return types.SimpleNamespace(**base)+++def _request(state, steps = 8):+ return dif._whole_tier_for_request(+ state, steps = steps, width = 1024, height = 1024, batch_size = 1, cfg = False, condition_pixels = 0+ )+++LIVE = ("flux.2-klein", "dense:bfloat16|attn:_native_cudnn", "default")+LOADED = ("flux.2-klein", "dense:bfloat16|attn:native", "default")+RECORDED = ("dense:bfloat16|attn:native", "off") # the eager renders' key+++def _learned(monkeypatch):+ seen = []++ def fake_peak(family, speed, **kw):+ seen.append((kw.get("quant"), speed, kw.get("phase")))+ return 2560 if (kw.get("quant"), speed) == RECORDED else None++ monkeypatch.setattr(dif, "_live_peak_identity", lambda state: LIVE)+ monkeypatch.setattr(dif, "measured_image_runtime_mib", fake_peak)+ monkeypatch.setattr(dif.cuda_graph, "live_graph_count", lambda index: 0)+ monkeypatch.setattr(dif, "_peak_device_index", lambda state: 0)+ return seen+++def test_request_gate_prices_and_reads_the_eager_peak(_learned):+ """A deferred compile tier records no peak: the eager renders' pre-decode peak sizes the pinned call."""+ state = _state()+ state.pipe._unsloth_peak_identity = LOADED+ whole, streamed, activation = _request(state)+ assert whole < streamed and activation == 2560+ assert _learned == [+ (LIVE[1], "default", "pre_decode"),+ (LIVE[1], "off", "pre_decode"),+ (LOADED[1], "default", "pre_decode"),+ (LOADED[1], "off", "pre_decode"),+ ]+ assert _request(state, steps = 4) is None+++def test_request_gate_reads_the_live_key_first(_learned, monkeypatch):+ monkeypatch.setattr(dif, "_live_peak_identity", lambda state: ("flux.2-klein", *RECORDED))+ state = _state()+ state.pipe._unsloth_peak_identity = LOADED+ assert _request(state)[2] == 2560+ assert _learned == [(*RECORDED, "pre_decode")]+++ "kw",+ [+ dict(offload_policy = dm.OFFLOAD_MODEL),+ dict(transformer_cache = "fbcache"),+ dict(bg_compile = _Bg(True)),+ dict(graphs = (_Graph(declined = False),)),+ ],+)+def test_request_gate_keeps_the_load_plan(_learned, kw):+ assert _request(_state(**kw)) is None+++def test_request_gate_after_graphs_settle(_learned, monkeypatch):+ def state(**kw):+ out = _state(**kw)+ out.pipe._unsloth_peak_identity = LOADED+ return out++ assert _request(state(bg_compile = _Bg(False), graphs = (_Graph(declined = True),))) is not None+ assert _request(state(graphs = (_Graph(declined = False, poisoned = True),))) is not None+ monkeypatch.setattr(dif.cuda_graph, "live_graph_count", lambda index: 1)+ assert _request(state(graphs = (_Graph(declined = True),))) is None+++def test_request_gate_without_a_learned_peak_streams(_learned, monkeypatch):+ monkeypatch.setattr(dif, "measured_image_runtime_mib", lambda *a, **k: None)+ assert _request(_state()) is None+++def test_request_gate_declines_condition_images(_learned):+ """Condition-image requests stream."""+ state = _state()+ state.pipe._unsloth_peak_identity = LOADED+ assert (+ dif._whole_tier_for_request(+ state,+ steps = 8,+ width = 1024,+ height = 1024,+ batch_size = 1,+ cfg = False,+ condition_pixels = 1 << 20,+ )+ is None+ )+ assert _learned == []+++def test_request_gate_reads_the_workflow_peak(_learned):+ """An img2img request is sized from the peak recorded under its own workflow, never the text-to-image one."""+ state = _state()+ state.pipe._unsloth_peak_identity = LOADED+ out = dif._whole_tier_for_request(+ state,+ steps = 8,+ width = 1024,+ height = 1024,+ batch_size = 1,+ cfg = False,+ condition_pixels = 0,+ workflow = "img2img",+ )+ assert out is None # only the text-to-image key holds a peak in the fixture+ assert _learned and all(q.endswith("|wf:img2img") for q, _, _ in _learned)+++def test_the_request_gate_prices_the_largest_chunk():+ """The tier is priced on the forward the request runs, not the activation guard's singleton floor."""+ import inspect++ src = inspect.getsource(dif)+ call = src.split("whole_tier = _whole_tier_for_request(", 1)[1].split(")\n", 1)[0]+ assert "batch_size = max((len(chunk) for chunk in chunks), default = 1)" in call+ assert "guard_batch" not in call+++def _cuda_net():+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("CUDA-only: diffusers group offloading with a copy stream")+ pytest.importorskip("diffusers.hooks")+ from diffusers.hooks import apply_group_offloading++ class Net(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj_in = torch.nn.Linear(64, 1024)+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(1024, 1024) for _ in range(6))+ self.proj_out = torch.nn.Linear(1024, 64)++ def forward(self, x):+ x = self.proj_in(x)+ for block in self.blocks:+ x = torch.nn.functional.gelu(block(x))+ return self.proj_out(x)++ torch.manual_seed(0)+ net = Net()+ x = torch.randn(3, 64)+ ref = net.to("cuda")(x.cuda()).cpu()+ net.to("cpu")+ apply_group_offloading(+ net,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ record_stream = True,+ non_blocking = True,+ )+ return torch, net, x, ref+++def test_pin_for_call_is_bit_identical_and_streams_again():+ """Every group resident for the call, same output, then the exact pre-call placement (same host tensors)."""+ torch, net, x, ref = _cuda_net()+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})+ hosts = {id(p): p.data for p in net.blocks.parameters()}+ release = dm.pin_denoiser_for_call(pipe, "cuda", activation_mib = 0)+ assert release is not None+ assert all(p.device.type == "cuda" for p in net.parameters())+ assert dm.denoisers_pinned_resident(pipe)+ for _ in range(2):+ assert torch.equal(net(x.cuda()).cpu(), ref)+ release()+ release() # idempotent+ assert all(p.device.type == "cpu" for p in net.blocks.parameters())+ assert all(p.data.data_ptr() == hosts[id(p)].data_ptr() for p in net.blocks.parameters())+ assert not getattr(net, "_unsloth_resident_room", None)+ assert not any(getattr(g, "_unsloth_resident", False) for g in dm._offload_groups(net))+ assert dm.resident_group_mib(pipe) == 0+ for _ in range(2):+ assert torch.equal(net(x.cuda()).cpu(), ref)+++def test_pin_for_call_needs_the_room(monkeypatch):+ torch, net, x, ref = _cuda_net()+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})+ torch.cuda.empty_cache() # the room counts the allocator's reusable cache+ free, total = torch.cuda.mem_get_info()+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1 << 20, total))+ assert dm.pin_denoiser_for_call(pipe, "cuda", activation_mib = 0) is None+ assert all(p.device.type == "cpu" for p in net.blocks.parameters())+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (free, total))+ assert dm.pin_denoiser_for_call(pipe, "cuda", activation_mib = 1 << 30) is None+ release = dm.pin_denoiser_for_call(pipe, "cuda", activation_mib = 64)+ assert release is not None+ release()+ assert torch.equal(net(x.cuda()).cpu(), ref)+++def test_request_gate_keeps_the_eager_key_once_the_profile_rewrites_the_load_key(+ _learned, monkeypatch+):+ """The deferred profile rewrites the load's key and nothing records after capture: the eager key is remembered."""+ live = {"id": ("flux.2-klein", *RECORDED)}+ monkeypatch.setattr(dif, "_live_peak_identity", lambda state: live["id"])+ state = _state()+ state.pipe._unsloth_peak_identity = ("flux.2-klein", *RECORDED)+ assert _request(state, steps = 2) is None # streamed, but the eager key is kept+ live["id"] = LIVE+ state.pipe._unsloth_peak_identity = LIVE+ _learned.clear()+ assert _request(state)[2] == 2560+ assert _learned[-1] == (*RECORDED, "pre_decode")diff --git a/studio/backend/tests/test_diffusion_placement_monotonic.py b/studio/backend/tests/test_diffusion_placement_monotonic.pyindex 43a5951e903..87cc32ac055 100644--- a/studio/backend/tests/test_diffusion_placement_monotonic.py+++ b/studio/backend/tests/test_diffusion_placement_monotonic.py@@ -379,3 +379,182 @@ def test_unmeasured_denoisers_keep_the_streamed_plan(monkeypatch, family, quant)plan = _plan(12544, 24 * GIB, flat, family = family, torchao = torchao, compute = compute)assert plan.offload_policy == dm.OFFLOAD_GROUP and plan.stream_transformerassert "whole_module_kept_below_floor_mib" not in plan.estimates+++# (GiB/s, TFLOPS) probes reproducing measured klein-4B bf16 1024^2 per-step transfer / compute.+KLEIN_DIT_MIB = 7393+CARDS = {+ "b200": (0.142, 0.0296),+ "a100": (0.541, 0.150),+ "rtx_pro_6000": (0.1415, 0.0985),+}+++def _card_probe(card):+ transfer, compute = CARDS[card]+ flops = 2.0 * KLEIN_DIT_MIB * 2**20 / 2.0 * (1024 // 16) ** 2+ return (KLEIN_DIT_MIB / 1024.0 / transfer, dm._DIT_COMPUTE_EFFICIENCY * flops / compute / 1e12)+++def _klein_above_floor(monkeypatch, *, compute = None):+ q8, dense, encoders, vae = FAMILIES["flux.2-klein"]+ te = sum(encoders.values())+ loaded = {+ "transformer": (KLEIN_DIT_MIB, "dit"),+ "text_encoder": (te, "text_encoder"),+ "vae": (vae, "other"),+ }+ _patch_weights(monkeypatch, loaded, torchao = False, compute = compute)+ flat = dict(+ model_dense_mib = KLEIN_DIT_MIB + te + vae,+ companion_dense_mib = te + vae,+ text_encoder_dense_mib = te,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(+ width = None, height = None, family = "flux.2-klein"+ ),+ )+ return flat+++def _priced(+ monkeypatch,+ free_mib,+ flat,+ *,+ steps,+ card,+ compute = None,+):+ import torch++ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: _card_probe(card))+ plan = dm.plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = _discrete(free_mib, 24 * GIB),+ **flat,+ )+ dtype = torch.float16 if compute == 2 else torch.bfloat16+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = dtype))+ refined = dm.refine_plan_from_loaded_weights(+ pipe, plan, family = "flux.2-klein", speed_mode = "default"+ )+ return refined, dm.prefer_whole_module_offload(pipe, refined, steps = steps)+++ "card, steps, whole",+ [+ ("b200", 2, True),+ ("b200", 4, True),+ ("b200", 8, True),+ ("a100", 4, True),+ ("a100", 8, True),+ ("rtx_pro_6000", 2, False),+ ("rtx_pro_6000", 4, False),+ ("rtx_pro_6000", 8, True),+ ],+)+def test_whole_module_offload_is_kept_only_where_it_is_cheaper(monkeypatch, card, steps, whole):+ """B200 0.336 vs 0.595 s and A100 2.336 vs 3.429 s at 4 steps (whole-module faster); RTX PRO 6000 0.762 vs 0.743 s+ at 4 steps (streamed faster), 1.022 vs 1.181 s at 8, 0.561 vs 0.458 s at 2."""+ flat = _klein_above_floor(monkeypatch)+ refined, plan = _priced(monkeypatch, 12544, flat, steps = steps, card = card)+ assert refined.offload_policy == dm.OFFLOAD_GROUP and refined.stream_transformer+ model_s, group_s = plan.estimates["whole_module_cost_s"], plan.estimates["streamed_cost_s"]+ assert (model_s < group_s) is whole+ if whole:+ assert plan.offload_policy == dm.OFFLOAD_MODEL+ assert plan.estimates["whole_module_kept_below_floor_mib"] == 10400+ else:+ assert plan.offload_policy == dm.OFFLOAD_GROUP+ assert "whole_module_kept_below_floor_mib" not in plan.estimates+ assert replace_fields(plan) == replace_fields(refined)+ assert any("streaming kept over whole-module offload" in r for r in plan.reasons)+++def replace_fields(plan):+ return dm.replace(plan, reasons = (), estimates = {})+++def test_cost_model_fits_the_measured_crossovers():+ """Per-card crossover of whole-module vs streamed (steps where they cost the same): RTX PRO 6000 measured 4.2-4.35."""++ def crossover(card):+ h2d, tflops = _card_probe(card)+ (w1, s1), (w2, _) = (+ dm._whole_module_offload_costs(+ KLEIN_DIT_MIB, steps = n, h2d_gib_s = h2d, bf16_tflops = tflops+ )+ for n in (1, 2)+ )+ compute = w2 - w1+ return (w1 - compute) / (s1 - compute)++ assert crossover("b200") <= 2.0+ assert crossover("a100") <= 4.0+ assert 4.0 < crossover("rtx_pro_6000") < 5.0+++def test_unpriced_or_kill_switch_keep_their_meaning(monkeypatch):+ """No steps or no measurement: the unconditional switch (#15129). ``1`` always switches, ``0`` never does."""+ flat = _klein_above_floor(monkeypatch)+ _, unpriced = _priced(monkeypatch, 12544, flat, steps = None, card = "rtx_pro_6000")+ assert unpriced.offload_policy == dm.OFFLOAD_MODEL+ assert "whole_module_cost_s" not in unpriced.estimates+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: None)+ q8, dense, encoders, vae = FAMILIES["flux.2-klein"]+ import torch++ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = torch.bfloat16))+ plan = dm.plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = _discrete(12544, 24 * GIB),+ **flat,+ )+ refined = dm.refine_plan_from_loaded_weights(+ pipe, plan, family = "flux.2-klein", speed_mode = "default"+ )+ assert dm.prefer_whole_module_offload(pipe, refined, steps = 4).offload_policy == dm.OFFLOAD_MODEL+ monkeypatch.setenv(dm.WHOLE_MODULE_OFFLOAD_ENV, "1")+ _, forced = _priced(monkeypatch, 12544, flat, steps = 4, card = "rtx_pro_6000")+ assert (+ forced.offload_policy == dm.OFFLOAD_MODEL and "whole_module_cost_s" not in forced.estimates+ )+ monkeypatch.setenv(dm.WHOLE_MODULE_OFFLOAD_ENV, "0")+ refined, off = _priced(monkeypatch, 12544, flat, steps = 4, card = "b200")+ assert off is refined++[email protected]("card", sorted(CARDS))[email protected]("steps", (2, 4, 8))+def test_plans_outside_the_window_are_unchanged_by_pricing(monkeypatch, card, steps):+ """Below the floor, at the resident tiers and on an fp16 denoiser the priced call returns the same plan."""+ for compute in (None, 2):+ flat = _klein_above_floor(monkeypatch, compute = compute)+ for free in range(4 * GIB, 24 * GIB + 1, STEP_MIB):+ refined, priced = _priced(+ monkeypatch, free, flat, steps = steps, card = card, compute = compute+ )+ import torch++ dtype = torch.float16 if compute == 2 else torch.bfloat16+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = dtype))+ unpriced = dm.prefer_whole_module_offload(pipe, refined)+ if "whole_module_kept_below_floor_mib" not in unpriced.estimates:+ assert priced == unpriced == refined, (free, compute)+++ "probe, decisions",+ [+ # real probe readings: B200, and RTX PRO 6000 (measured streamed at 2 and 4 steps, whole at 8)+ ((51.5, 1490.0), {2: True, 4: True, 8: True}),+ ((52.97, 309.0), {2: False, 4: False, 8: True}),+ ],+)+def test_real_probe_readings_make_the_measured_call(probe, decisions):+ for steps, whole in decisions.items():+ model_s, group_s = dm._whole_module_offload_costs(+ 7438, steps = steps, h2d_gib_s = probe[0], bf16_tflops = probe[1]+ )+ assert (model_s < group_s) is whole, (probe, steps, model_s, group_s)