Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: keep whole-module offload for a bf16 denoiser past the streamed floor (#15129)

Daniel Han609524dec1bd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 73be7760f2d..7a76663a8c4 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -220,6 +220,7 @@ from .diffusion_memory import (
refine_balanced_plan_for_components,
refine_memory_plan_for_components,
refine_plan_from_loaded_weights,
+ prefer_whole_module_offload,
learned_peak_label,
CFG_OFF_PART,
cfg_off_peaks_enabled,
@@ -10400,6 +10401,7 @@ class DiffusionBackend:
cache = peak_cache,
graphs = peak_graphs,
)
+ plan = prefer_whole_module_offload(pipe, plan, logger = logger)
headroom = plan.estimates.get("measured_runtime_headroom_mib")
if headroom and (
plan.resident_transformer_mib or plan.resident_text_encoder_mib
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index eae23158103..6569a16e2dd 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -3930,6 +3930,99 @@ def refine_plan_from_loaded_weights(
return refined
+def _denoiser_is_dense_bf16(pipe: Any) -> bool:
+ """A plain bf16 denoiser: no torchao, packed (GGUF / int8) or fp8-stored weights."""
+ try:
+ import torch
+
+ module = next(
+ (
+ getattr(pipe, n, None)
+ for n in ("transformer", "unet")
+ if getattr(pipe, n, None) is not None
+ ),
+ None,
+ )
+ if module is None or getattr(module, "dtype", None) != torch.bfloat16:
+ return False
+ if _pipe_denoisers_hold_torchao(pipe) or _pipe_denoisers_hold_packed(pipe):
+ return False
+ return not any(
+ _holds_fp8_stored(getattr(pipe, n, None))
+ for n in ("transformer", "transformer_2", "unet")
+ )
+ except Exception: # noqa: BLE001 - unreadable: keep the streamed plan
+ return False
+
+
+_WHOLE_MODULE_KEPT_REASON = (
+ "whole-module offload ran the {dit} MiB denoiser whole below the streamed tiers' {floor} MiB floor; it still "
+ "fits, and uploading it once per call beats streaming it every step"
+)
+
+
+def prefer_whole_module_offload(
+ pipe: Any,
+ refined: MemoryPlan,
+ *,
+ logger: Any = None,
+) -> MemoryPlan:
+ """``refined`` (what ``refine_plan_from_loaded_weights`` returned), or whole-module offload in its place.
+
+ Below the streamed tiers' floor whole-module offload holds the whole denoiser on the card while it runs; a few
+ hundred MiB more free VRAM used to switch to streaming all of it every step, but uploading it once per call beats
+ that (FLUX.2-klein-4B bf16, B200: 0.34 vs 0.60 s per image). Plain bf16 only: an fp16 T4 run pointed the other way."""
+ try:
+ if (
+ getattr(refined, "requested_mode", None) != MEMORY_MODE_AUTO
+ or refined.offload_policy != OFFLOAD_GROUP
+ or not bool(getattr(refined, "stream_transformer", True))
+ or getattr(refined, "secondary_ordinal", None) is not None
+ ):
+ return refined
+ est = refined.estimates
+ model_dense, companions = est.get("model_dense_mib"), est.get("companion_dense_mib")
+ floors = [
+ int(f)
+ for f in (est.get("group_floor_mib"), est.get("group_floor_streamed_te_mib"))
+ if f is not None
+ ]
+ if model_dense is None or companions is None or not floors:
+ return refined
+ transformer = int(model_dense) - int(companions)
+ te = max(0, int(est.get("text_encoder_dense_mib") or 0))
+ others = max(0, int(companions) - te)
+ floor = min(floors)
+ if transformer <= 0 or max(transformer, te, others) >= floor:
+ return refined
+ if not _denoiser_is_dense_bf16(pipe):
+ return refined
+ # A refined plan pinning the whole denoiser runs every step resident: kept.
+ sizes = _loaded_component_mib(pipe) or {}
+ dit = sum(m for m, r in sizes.values() if r == "dit") or transformer
+ if int(getattr(refined, "resident_transformer_mib", None) or 0) >= dit:
+ return refined
+ reason = _WHOLE_MODULE_KEPT_REASON.format(dit = transformer, floor = floor)
+ if logger is not None:
+ logger.info("diffusion.memory: %s", reason)
+ return replace(
+ refined,
+ offload_policy = OFFLOAD_MODEL,
+ stream_text_encoders = False,
+ stream_transformer = True,
+ vae_tiling = True,
+ vae_slicing = True,
+ resident_transformer_mib = None,
+ resident_text_encoder_mib = None,
+ reasons = refined.reasons + (reason,),
+ estimates = {**refined.estimates, "whole_module_kept_below_floor_mib": floor},
+ )
+ except Exception as exc: # noqa: BLE001 - keep the streamed refinement
+ if logger is not None:
+ logger.debug("diffusion.memory: whole-module comparison skipped (%s)", exc)
+ return refined
+
+
def _refine_plan_from_loaded_weights(
pipe: Any,
plan: MemoryPlan,
diff --git a/studio/backend/tests/test_diffusion_placement_monotonic.py b/studio/backend/tests/test_diffusion_placement_monotonic.py
index 7499eab4fe1..43a5951e903 100644
--- a/studio/backend/tests/test_diffusion_placement_monotonic.py
+++ b/studio/backend/tests/test_diffusion_placement_monotonic.py
@@ -93,7 +93,10 @@ def _plan(
speed_mode = "default",
torchao = True,
calibrated = None,
+ compute = None,
):
+ import torch
+
plan = dm.plan_diffusion_memory(
target = types.SimpleNamespace(supports_model_cpu_offload = True),
device_memory = _discrete(free_mib, total_mib),
@@ -102,9 +105,10 @@ def _plan(
)
if plan.offload_policy == dm.OFFLOAD_MODEL and torchao:
plan = dm.torchao_streaming_plan(plan)
- return dm.refine_plan_from_loaded_weights(
- types.SimpleNamespace(transformer = object()), plan, family = family, speed_mode = speed_mode
- )
+ dtype = torch.float16 if compute == 2 else torch.bfloat16
+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = dtype))
+ refined = dm.refine_plan_from_loaded_weights(pipe, plan, family = family, speed_mode = speed_mode)
+ return dm.prefer_whole_module_offload(pipe, refined)
def _denoiser_key(plan, dit_mib):
@@ -148,7 +152,7 @@ def test_more_free_vram_never_keeps_less_of_the_denoiser(monkeypatch, family, qu
total = gib * GIB
previous = None
for free in range(4 * GIB, min(80, gib) * GIB + 1, STEP_MIB):
- plan = _plan(free, total, flat, family = family, torchao = torchao)
+ plan = _plan(free, total, flat, family = family, torchao = torchao, compute = compute)
key = _denoiser_key(plan, dit)
if key is None:
continue
@@ -203,3 +207,175 @@ def test_explicit_modes_are_untouched(monkeypatch, mode):
object(), plan, family = "flux.1", speed_mode = "default"
)
assert refined is plan
+
+
+def _speed_key(plan, dit_mib):
+ """(tier, -resident denoiser MiB), smaller is faster: 0 the whole denoiser stays on the device every step, 1
+ whole-module offload with the denoiser fitting the budget whole (uploaded once per call), 2 the denoiser streams
+ every step, 3 whole-module offload of a denoiser larger than the budget."""
+ if plan.offload_policy == dm.OFFLOAD_NONE or (
+ plan.offload_policy == dm.OFFLOAD_GROUP and not plan.stream_transformer
+ ):
+ return (0, -dit_mib)
+ if plan.offload_policy == dm.OFFLOAD_MODEL:
+ fits = dit_mib <= int(plan.estimates.get("safe_device_budget_mib") or 0)
+ return (1, 0) if fits else (3, 0)
+ resident = int(plan.resident_transformer_mib or 0)
+ return (0, -dit_mib) if resident >= dit_mib else (2, -resident)
+
+
[email protected]("calibrated", (False, True))
[email protected]("learned", (None, 3072))
[email protected]("quant", ("bf16",))
[email protected]("family", sorted(FAMILIES))
+def test_more_free_vram_never_trades_whole_module_offload_for_streaming(
+ monkeypatch, family, quant, learned, calibrated
+):
+ """Below the streamed tiers' floor whole-module offload holds the whole denoiser; a few hundred MiB more used to
+ stream it every step instead (FLUX.2-klein-4B bf16 on B200: 0.34 s -> 0.60 s per 4-step 1024^2 image)."""
+ q8, dense, encoders, vae = FAMILIES[family]
+ eight, torchao, fp8_stored, compute = {**QUANTS, "bf16": (False, False, False, None)}[quant]
+ dit = q8 if eight else dense
+ te = sum(encoders.values())
+ loaded = {
+ "transformer": (dit, "dit"),
+ **{name: (mib, "text_encoder") for name, mib in encoders.items()},
+ "vae": (vae, "other"),
+ }
+ _patch_weights(
+ monkeypatch,
+ loaded,
+ torchao = torchao,
+ fp8_stored = fp8_stored,
+ compute = compute,
+ learned = learned,
+ )
+ flat = dict(
+ model_dense_mib = dit + te + vae,
+ companion_dense_mib = te + vae,
+ text_encoder_dense_mib = te,
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(width = None, height = None, family = family),
+ )
+ act = dm.calibrated_image_activation(family) if calibrated else None
+ if calibrated and act is None:
+ pytest.skip("no calibrated activation for this family")
+ previous = None
+ for free in range(4 * GIB, 24 * GIB + 1, STEP_MIB):
+ plan = _plan(
+ free, 24 * GIB, flat, family = family, torchao = torchao, calibrated = act, compute = compute
+ )
+ key = _speed_key(plan, dit)
+ if previous is not None:
+ assert key <= previous[1], (free, previous, key, plan.reasons)
+ previous = (free, key)
+
+
+def test_bf16_klein_keeps_whole_module_offload_past_the_streamed_floor(monkeypatch):
+ q8, dense, encoders, vae = FAMILIES["flux.2-klein"]
+ te = sum(encoders.values())
+ loaded = {
+ "transformer": (dense, "dit"),
+ "text_encoder": (te, "text_encoder"),
+ "vae": (vae, "other"),
+ }
+ _patch_weights(monkeypatch, loaded, torchao = False)
+ flat = dict(
+ model_dense_mib = dense + te + vae,
+ companion_dense_mib = te + vae,
+ text_encoder_dense_mib = te,
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(
+ width = None, height = None, family = "flux.2-klein"
+ ),
+ )
+ below = _plan(12288, 24 * GIB, flat, family = "flux.2-klein", torchao = False)
+ above = _plan(12544, 24 * GIB, flat, family = "flux.2-klein", torchao = False)
+ assert below.offload_policy == dm.OFFLOAD_MODEL
+ assert "whole_module_kept_below_floor_mib" not in below.estimates
+ assert above.offload_policy == dm.OFFLOAD_MODEL
+ assert above.estimates["whole_module_kept_below_floor_mib"] == 10400
+ assert any("uploading it once per call" in reason for reason in above.reasons)
+ # The resident-transformer tier still wins once it fits.
+ resident = _plan(20 * GIB, 24 * GIB, flat, family = "flux.2-klein", torchao = False)
+ assert resident.offload_policy == dm.OFFLOAD_GROUP and not resident.stream_transformer
+
+
+def test_streamed_plans_that_pin_the_whole_denoiser_or_hold_torchao_are_kept(monkeypatch):
+ """A refined group plan already holding the whole denoiser beats whole-module offload; a torchao denoiser's
+ whole-module tier is streaming, so its ladder is unchanged."""
+ q8, dense, encoders, vae = FAMILIES["flux.2-klein"]
+ te = sum(encoders.values())
+ loaded = {
+ "transformer": (q8, "dit"),
+ "text_encoder": (te, "text_encoder"),
+ "vae": (vae, "other"),
+ }
+ flat = dict(
+ model_dense_mib = q8 + te + vae,
+ companion_dense_mib = te + vae,
+ text_encoder_dense_mib = te,
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(
+ width = None, height = None, family = "flux.2-klein"
+ ),
+ )
+ _patch_weights(monkeypatch, loaded, torchao = False, fp8_stored = True, learned = 3072)
+ pinned = _plan(14 * GIB, 24 * GIB, flat, family = "flux.2-klein", torchao = False)
+ assert pinned.offload_policy == dm.OFFLOAD_GROUP and pinned.resident_transformer_mib == q8
+ _patch_weights(monkeypatch, loaded, torchao = True)
+ streamed = _plan(14 * GIB, 24 * GIB, flat, family = "flux.2-klein", torchao = True)
+ assert streamed.offload_policy == dm.OFFLOAD_GROUP
+ assert "whole_module_kept_below_floor_mib" not in streamed.estimates
+
+
+def test_bf16_partial_residency_gives_way_to_whole_module_offload(monkeypatch):
+ """A learned peak pins 4.7 of the 7.4 GB bf16 denoiser past the floor; whole-module offload still holds all of it
+ (B200, 12.25 GiB free: 0.37 s whole-module vs 0.73 s with 5.5 GB pinned)."""
+ q8, dense, encoders, vae = FAMILIES["flux.2-klein"]
+ te = sum(encoders.values())
+ loaded = {
+ "transformer": (dense, "dit"),
+ "text_encoder": (te, "text_encoder"),
+ "vae": (vae, "other"),
+ }
+ flat = dict(
+ model_dense_mib = dense + te + vae,
+ companion_dense_mib = te + vae,
+ text_encoder_dense_mib = te,
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(
+ width = None, height = None, family = "flux.2-klein"
+ ),
+ )
+ _patch_weights(monkeypatch, loaded, torchao = False, learned = 3072)
+ plan = _plan(12544, 24 * GIB, flat, family = "flux.2-klein", torchao = False)
+ assert plan.offload_policy == dm.OFFLOAD_MODEL
+ # the same learned peak on an fp16 denoiser (the T4 dtype) keeps its partial residency
+ _patch_weights(monkeypatch, loaded, torchao = False, compute = 2, learned = 3072)
+ fp16 = _plan(12544, 24 * GIB, flat, family = "flux.2-klein", torchao = False, compute = 2)
+ assert fp16.offload_policy == dm.OFFLOAD_GROUP and 0 < fp16.resident_transformer_mib < dense
+
+
+ "family, quant",
+ [("flux.2-klein", "fp16"), ("flux.2-klein", "fp8_stored"), ("z-image", "fp8_stored")],
+)
+def test_unmeasured_denoisers_keep_the_streamed_plan(monkeypatch, family, quant):
+ """fp16 (the T4 dtype, where stock whole-module offload measured slower than streaming), fp8-stored and packed
+ denoisers keep the streamed group plan past the floor: only bf16 was measured faster whole-module."""
+ q8, dense, encoders, vae = FAMILIES[family]
+ eight, torchao, fp8_stored, compute = QUANTS[quant]
+ dit = q8 if eight else dense
+ te = sum(encoders.values())
+ loaded = {
+ "transformer": (dit, "dit"),
+ "text_encoder": (te, "text_encoder"),
+ "vae": (vae, "other"),
+ }
+ _patch_weights(monkeypatch, loaded, torchao = torchao, fp8_stored = fp8_stored, compute = compute)
+ flat = dict(
+ model_dense_mib = dit + te + vae,
+ companion_dense_mib = te + vae,
+ text_encoder_dense_mib = te,
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(width = None, height = None, family = family),
+ )
+ plan = _plan(12544, 24 * GIB, flat, family = family, torchao = torchao, compute = compute)
+ assert plan.offload_policy == dm.OFFLOAD_GROUP and plan.stream_transformer
+ assert "whole_module_kept_below_floor_mib" not in plan.estimates