Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: keep whole-module offload for a bf16 denoiser past the streamed floor (#15129)
Daniel Han609524dec1bd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 73be7760f2d..7a76663a8c4 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -220,6 +220,7 @@ from .diffusion_memory import (refine_balanced_plan_for_components,refine_memory_plan_for_components,refine_plan_from_loaded_weights,+ prefer_whole_module_offload,learned_peak_label,CFG_OFF_PART,cfg_off_peaks_enabled,@@ -10400,6 +10401,7 @@ class DiffusionBackend:cache = peak_cache,graphs = peak_graphs,)+ plan = prefer_whole_module_offload(pipe, plan, logger = logger)headroom = plan.estimates.get("measured_runtime_headroom_mib")if headroom and (plan.resident_transformer_mib or plan.resident_text_encoder_mibdiff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex eae23158103..6569a16e2dd 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -3930,6 +3930,99 @@ def refine_plan_from_loaded_weights(return refined+def _denoiser_is_dense_bf16(pipe: Any) -> bool:+ """A plain bf16 denoiser: no torchao, packed (GGUF / int8) or fp8-stored weights."""+ try:+ import torch++ module = next(+ (+ getattr(pipe, n, None)+ for n in ("transformer", "unet")+ if getattr(pipe, n, None) is not None+ ),+ None,+ )+ if module is None or getattr(module, "dtype", None) != torch.bfloat16:+ return False+ if _pipe_denoisers_hold_torchao(pipe) or _pipe_denoisers_hold_packed(pipe):+ return False+ return not any(+ _holds_fp8_stored(getattr(pipe, n, None))+ for n in ("transformer", "transformer_2", "unet")+ )+ except Exception: # noqa: BLE001 - unreadable: keep the streamed plan+ return False+++_WHOLE_MODULE_KEPT_REASON = (+ "whole-module offload ran the {dit} MiB denoiser whole below the streamed tiers' {floor} MiB floor; it still "+ "fits, and uploading it once per call beats streaming it every step"+)+++def prefer_whole_module_offload(+ pipe: Any,+ refined: MemoryPlan,+ *,+ logger: Any = None,+) -> MemoryPlan:+ """``refined`` (what ``refine_plan_from_loaded_weights`` returned), or whole-module offload in its place.++ Below the streamed tiers' floor whole-module offload holds the whole denoiser on the card while it runs; a few+ hundred MiB more free VRAM used to switch to streaming all of it every step, but uploading it once per call beats+ that (FLUX.2-klein-4B bf16, B200: 0.34 vs 0.60 s per image). Plain bf16 only: an fp16 T4 run pointed the other way."""+ try:+ if (+ getattr(refined, "requested_mode", None) != MEMORY_MODE_AUTO+ or refined.offload_policy != OFFLOAD_GROUP+ or not bool(getattr(refined, "stream_transformer", True))+ or getattr(refined, "secondary_ordinal", None) is not None+ ):+ return refined+ est = refined.estimates+ model_dense, companions = est.get("model_dense_mib"), est.get("companion_dense_mib")+ floors = [+ int(f)+ for f in (est.get("group_floor_mib"), est.get("group_floor_streamed_te_mib"))+ if f is not None+ ]+ if model_dense is None or companions is None or not floors:+ return refined+ transformer = int(model_dense) - int(companions)+ te = max(0, int(est.get("text_encoder_dense_mib") or 0))+ others = max(0, int(companions) - te)+ floor = min(floors)+ if transformer <= 0 or max(transformer, te, others) >= floor:+ return refined+ if not _denoiser_is_dense_bf16(pipe):+ return refined+ # A refined plan pinning the whole denoiser runs every step resident: kept.+ sizes = _loaded_component_mib(pipe) or {}+ dit = sum(m for m, r in sizes.values() if r == "dit") or transformer+ if int(getattr(refined, "resident_transformer_mib", None) or 0) >= dit:+ return refined+ reason = _WHOLE_MODULE_KEPT_REASON.format(dit = transformer, floor = floor)+ if logger is not None:+ logger.info("diffusion.memory: %s", reason)+ return replace(+ refined,+ offload_policy = OFFLOAD_MODEL,+ stream_text_encoders = False,+ stream_transformer = True,+ vae_tiling = True,+ vae_slicing = True,+ resident_transformer_mib = None,+ resident_text_encoder_mib = None,+ reasons = refined.reasons + (reason,),+ estimates = {**refined.estimates, "whole_module_kept_below_floor_mib": floor},+ )+ except Exception as exc: # noqa: BLE001 - keep the streamed refinement+ if logger is not None:+ logger.debug("diffusion.memory: whole-module comparison skipped (%s)", exc)+ return refined++def _refine_plan_from_loaded_weights(pipe: Any,plan: MemoryPlan,diff --git a/studio/backend/tests/test_diffusion_placement_monotonic.py b/studio/backend/tests/test_diffusion_placement_monotonic.pyindex 7499eab4fe1..43a5951e903 100644--- a/studio/backend/tests/test_diffusion_placement_monotonic.py+++ b/studio/backend/tests/test_diffusion_placement_monotonic.py@@ -93,7 +93,10 @@ def _plan(speed_mode = "default",torchao = True,calibrated = None,+ compute = None,):+ import torch+plan = dm.plan_diffusion_memory(target = types.SimpleNamespace(supports_model_cpu_offload = True),device_memory = _discrete(free_mib, total_mib),@@ -102,9 +105,10 @@ def _plan()if plan.offload_policy == dm.OFFLOAD_MODEL and torchao:plan = dm.torchao_streaming_plan(plan)- return dm.refine_plan_from_loaded_weights(- types.SimpleNamespace(transformer = object()), plan, family = family, speed_mode = speed_mode- )+ dtype = torch.float16 if compute == 2 else torch.bfloat16+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = dtype))+ refined = dm.refine_plan_from_loaded_weights(pipe, plan, family = family, speed_mode = speed_mode)+ return dm.prefer_whole_module_offload(pipe, refined)def _denoiser_key(plan, dit_mib):@@ -148,7 +152,7 @@ def test_more_free_vram_never_keeps_less_of_the_denoiser(monkeypatch, family, qutotal = gib * GIBprevious = Nonefor free in range(4 * GIB, min(80, gib) * GIB + 1, STEP_MIB):- plan = _plan(free, total, flat, family = family, torchao = torchao)+ plan = _plan(free, total, flat, family = family, torchao = torchao, compute = compute)key = _denoiser_key(plan, dit)if key is None:continue@@ -203,3 +207,175 @@ def test_explicit_modes_are_untouched(monkeypatch, mode):object(), plan, family = "flux.1", speed_mode = "default")assert refined is plan+++def _speed_key(plan, dit_mib):+ """(tier, -resident denoiser MiB), smaller is faster: 0 the whole denoiser stays on the device every step, 1+ whole-module offload with the denoiser fitting the budget whole (uploaded once per call), 2 the denoiser streams+ every step, 3 whole-module offload of a denoiser larger than the budget."""+ if plan.offload_policy == dm.OFFLOAD_NONE or (+ plan.offload_policy == dm.OFFLOAD_GROUP and not plan.stream_transformer+ ):+ return (0, -dit_mib)+ if plan.offload_policy == dm.OFFLOAD_MODEL:+ fits = dit_mib <= int(plan.estimates.get("safe_device_budget_mib") or 0)+ return (1, 0) if fits else (3, 0)+ resident = int(plan.resident_transformer_mib or 0)+ return (0, -dit_mib) if resident >= dit_mib else (2, -resident)++[email protected]("calibrated", (False, True))[email protected]("learned", (None, 3072))[email protected]("quant", ("bf16",))[email protected]("family", sorted(FAMILIES))+def test_more_free_vram_never_trades_whole_module_offload_for_streaming(+ monkeypatch, family, quant, learned, calibrated+):+ """Below the streamed tiers' floor whole-module offload holds the whole denoiser; a few hundred MiB more used to+ stream it every step instead (FLUX.2-klein-4B bf16 on B200: 0.34 s -> 0.60 s per 4-step 1024^2 image)."""+ q8, dense, encoders, vae = FAMILIES[family]+ eight, torchao, fp8_stored, compute = {**QUANTS, "bf16": (False, False, False, None)}[quant]+ dit = q8 if eight else dense+ te = sum(encoders.values())+ loaded = {+ "transformer": (dit, "dit"),+ **{name: (mib, "text_encoder") for name, mib in encoders.items()},+ "vae": (vae, "other"),+ }+ _patch_weights(+ monkeypatch,+ loaded,+ torchao = torchao,+ fp8_stored = fp8_stored,+ compute = compute,+ learned = learned,+ )+ flat = dict(+ model_dense_mib = dit + te + vae,+ companion_dense_mib = te + vae,+ text_encoder_dense_mib = te,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(width = None, height = None, family = family),+ )+ act = dm.calibrated_image_activation(family) if calibrated else None+ if calibrated and act is None:+ pytest.skip("no calibrated activation for this family")+ previous = None+ for free in range(4 * GIB, 24 * GIB + 1, STEP_MIB):+ plan = _plan(+ free, 24 * GIB, flat, family = family, torchao = torchao, calibrated = act, compute = compute+ )+ key = _speed_key(plan, dit)+ if previous is not None:+ assert key <= previous[1], (free, previous, key, plan.reasons)+ previous = (free, key)+++def test_bf16_klein_keeps_whole_module_offload_past_the_streamed_floor(monkeypatch):+ q8, dense, encoders, vae = FAMILIES["flux.2-klein"]+ te = sum(encoders.values())+ loaded = {+ "transformer": (dense, "dit"),+ "text_encoder": (te, "text_encoder"),+ "vae": (vae, "other"),+ }+ _patch_weights(monkeypatch, loaded, torchao = False)+ flat = dict(+ model_dense_mib = dense + te + vae,+ companion_dense_mib = te + vae,+ text_encoder_dense_mib = te,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(+ width = None, height = None, family = "flux.2-klein"+ ),+ )+ below = _plan(12288, 24 * GIB, flat, family = "flux.2-klein", torchao = False)+ above = _plan(12544, 24 * GIB, flat, family = "flux.2-klein", torchao = False)+ assert below.offload_policy == dm.OFFLOAD_MODEL+ assert "whole_module_kept_below_floor_mib" not in below.estimates+ assert above.offload_policy == dm.OFFLOAD_MODEL+ assert above.estimates["whole_module_kept_below_floor_mib"] == 10400+ assert any("uploading it once per call" in reason for reason in above.reasons)+ # The resident-transformer tier still wins once it fits.+ resident = _plan(20 * GIB, 24 * GIB, flat, family = "flux.2-klein", torchao = False)+ assert resident.offload_policy == dm.OFFLOAD_GROUP and not resident.stream_transformer+++def test_streamed_plans_that_pin_the_whole_denoiser_or_hold_torchao_are_kept(monkeypatch):+ """A refined group plan already holding the whole denoiser beats whole-module offload; a torchao denoiser's+ whole-module tier is streaming, so its ladder is unchanged."""+ q8, dense, encoders, vae = FAMILIES["flux.2-klein"]+ te = sum(encoders.values())+ loaded = {+ "transformer": (q8, "dit"),+ "text_encoder": (te, "text_encoder"),+ "vae": (vae, "other"),+ }+ flat = dict(+ model_dense_mib = q8 + te + vae,+ companion_dense_mib = te + vae,+ text_encoder_dense_mib = te,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(+ width = None, height = None, family = "flux.2-klein"+ ),+ )+ _patch_weights(monkeypatch, loaded, torchao = False, fp8_stored = True, learned = 3072)+ pinned = _plan(14 * GIB, 24 * GIB, flat, family = "flux.2-klein", torchao = False)+ assert pinned.offload_policy == dm.OFFLOAD_GROUP and pinned.resident_transformer_mib == q8+ _patch_weights(monkeypatch, loaded, torchao = True)+ streamed = _plan(14 * GIB, 24 * GIB, flat, family = "flux.2-klein", torchao = True)+ assert streamed.offload_policy == dm.OFFLOAD_GROUP+ assert "whole_module_kept_below_floor_mib" not in streamed.estimates+++def test_bf16_partial_residency_gives_way_to_whole_module_offload(monkeypatch):+ """A learned peak pins 4.7 of the 7.4 GB bf16 denoiser past the floor; whole-module offload still holds all of it+ (B200, 12.25 GiB free: 0.37 s whole-module vs 0.73 s with 5.5 GB pinned)."""+ q8, dense, encoders, vae = FAMILIES["flux.2-klein"]+ te = sum(encoders.values())+ loaded = {+ "transformer": (dense, "dit"),+ "text_encoder": (te, "text_encoder"),+ "vae": (vae, "other"),+ }+ flat = dict(+ model_dense_mib = dense + te + vae,+ companion_dense_mib = te + vae,+ text_encoder_dense_mib = te,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(+ width = None, height = None, family = "flux.2-klein"+ ),+ )+ _patch_weights(monkeypatch, loaded, torchao = False, learned = 3072)+ plan = _plan(12544, 24 * GIB, flat, family = "flux.2-klein", torchao = False)+ assert plan.offload_policy == dm.OFFLOAD_MODEL+ # the same learned peak on an fp16 denoiser (the T4 dtype) keeps its partial residency+ _patch_weights(monkeypatch, loaded, torchao = False, compute = 2, learned = 3072)+ fp16 = _plan(12544, 24 * GIB, flat, family = "flux.2-klein", torchao = False, compute = 2)+ assert fp16.offload_policy == dm.OFFLOAD_GROUP and 0 < fp16.resident_transformer_mib < dense+++ "family, quant",+ [("flux.2-klein", "fp16"), ("flux.2-klein", "fp8_stored"), ("z-image", "fp8_stored")],+)+def test_unmeasured_denoisers_keep_the_streamed_plan(monkeypatch, family, quant):+ """fp16 (the T4 dtype, where stock whole-module offload measured slower than streaming), fp8-stored and packed+ denoisers keep the streamed group plan past the floor: only bf16 was measured faster whole-module."""+ q8, dense, encoders, vae = FAMILIES[family]+ eight, torchao, fp8_stored, compute = QUANTS[quant]+ dit = q8 if eight else dense+ te = sum(encoders.values())+ loaded = {+ "transformer": (dit, "dit"),+ "text_encoder": (te, "text_encoder"),+ "vae": (vae, "other"),+ }+ _patch_weights(monkeypatch, loaded, torchao = torchao, fp8_stored = fp8_stored, compute = compute)+ flat = dict(+ model_dense_mib = dit + te + vae,+ companion_dense_mib = te + vae,+ text_encoder_dense_mib = te,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(width = None, height = None, family = family),+ )+ plan = _plan(12544, 24 * GIB, flat, family = family, torchao = torchao, compute = compute)+ assert plan.offload_policy == dm.OFFLOAD_GROUP and plan.stream_transformer+ assert "whole_module_kept_below_floor_mib" not in plan.estimates