Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: measured activations for Lumina-Image-2.0 (int8 stays resident on 16-24 GB cards, ~3% per render) (#16808)
Daniel Hanb75a4cd72146
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 88322b86a3b..32e182d9845 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -10937,7 +10937,7 @@ class DiffusionBackend:except Exception: # noqa: BLE001 - unpriced: the unconditional switchplan_steps = Noneplan = prefer_whole_module_offload(- pipe, plan, logger = logger, steps = plan_steps+ pipe, plan, logger = logger, steps = plan_steps, family = fam.name)streamed_headroom = plan.estimates.get("measured_streamed_headroom_mib")headroom = streamed_headroom or plan.estimates.get(diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 96d5bdeffbf..c2da4767502 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1673,9 +1673,14 @@ _MEASURED_IMAGE_ACTIVATION_MIB: dict["z-image": ((744, 1_199, 2_666, 2_456, 4_327), (744, 1_271, 2_669, 2_456, 4_327)),# SDXL base, T4 fp16, 1024 (2048 tiled decode 2_455; untiled 7_706 left to the OOM retry); compiled unmeasured."sdxl": ((9, 327, 2_434, 2_434, 2_455), (9, 327, 2_434, 2_434, 2_455)),+ # Lumina-Image-2.0, worst per column over tiers / A100 / L4; decode = the first compiled-tier decode (cudnn workspace).+ "lumina-2": ((1_130, 486, 4_497, 4_497, 4_497), (1_130, 486, 4_497, 4_497, 4_497)),}# U-Nets cannot stream encoders on group tiers: the measurement only decides all-resident vs whole-module offload._RESIDENT_ONLY_CALIBRATED = frozenset({"sdxl"})+# A plain bf16 denoiser keeps the whole-module tier the flat plan would take over the measured resident-transformer+# tier: Lumina-2's encoder is as large as its DiT, and streaming it per prompt was 1% slower per render (L4, 16 GiB).+_CALIBRATED_KEEPS_WHOLE_MODULE = frozenset({"lumina-2"})def _resident_only_hint(family: Optional[str]) -> bool:@@ -2904,7 +2909,10 @@ def plan_diffusion_memory(stream_transformer = stream_transformer,)if faster is not None:+ flat_streamed = policy == OFFLOAD_GROUP and stream_transformerpolicy, stream_text_encoders, stream_transformer, tile, reason = faster+ if flat_streamed and policy == OFFLOAD_GROUP and not stream_transformer:+ estimates["calibrated_over_streamed_transformer"] = 1any_offload = policy != OFFLOAD_NONEreasons.append(reason)estimates["calibrated_headroom_mib"] = calibrated_activation.headroom(tile)@@ -4276,10 +4284,12 @@ def prefer_whole_module_offload(*,logger: Any = None,steps: Optional[int] = None,+ family: Optional[str] = None,) -> MemoryPlan:"""``refined``, or whole-module offload in its place where it beats streaming. With ``steps`` the switch ispriced from this card's measured bandwidth and bf16 throughput (RTX PRO 6000 streams faster at 4 steps, B200 doesnot); unmeasurable or without ``steps`` it is kept. Plain bf16 only: an fp16 T4 run pointed the other way.+ A ``_CALIBRATED_KEEPS_WHOLE_MODULE`` family's measured plan is priced as the streamed plan it replaced."""switch = (os.environ.get(WHOLE_MODULE_OFFLOAD_ENV) or "auto").strip().lower()if switch in ("0", "off", "false", "no", "never"):@@ -4288,11 +4298,16 @@ def prefer_whole_module_offload(if (getattr(refined, "requested_mode", None) != MEMORY_MODE_AUTOor refined.offload_policy != OFFLOAD_GROUP- or not bool(getattr(refined, "stream_transformer", True))or getattr(refined, "secondary_ordinal", None) is not None):return refinedest = refined.estimates+ calibrated = bool(+ str(family or "") in _CALIBRATED_KEEPS_WHOLE_MODULE+ and est.get("calibrated_over_streamed_transformer")+ )+ if not calibrated and not bool(getattr(refined, "stream_transformer", True)):+ return refinedmodel_dense, companions = est.get("model_dense_mib"), est.get("companion_dense_mib")floors = [int(f)@@ -4312,7 +4327,7 @@ def prefer_whole_module_offload(# A refined plan pinning the whole denoiser runs every step resident: kept.sizes = _loaded_component_mib(pipe) or {}dit = sum(m for m, r in sizes.values() if r == "dit") or transformer- if int(getattr(refined, "resident_transformer_mib", None) or 0) >= dit:+ if not calibrated and int(getattr(refined, "resident_transformer_mib", None) or 0) >= dit:return refinedreason = _WHOLE_MODULE_KEPT_REASON.format(dit = transformer, floor = floor)priced: dict = {}@@ -4347,6 +4362,8 @@ def prefer_whole_module_offload(declined = _WHOLE_MODULE_DECLINED_REASON.format(**fields)if logger is not None:logger.info("diffusion.memory: %s", declined)+ if calibrated:+ return refinedif per_request_whole_enabled():try:setattr(@@ -4364,6 +4381,10 @@ def prefer_whole_module_offload(reason += _WHOLE_MODULE_COST_REASON.format(**fields)if logger is not None:logger.info("diffusion.memory: %s", reason)+ kept = dict(refined.estimates)+ if calibrated:+ # the generate guard prices it as main's flat whole-module plan+ kept.pop("calibrated_headroom_mib", None)return replace(refined,offload_policy = OFFLOAD_MODEL,@@ -4374,7 +4395,7 @@ def prefer_whole_module_offload(resident_transformer_mib = None,resident_text_encoder_mib = None,reasons = refined.reasons + (reason,),- estimates = {**refined.estimates, "whole_module_kept_below_floor_mib": floor, **priced},+ estimates = {**kept, "whole_module_kept_below_floor_mib": floor, **priced},)except Exception as exc: # noqa: BLE001 - keep the streamed refinementif logger is not None:diff --git a/studio/backend/tests/test_diffusion_calibrated_activation.py b/studio/backend/tests/test_diffusion_calibrated_activation.pyindex b74ad1cd2d2..d1c9ee79758 100644--- a/studio/backend/tests/test_diffusion_calibrated_activation.py+++ b/studio/backend/tests/test_diffusion_calibrated_activation.py@@ -660,3 +660,90 @@ def test_controlnet_surcharge_still_applies_to_transformer_families_named_like_s_guard(6 * GIB, 24 * GIB, calibrated = True, controlnet = True, **kw).action== dm.ACTIVATION_REFUSE)+++# Real Colab L4 load of Alpha-VLLM/Lumina-Image-2.0 at bf16 (transformer_quant none): the planner inputs it logged.+LUMINA2_BF16 = dict(model_dense_mib = 10_124, companion_dense_mib = 5_149, text_encoder_dense_mib = 4_959)+LUMINA2_INT8 = dict(LUMINA2_BF16, model_dense_mib = 7_876)+++def _l4_plan(sizes, act):+ """A Colab L4 as Studio read it (22_563 MiB total, 20_321 MiB budget): flat bf16 needs 20_364 MiB."""+ return plan_diffusion_memory(+ target = _target(),+ device_memory = DeviceMemory("cuda", "cuda", "discrete_vram", 22_369, 22_563),+ runtime_headroom_mib = 8192,+ calibrated_activation = act,+ **sizes,+ )+++def test_lumina_2_measured_activation_keeps_the_int8_transformer_resident_on_16gb():+ """The flat 8 GiB guess streamed Lumina-2's int8 transformer every step on 16 GB cards; the measured peak fits it."""+ act = calibrated_image_activation("lumina-2")+ flat, plan = _plan(16, LUMINA2_INT8, None), _plan(16, LUMINA2_INT8, act)+ assert flat.offload_policy == OFFLOAD_GROUP and flat.stream_transformer+ assert plan.offload_policy == OFFLOAD_GROUP+ assert plan.stream_transformer is False and plan.stream_text_encoders is True+ assert not plan.vae_tiling+++def test_lumina_2_measured_activation_keeps_int8_resident_on_an_l4():+ """On a 22.5 GB L4 the flat guess streamed the encoder every render; with an int8 DiT everything now fits. bf16+ keeps streaming the encoder: the measured first-decode peak leaves it short of resident."""+ act = calibrated_image_activation("lumina-2", max_speed = False)+ for sizes in (LUMINA2_BF16, LUMINA2_INT8):+ flat = _l4_plan(sizes, None)+ assert flat.offload_policy == OFFLOAD_GROUP and flat.stream_text_encoders+ plan = _l4_plan(LUMINA2_INT8, act)+ assert plan.offload_policy == OFFLOAD_NONE and not plan.vae_tiling+ assert _l4_plan(LUMINA2_BF16, act).offload_policy == OFFLOAD_GROUP+++def test_lumina_2_measured_activation_never_picks_a_slower_tier():+ for sizes in (LUMINA2_BF16, LUMINA2_INT8):+ act = calibrated_image_activation("lumina-2")+ best = None+ for gib in (8, 10, 12, 14, 15, 16, 18, 20, 22, 24, 32, 48):+ flat, plan = _plan(gib, sizes, None), _plan(gib, sizes, act)+ rank = _rank(plan, sizes)+ assert rank <= _rank(flat, sizes), (gib, plan.reasons)+ assert best is None or rank <= best, gib+ best = rank+++def _whole_module(+ monkeypatch,+ plan,+ *,+ family = "lumina-2",+ dense_bf16 = True,+ h2d = 11.0,+ tflops = 76.0,+):+ """``prefer_whole_module_offload`` on ``plan`` with the probe the L4 load logged (11 GiB/s, 76 bf16 TFLOPS)."""+ monkeypatch.delenv(dm.WHOLE_MODULE_OFFLOAD_ENV, raising = False)+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: (h2d, tflops))+ monkeypatch.setattr(dm, "_denoiser_is_dense_bf16", lambda pipe: dense_bf16)+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: {})+ pipe = types.SimpleNamespace(transformer = object())+ return pipe, dm.prefer_whole_module_offload(pipe, plan, steps = 50, family = family)+++def test_lumina_2_bf16_keeps_the_whole_module_offload_the_flat_plan_took(monkeypatch):+ """16 GiB, plain bf16: the flat streamed plan became whole-module offload; the measured resident-transformer plan+ (encoder streamed per prompt) was 1% slower per render on an L4, so the same whole-module plan is kept."""+ act = calibrated_image_activation("lumina-2", max_speed = False)+ flat, plan = _plan(16, LUMINA2_BF16, None), _plan(16, LUMINA2_BF16, act)+ assert flat.stream_transformer and not plan.stream_transformer+ _, flat_whole = _whole_module(monkeypatch, flat)+ _, kept = _whole_module(monkeypatch, plan)+ assert flat_whole.offload_policy == kept.offload_policy == OFFLOAD_MODEL+ for field in ("stream_transformer", "stream_text_encoders", "vae_tiling", "vae_slicing"):+ assert getattr(kept, field) == getattr(flat_whole, field), field+ assert "calibrated_headroom_mib" not in kept.estimates+ # int8 (not plain bf16), another family, or a card that streams cheaper: the measured plan stands+ for kw in (dict(dense_bf16 = False), dict(family = "z-image"), dict(h2d = 200.0)):+ pipe, other = _whole_module(monkeypatch, plan, **kw)+ assert other is plan, kw+ assert getattr(pipe, dm.WHOLE_TIER_ATTR, None) is None