Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: measured activations for Lumina-Image-2.0 (int8 stays resident on 16-24 GB cards, ~3% per render) (#16808)

Daniel Hanb75a4cd72146
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 88322b86a3b..32e182d9845 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -10937,7 +10937,7 @@ class DiffusionBackend:
except Exception: # noqa: BLE001 - unpriced: the unconditional switch
plan_steps = None
plan = prefer_whole_module_offload(
- pipe, plan, logger = logger, steps = plan_steps
+ pipe, plan, logger = logger, steps = plan_steps, family = fam.name
)
streamed_headroom = plan.estimates.get("measured_streamed_headroom_mib")
headroom = streamed_headroom or plan.estimates.get(
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 96d5bdeffbf..c2da4767502 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1673,9 +1673,14 @@ _MEASURED_IMAGE_ACTIVATION_MIB: dict[
"z-image": ((744, 1_199, 2_666, 2_456, 4_327), (744, 1_271, 2_669, 2_456, 4_327)),
# SDXL base, T4 fp16, 1024 (2048 tiled decode 2_455; untiled 7_706 left to the OOM retry); compiled unmeasured.
"sdxl": ((9, 327, 2_434, 2_434, 2_455), (9, 327, 2_434, 2_434, 2_455)),
+ # Lumina-Image-2.0, worst per column over tiers / A100 / L4; decode = the first compiled-tier decode (cudnn workspace).
+ "lumina-2": ((1_130, 486, 4_497, 4_497, 4_497), (1_130, 486, 4_497, 4_497, 4_497)),
}
# U-Nets cannot stream encoders on group tiers: the measurement only decides all-resident vs whole-module offload.
_RESIDENT_ONLY_CALIBRATED = frozenset({"sdxl"})
+# A plain bf16 denoiser keeps the whole-module tier the flat plan would take over the measured resident-transformer
+# tier: Lumina-2's encoder is as large as its DiT, and streaming it per prompt was 1% slower per render (L4, 16 GiB).
+_CALIBRATED_KEEPS_WHOLE_MODULE = frozenset({"lumina-2"})
def _resident_only_hint(family: Optional[str]) -> bool:
@@ -2904,7 +2909,10 @@ def plan_diffusion_memory(
stream_transformer = stream_transformer,
)
if faster is not None:
+ flat_streamed = policy == OFFLOAD_GROUP and stream_transformer
policy, stream_text_encoders, stream_transformer, tile, reason = faster
+ if flat_streamed and policy == OFFLOAD_GROUP and not stream_transformer:
+ estimates["calibrated_over_streamed_transformer"] = 1
any_offload = policy != OFFLOAD_NONE
reasons.append(reason)
estimates["calibrated_headroom_mib"] = calibrated_activation.headroom(tile)
@@ -4276,10 +4284,12 @@ def prefer_whole_module_offload(
*,
logger: Any = None,
steps: Optional[int] = None,
+ family: Optional[str] = None,
) -> MemoryPlan:
"""``refined``, or whole-module offload in its place where it beats streaming. With ``steps`` the switch is
priced from this card's measured bandwidth and bf16 throughput (RTX PRO 6000 streams faster at 4 steps, B200 does
not); unmeasurable or without ``steps`` it is kept. Plain bf16 only: an fp16 T4 run pointed the other way.
+ A ``_CALIBRATED_KEEPS_WHOLE_MODULE`` family's measured plan is priced as the streamed plan it replaced.
"""
switch = (os.environ.get(WHOLE_MODULE_OFFLOAD_ENV) or "auto").strip().lower()
if switch in ("0", "off", "false", "no", "never"):
@@ -4288,11 +4298,16 @@ def prefer_whole_module_offload(
if (
getattr(refined, "requested_mode", None) != MEMORY_MODE_AUTO
or refined.offload_policy != OFFLOAD_GROUP
- or not bool(getattr(refined, "stream_transformer", True))
or getattr(refined, "secondary_ordinal", None) is not None
):
return refined
est = refined.estimates
+ calibrated = bool(
+ str(family or "") in _CALIBRATED_KEEPS_WHOLE_MODULE
+ and est.get("calibrated_over_streamed_transformer")
+ )
+ if not calibrated and not bool(getattr(refined, "stream_transformer", True)):
+ return refined
model_dense, companions = est.get("model_dense_mib"), est.get("companion_dense_mib")
floors = [
int(f)
@@ -4312,7 +4327,7 @@ def prefer_whole_module_offload(
# A refined plan pinning the whole denoiser runs every step resident: kept.
sizes = _loaded_component_mib(pipe) or {}
dit = sum(m for m, r in sizes.values() if r == "dit") or transformer
- if int(getattr(refined, "resident_transformer_mib", None) or 0) >= dit:
+ if not calibrated and int(getattr(refined, "resident_transformer_mib", None) or 0) >= dit:
return refined
reason = _WHOLE_MODULE_KEPT_REASON.format(dit = transformer, floor = floor)
priced: dict = {}
@@ -4347,6 +4362,8 @@ def prefer_whole_module_offload(
declined = _WHOLE_MODULE_DECLINED_REASON.format(**fields)
if logger is not None:
logger.info("diffusion.memory: %s", declined)
+ if calibrated:
+ return refined
if per_request_whole_enabled():
try:
setattr(
@@ -4364,6 +4381,10 @@ def prefer_whole_module_offload(
reason += _WHOLE_MODULE_COST_REASON.format(**fields)
if logger is not None:
logger.info("diffusion.memory: %s", reason)
+ kept = dict(refined.estimates)
+ if calibrated:
+ # the generate guard prices it as main's flat whole-module plan
+ kept.pop("calibrated_headroom_mib", None)
return replace(
refined,
offload_policy = OFFLOAD_MODEL,
@@ -4374,7 +4395,7 @@ def prefer_whole_module_offload(
resident_transformer_mib = None,
resident_text_encoder_mib = None,
reasons = refined.reasons + (reason,),
- estimates = {**refined.estimates, "whole_module_kept_below_floor_mib": floor, **priced},
+ estimates = {**kept, "whole_module_kept_below_floor_mib": floor, **priced},
)
except Exception as exc: # noqa: BLE001 - keep the streamed refinement
if logger is not None:
diff --git a/studio/backend/tests/test_diffusion_calibrated_activation.py b/studio/backend/tests/test_diffusion_calibrated_activation.py
index b74ad1cd2d2..d1c9ee79758 100644
--- a/studio/backend/tests/test_diffusion_calibrated_activation.py
+++ b/studio/backend/tests/test_diffusion_calibrated_activation.py
@@ -660,3 +660,90 @@ def test_controlnet_surcharge_still_applies_to_transformer_families_named_like_s
_guard(6 * GIB, 24 * GIB, calibrated = True, controlnet = True, **kw).action
== dm.ACTIVATION_REFUSE
)
+
+
+# Real Colab L4 load of Alpha-VLLM/Lumina-Image-2.0 at bf16 (transformer_quant none): the planner inputs it logged.
+LUMINA2_BF16 = dict(model_dense_mib = 10_124, companion_dense_mib = 5_149, text_encoder_dense_mib = 4_959)
+LUMINA2_INT8 = dict(LUMINA2_BF16, model_dense_mib = 7_876)
+
+
+def _l4_plan(sizes, act):
+ """A Colab L4 as Studio read it (22_563 MiB total, 20_321 MiB budget): flat bf16 needs 20_364 MiB."""
+ return plan_diffusion_memory(
+ target = _target(),
+ device_memory = DeviceMemory("cuda", "cuda", "discrete_vram", 22_369, 22_563),
+ runtime_headroom_mib = 8192,
+ calibrated_activation = act,
+ **sizes,
+ )
+
+
+def test_lumina_2_measured_activation_keeps_the_int8_transformer_resident_on_16gb():
+ """The flat 8 GiB guess streamed Lumina-2's int8 transformer every step on 16 GB cards; the measured peak fits it."""
+ act = calibrated_image_activation("lumina-2")
+ flat, plan = _plan(16, LUMINA2_INT8, None), _plan(16, LUMINA2_INT8, act)
+ assert flat.offload_policy == OFFLOAD_GROUP and flat.stream_transformer
+ assert plan.offload_policy == OFFLOAD_GROUP
+ assert plan.stream_transformer is False and plan.stream_text_encoders is True
+ assert not plan.vae_tiling
+
+
+def test_lumina_2_measured_activation_keeps_int8_resident_on_an_l4():
+ """On a 22.5 GB L4 the flat guess streamed the encoder every render; with an int8 DiT everything now fits. bf16
+ keeps streaming the encoder: the measured first-decode peak leaves it short of resident."""
+ act = calibrated_image_activation("lumina-2", max_speed = False)
+ for sizes in (LUMINA2_BF16, LUMINA2_INT8):
+ flat = _l4_plan(sizes, None)
+ assert flat.offload_policy == OFFLOAD_GROUP and flat.stream_text_encoders
+ plan = _l4_plan(LUMINA2_INT8, act)
+ assert plan.offload_policy == OFFLOAD_NONE and not plan.vae_tiling
+ assert _l4_plan(LUMINA2_BF16, act).offload_policy == OFFLOAD_GROUP
+
+
+def test_lumina_2_measured_activation_never_picks_a_slower_tier():
+ for sizes in (LUMINA2_BF16, LUMINA2_INT8):
+ act = calibrated_image_activation("lumina-2")
+ best = None
+ for gib in (8, 10, 12, 14, 15, 16, 18, 20, 22, 24, 32, 48):
+ flat, plan = _plan(gib, sizes, None), _plan(gib, sizes, act)
+ rank = _rank(plan, sizes)
+ assert rank <= _rank(flat, sizes), (gib, plan.reasons)
+ assert best is None or rank <= best, gib
+ best = rank
+
+
+def _whole_module(
+ monkeypatch,
+ plan,
+ *,
+ family = "lumina-2",
+ dense_bf16 = True,
+ h2d = 11.0,
+ tflops = 76.0,
+):
+ """``prefer_whole_module_offload`` on ``plan`` with the probe the L4 load logged (11 GiB/s, 76 bf16 TFLOPS)."""
+ monkeypatch.delenv(dm.WHOLE_MODULE_OFFLOAD_ENV, raising = False)
+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: (h2d, tflops))
+ monkeypatch.setattr(dm, "_denoiser_is_dense_bf16", lambda pipe: dense_bf16)
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: {})
+ pipe = types.SimpleNamespace(transformer = object())
+ return pipe, dm.prefer_whole_module_offload(pipe, plan, steps = 50, family = family)
+
+
+def test_lumina_2_bf16_keeps_the_whole_module_offload_the_flat_plan_took(monkeypatch):
+ """16 GiB, plain bf16: the flat streamed plan became whole-module offload; the measured resident-transformer plan
+ (encoder streamed per prompt) was 1% slower per render on an L4, so the same whole-module plan is kept."""
+ act = calibrated_image_activation("lumina-2", max_speed = False)
+ flat, plan = _plan(16, LUMINA2_BF16, None), _plan(16, LUMINA2_BF16, act)
+ assert flat.stream_transformer and not plan.stream_transformer
+ _, flat_whole = _whole_module(monkeypatch, flat)
+ _, kept = _whole_module(monkeypatch, plan)
+ assert flat_whole.offload_policy == kept.offload_policy == OFFLOAD_MODEL
+ for field in ("stream_transformer", "stream_text_encoders", "vae_tiling", "vae_slicing"):
+ assert getattr(kept, field) == getattr(flat_whole, field), field
+ assert "calibrated_headroom_mib" not in kept.estimates
+ # int8 (not plain bf16), another family, or a card that streams cheaper: the measured plan stands
+ for kw in (dict(dense_bf16 = False), dict(family = "z-image"), dict(h2d = 200.0)):
+ pipe, other = _whole_module(monkeypatch, plan, **kw)
+ assert other is plan, kw
+ assert getattr(pipe, dm.WHOLE_TIER_ATTR, None) is None