Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: keep SDXL resident on 15-16 GB cards from its measured activations (#15144)

Daniel Hane3d92b48e549
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 6569a16e2dd..e70f612f9b0 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1559,6 +1559,7 @@ class CalibratedImageActivation:
decode_mib: int
tiled_decode_mib: int
max_canvas_mib: int
+ resident_only: bool = False
def headroom(self, tiled: bool) -> int:
return max(
@@ -1591,7 +1592,11 @@ _MEASURED_IMAGE_ACTIVATION_MIB: dict[
# MOSS-Image SFT, B200 bf16 measurements; both tiers take the eager worst.
"moss-image": ((1_940, 528, 3_666, 3_662, 3_750), (1_940, 528, 3_666, 3_662, 3_750)),
"z-image": ((744, 1_199, 2_666, 2_456, 4_327), (744, 1_271, 2_669, 2_456, 4_327)),
+ # SDXL base, T4 fp16, 1024 (2048 tiled decode 2_455; untiled 7_706 left to the OOM retry); compiled unmeasured.
+ "sdxl": ((9, 327, 2_434, 2_434, 2_455), (9, 327, 2_434, 2_434, 2_455)),
}
+# U-Nets cannot stream encoders on group tiers: the measurement only decides all-resident vs whole-module offload.
+_RESIDENT_ONLY_CALIBRATED = frozenset({"sdxl"})
def calibrated_image_activation(
@@ -1601,7 +1606,8 @@ def calibrated_image_activation(
if measured is None:
return None
return CalibratedImageActivation(
- *(int(v * _ACTIVATION_MARGIN) for v in measured[1 if max_speed else 0])
+ *(int(v * _ACTIVATION_MARGIN) for v in measured[1 if max_speed else 0]),
+ resident_only = str(family or "") in _RESIDENT_ONLY_CALIBRATED,
)
@@ -2739,10 +2745,17 @@ def plan_diffusion_memory(
policy = OFFLOAD_NONE
fast_budget = _fast_device_budget_mib(device_memory)
estimates["resident_budget_mib"] = fast_budget
- if fast_budget is not None and required is not None and required > fast_budget:
+ fast_required = required
+ if calibrated_activation is not None and calibrated_activation.resident_only:
+ # the flat 8 GiB guess demoted an explicit fast SDXL on 15-16 GB cards, where it fits with room to spare
+ fast_required = _sum_required(
+ model_dense_mib, calibrated_activation.headroom(False), base_overhead_mib
+ )
+ estimates["calibrated_headroom_mib"] = calibrated_activation.headroom(False)
+ if fast_budget is not None and fast_required is not None and fast_required > fast_budget:
policy, stream_text_encoders, stream_transformer = _offload_tier()
reasons.append(
- f"fast requested but weights do not fit resident ({required} MiB needed, "
+ f"fast requested but weights do not fit resident ({fast_required} MiB needed, "
f"{fast_budget} MiB free after the reserve); offloading"
)
if not stream_transformer:
@@ -2979,6 +2992,22 @@ def _calibrated_faster_tier(
budget = int(budget)
free = int(free_mib) - overhead
room = free - act.max_canvas_mib
+ if act.resident_only:
+ # Larger requests are still checked by the generation guard; an oversized decode is tiled by its OOM retry.
+ weights = transformer + te + others
+ if (
+ policy != OFFLOAD_NONE
+ and weights + act.headroom(False) + overhead <= budget
+ and weights <= room
+ ):
+ return (
+ OFFLOAD_NONE,
+ False,
+ True,
+ False,
+ "measured activations fit resident; the U-Net's only other tier uploads every component per call",
+ )
+ return None
if policy == OFFLOAD_NONE:
flat_rank = 0
elif policy == OFFLOAD_GROUP and not stream_transformer:
diff --git a/studio/backend/tests/test_diffusion_calibrated_activation.py b/studio/backend/tests/test_diffusion_calibrated_activation.py
index 7839cf3d8d6..b8fb88d2362 100644
--- a/studio/backend/tests/test_diffusion_calibrated_activation.py
+++ b/studio/backend/tests/test_diffusion_calibrated_activation.py
@@ -228,7 +228,8 @@ def test_only_measured_families_are_calibrated():
assert calibrated_image_activation("hidream-i1") is None
assert calibrated_image_activation("qwen-image-edit") is None
assert calibrated_image_activation("qwen-image") is None
- assert calibrated_image_activation("sdxl") is None
+ assert calibrated_image_activation("sdxl").resident_only
+ assert not calibrated_image_activation("flux.1").resident_only
for family, tiers in dm._MEASURED_IMAGE_ACTIVATION_MIB.items():
for max_speed, raw in zip((False, True), tiers):
act = calibrated_image_activation(family, max_speed = max_speed)
@@ -580,3 +581,47 @@ def test_generate_passes_the_calibrated_placement_and_controlnet_to_the_guard():
src = inspect.getsource(d)
assert 'calibrated_placement = bool(getattr(state, "calibrated_placement", False)),' in src
assert 'controlnet = workflow == "controlnet" and control_pil is not None,' in src
+
+
+# Real Tesla T4 load of stabilityai/stable-diffusion-xl-base-1.0 at fp16: the planner inputs it logged.
+SDXL_FP16 = dict(model_dense_mib = 6_775, companion_dense_mib = 6_775, text_encoder_dense_mib = 1_559)
+
+
+def _t4_plan(act, mode = None):
+ return plan_diffusion_memory(
+ target = _target(),
+ device_memory = DeviceMemory("cuda", "cuda", "discrete_vram", 14_807, 14_912),
+ runtime_headroom_mib = 8192,
+ requested_mode = mode,
+ calibrated_activation = act,
+ **SDXL_FP16,
+ )
+
+
+def test_sdxl_measured_activation_keeps_a_t4_resident():
+ """The flat 8 GiB guess offloaded (and demoted explicit fast) SDXL on a 15 GB card; the measured peak fits."""
+ act = calibrated_image_activation("sdxl")
+ assert _t4_plan(None).offload_policy == OFFLOAD_MODEL
+ assert _t4_plan(None, MEMORY_MODE_FAST).offload_policy == OFFLOAD_MODEL
+ for mode in (None, MEMORY_MODE_FAST):
+ plan = _t4_plan(act, mode)
+ assert plan.offload_policy == OFFLOAD_NONE, mode
+ assert not plan.vae_tiling and not plan.vae_slicing
+ assert _t4_plan(act, MEMORY_MODE_LOW_VRAM).offload_policy == OFFLOAD_MODEL
+ assert _t4_plan(act, MEMORY_MODE_BALANCED).offload_policy == OFFLOAD_GROUP
+
+
+def test_sdxl_measured_activation_only_ever_adds_the_resident_tier():
+ """Only whole-module offload may become resident (never a group tier); more VRAM never picks a slower tier."""
+ act = calibrated_image_activation("sdxl")
+ resident_from = None
+ for gib in (6, 8, 10, 12, 14, 15, 16, 20, 24, 48):
+ flat, plan = _plan(gib, SDXL_FP16, None), _plan(gib, SDXL_FP16, act)
+ if plan.offload_policy != flat.offload_policy:
+ assert plan.offload_policy == OFFLOAD_NONE, gib
+ if plan.offload_policy == OFFLOAD_NONE and resident_from is None:
+ resident_from = gib
+ if resident_from is not None:
+ assert plan.offload_policy == OFFLOAD_NONE, gib
+ assert resident_from == 15
+ assert _plan(12, SDXL_FP16, act).offload_policy == OFFLOAD_MODEL