Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: seed the dense bf16 placement from FLUX.1's measured peak (#15149)

Daniel Han68c1c159c768
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 1b188d0bda7..1f031033d9f 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -10444,6 +10444,7 @@ class DiffusionBackend:
plan.estimates.get("measured_learned_peak_quadratic"),
plan.estimates.get("measured_static_seed", True),
plan.estimates.get("measured_learned_cfg_off", False),
+ plan.estimates.get("measured_dense_bf16", False),
)
setattr(pipe, DECODE_LEND_ATTR, decode_lend_from_plan(plan))
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index e70f612f9b0..3baec144685 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -3330,19 +3330,52 @@ _MEASURED_DENSE_SPEED_MODES = ("off",)
MEASURED_ACTIVATION_DENSE_TIERS_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE_TIERS"
+# Dense bf16, eager tier: worst recorded whole-render peak at 1024x1024, all groups streamed (what a learned peak holds).
+# family -> (peak MiB, largest loaded DiT MiB it covers). "0" = learned peak alone.
+MEASURED_ACTIVATION_BF16_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16"
+_MEASURED_DENSE_BF16_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {
+ # FLUX.1-schnell, Colab L4, 4 steps: 3417 (4 loads), 3581 (1 load).
+ "flux.1": (3581, 23800),
+ # Not seeded: FLUX.2-klein-4B's 2446 used a tiled decode; Qwen-Image-2.1's 5486 was recorded resident, unstreamed.
+}
+
+
def _env_off(name: str) -> bool:
return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")
+def dense_bf16_seeded(family: Optional[str]) -> bool:
+ return not _env_off(MEASURED_ACTIVATION_BF16_ENV) and (
+ str(family or "").lower() in _MEASURED_DENSE_BF16_EAGER_PEAK_MIB
+ )
+
+
def _dense_seed_mib(
- family: str, speed_mode: str, dense_mib: int, widen: int, tier_seeds: bool
+ family: str,
+ speed_mode: str,
+ dense_mib: int,
+ widen: int,
+ tier_seeds: bool,
+ bf16: bool = False,
) -> Optional[int]:
"""Seed MiB for a dense fp16 / fp32 denoiser (``widen`` = its element size), or None. Eager: the dense table. Default
/ max: the larger of the dense table and the tier's worst phase (encoder, denoise, untiled decode), only for a
family the dense table measured at this DiT size. Families it lacks keep the flat estimate: on a T4, Z-Image's 11.5
GiB DiT would then be kept partly resident, which ran slower per step than streaming it whole (3.8-4.4 vs 3.2-3.6
s/step). ``tier_seeds``: the target is where the phase table was measured (NVIDIA with a sub-quadratic attention
- kernel, ``diffusion._measured_target``); elsewhere (math-only attention, ROCm) the tiers keep the flat estimate."""
+ kernel, ``diffusion._measured_target``); elsewhere (math-only attention, ROCm) the tiers keep the flat estimate.
+ ``bf16``: the bf16 table, eager tier and ``tier_seeds`` targets only (all it was measured on)."""
+ if bf16:
+ entry = _MEASURED_DENSE_BF16_EAGER_PEAK_MIB.get(family)
+ if (
+ entry is None
+ or not tier_seeds
+ or _env_off(MEASURED_ACTIVATION_BF16_ENV)
+ or speed_mode not in _MEASURED_DENSE_SPEED_MODES
+ or not 0 < dense_mib <= entry[1]
+ ):
+ return None
+ return entry[0]
entry = _MEASURED_DENSE_EAGER_PEAK_MIB.get(family)
if entry is None or not 0 < dense_mib <= entry[1] * widen // 2:
return None
@@ -3374,6 +3407,7 @@ def measured_image_runtime_mib(
static_seed: bool = True,
dense_tier_seeds: bool = False,
phase: Optional[str] = None,
+ dense_bf16: bool = False,
) -> Optional[int]:
"""Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate).
``phase`` (``diffusion_measured_peaks.PHASES``) reads that phase's learned peak instead of the whole render's.
@@ -3399,6 +3433,7 @@ def measured_image_runtime_mib(
int(dense_transformer_mib),
max(2, int(compute_bytes or 2)),
dense_tier_seeds,
+ bf16 = dense_bf16,
)
elif static_seed and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES:
seed = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())
@@ -4095,17 +4130,24 @@ def _refine_plan_from_loaded_weights(
return plan
dense_mib = None if _pipe_denoisers_hold_torchao(pipe) else dit
compute_bytes = _denoiser_compute_bytes(pipe)
- # The dense table covers fp16 / fp32 unpacked weights only; everything else uses a learned peak alone.
- # Stored fp8 layers widen per forward, which no static table priced, even beside torchao layers.
+ # Dense tables cover unpacked fp16 / fp32 / seeded bf16 only; stored fp8 widens per forward, never priced.
+ packed = dense_mib is not None and _pipe_denoisers_hold_packed(pipe)
+ fp8_stored = any(
+ _holds_fp8_stored(getattr(pipe, name, None))
+ for name in ("transformer", "transformer_2", "unconditional_transformer", "unet")
+ )
+ dense_bf16 = bool(
+ dense_mib is not None
+ and compute_bytes is None
+ and not packed
+ and not fp8_stored
+ and dense_bf16_seeded(family)
+ and _denoiser_is_dense_bf16(pipe)
+ and bool(getattr(pipe, "_unsloth_dense_tier_seeds", False))
+ )
static_seed = not (
- any(
- _holds_fp8_stored(getattr(pipe, name, None))
- for name in ("transformer", "transformer_2", "unconditional_transformer", "unet")
- )
- or (
- dense_mib is not None
- and (compute_bytes is None or _pipe_denoisers_hold_packed(pipe))
- )
+ fp8_stored
+ or (dense_mib is not None and ((compute_bytes is None and not dense_bf16) or packed))
)
quant = learned_peak_label(pipe, attention, model, vae, te, cache, graphs)
try:
@@ -4122,6 +4164,7 @@ def _refine_plan_from_loaded_weights(
phase_seeds = bool(getattr(pipe, "_unsloth_phase_seeds", False)),
dense_tier_seeds = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False)),
static_seed = static_seed,
+ dense_bf16 = dense_bf16,
)
if headroom is None:
return plan
@@ -4141,6 +4184,7 @@ def _refine_plan_from_loaded_weights(
measured_dense_transformer_mib = dense_mib,
measured_compute_bytes = compute_bytes,
measured_static_seed = static_seed,
+ measured_dense_bf16 = dense_bf16,
measured_learned_cfg_off = bool(cfg_off_only and learned),
loaded_transformer_mib = dit,
loaded_text_encoder_mib = encoders,
@@ -4172,6 +4216,7 @@ def _refine_plan_from_loaded_weights(
dense_tier_seeds = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False)),
static_seed = static_seed,
phase = phase,
+ dense_bf16 = dense_bf16,
)
return value if value is not None and value < headroom else None
@@ -5079,6 +5124,7 @@ def measured_request_extra_mib(
sized["phase_seeds"] = bool(getattr(pipe, "_unsloth_phase_seeds", False))
sized["dense_tier_seeds"] = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False))
sized["static_seed"] = bool(reserve[8]) if len(reserve) > 8 and reserve[8] is not None else True
+ sized["dense_bf16"] = bool(len(reserve) > 10 and reserve[10])
need = measured_image_runtime_mib(family, speed_mode, **sized)
learned = reserve[5] if len(reserve) > 5 else None
# placed from a peak recorded without CFG: a request that runs it doubles the denoiser batch no measurement covers
@@ -5089,7 +5135,8 @@ def measured_request_extra_mib(
or (dense_mib is None and sized["phase_seeds"])
)
if (unpriced_references and tier_seeded) or (
- (learned or need is None)
+ # the bf16 seed is eager-only like a learned peak: same invalidation
+ (learned or need is None or sized["dense_bf16"])
and (
getattr(pipe, "_unsloth_peak_static", False)
or getattr(pipe, "_unsloth_loras", None)
diff --git a/studio/backend/tests/test_diffusion_bf16_seed.py b/studio/backend/tests/test_diffusion_bf16_seed.py
new file mode 100644
index 00000000000..5042cc53c8a
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_bf16_seed.py
@@ -0,0 +1,196 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A fresh bf16 load places like a returning one: the bf16 dense table seeds what only a learned peak did before.
+Numbers from FLUX.1-schnell bf16 on a Colab L4 (the load's own estimates and its recorded peaks). CPU only."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_memory as dm # noqa: E402
+
+# unsloth/FLUX.1-schnell bf16, L4 load: diffusion.loaded estimates
+FLUX_L4_LOADED = {
+ "transformer": (22681, "dit"),
+ "text_encoder_2": (9319, "text_encoder"),
+ "vae": (160, "other"),
+}
+L4_BUDGET = 20321
+ENVS = (
+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION",
+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE",
+ "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",
+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16",
+)
+
+
+def store(tmp_path, monkeypatch):
+ import core.inference.diffusion_measured_peaks as mp
+
+ for name in ENVS:
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(tmp_path / "peaks.json"))
+ monkeypatch.delenv(mp.LEARNED_PEAKS_ENV, raising = False)
+ mp._cache.update(path = None, mtime = None, data = None)
+ monkeypatch.setattr(mp, "current_device_name", lambda: "NVIDIA L4", raising = False)
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(FLUX_L4_LOADED))
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: False)
+ monkeypatch.setattr(dm, "denoiser_quant_label", lambda pipe: "dense:bfloat16")
+ return mp
+
+
+def _pipe(dtype = torch.bfloat16, measured_target = True):
+ return types.SimpleNamespace(
+ transformer = types.SimpleNamespace(dtype = dtype),
+ _unsloth_dense_tier_seeds = measured_target,
+ )
+
+
+def _l4_plan():
+ memory = dm.DeviceMemory("cuda", "cuda", "discrete_vram", 22000, 22563)
+ return dm.MemoryPlan(
+ requested_mode = dm.MEMORY_MODE_AUTO,
+ offload_policy = dm.OFFLOAD_GROUP,
+ vae_tiling = False,
+ vae_slicing = False,
+ device_memory = memory,
+ estimates = {
+ "safe_device_budget_mib": L4_BUDGET,
+ "base_overhead_mib": 2048,
+ "text_encoder_dense_mib": 9346,
+ },
+ stream_text_encoders = False,
+ stream_transformer = True,
+ )
+
+
+def _refine(pipe, plan):
+ return dm.refine_plan_from_loaded_weights(pipe, plan, family = "flux.1", speed_mode = "off")
+
+
+def test_a_fresh_bf16_flux_on_an_l4_keeps_part_of_its_denoiser_resident(store):
+ fresh = _refine(_pipe(), _l4_plan())
+ # 3581 x 1.15 -> 4352: the worst measured bf16 peak
+ assert fresh.estimates["measured_runtime_headroom_mib"] == 4352
+ assert fresh.estimates["measured_dense_bf16"] is True
+ assert fresh.estimates["measured_static_seed"] is True
+ assert fresh.stream_text_encoders and not fresh.resident_text_encoder_mib
+ # the returning load on the L4 kept 14017 MiB from its 3417 MiB peak (-> 4096); the seed is 256 MiB stricter
+ assert fresh.resident_transformer_mib == 14017 - 256
+
+
+def test_the_fresh_plan_matches_a_returning_load_within_the_seed_margin(store, monkeypatch):
+ fresh = _refine(_pipe(), _l4_plan())
+ store.record(
+ family = "flux.1",
+ quant = "dense:bfloat16",
+ speed_mode = "off",
+ width = 1024,
+ height = 1024,
+ peak_mib = 3417,
+ )
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16", "0")
+ returning = _refine(_pipe(), _l4_plan())
+ assert returning.resident_transformer_mib == 14017
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16")
+ both = _refine(_pipe(), _l4_plan())
+ assert both.resident_transformer_mib == fresh.resident_transformer_mib
+
+
+def test_the_bf16_kill_switch_restores_the_flat_plan(store, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16", "0")
+ plan = _l4_plan()
+ assert _refine(_pipe(), plan) is plan
+
+
+def test_only_the_eager_tier_and_measured_sizes_are_seeded(store):
+ m = dm.measured_image_runtime_mib
+ assert (
+ m("flux.1", "off", dense_transformer_mib = 22681, dense_bf16 = True, dense_tier_seeds = True)
+ == 4352
+ )
+ assert m("flux.1", "off", dense_transformer_mib = 22681, dense_bf16 = True) is None
+ assert (
+ m("flux.1", "default", dense_transformer_mib = 22681, dense_bf16 = True, dense_tier_seeds = True)
+ is None
+ )
+ assert (
+ m("flux.1", "off", dense_transformer_mib = 30000, dense_bf16 = True, dense_tier_seeds = True)
+ is None
+ )
+ assert (
+ m("z-image", "off", dense_transformer_mib = 11000, dense_bf16 = True, dense_tier_seeds = True)
+ is None
+ )
+ assert m("flux.1", "off", dense_transformer_mib = 22680) == 2816
+
+
+def test_math_only_attention_keeps_a_bf16_denoiser_learned_only(store):
+ """The seed was measured with a sub-quadratic SDPA kernel; math-only attention / ROCm keep the streamed plan."""
+ plan = _l4_plan()
+ assert _refine(_pipe(measured_target = False), plan) is plan
+
+
+def test_packed_and_fp8_stored_bf16_denoisers_stay_learned_only(store, monkeypatch):
+ plan = _l4_plan()
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_packed", lambda pipe: True)
+ assert _refine(_pipe(), plan) is plan
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_packed", lambda pipe: False)
+ monkeypatch.setattr(dm, "_holds_fp8_stored", lambda module: module is not None)
+ assert _refine(_pipe(), plan) is plan
+
+
+def test_the_request_guard_reads_the_bf16_seed_from_the_reserve(store):
+ pipe = types.SimpleNamespace(
+ _unsloth_dense_tier_seeds = True,
+ _unsloth_peak_identity = ("flux.1", "dense:bfloat16", "off"),
+ _unsloth_measured_reserve = (
+ 4352,
+ "flux.1",
+ "off",
+ 22681,
+ None,
+ None,
+ None,
+ False,
+ True,
+ False,
+ True,
+ ),
+ )
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == 0
+ # 2 MP: 2 x 3581 x 1.15 = 8236 -> 8448
+ assert dm.measured_request_extra_mib(pipe, width = 2048, height = 1024) == 8448 - 4352
+
+
+def test_the_deferred_profile_invalidates_the_bf16_seed(store):
+ """Generation 3 engages the compiled tier the eager seed never measured: requests size from the flat estimate."""
+ pipe = types.SimpleNamespace(
+ _unsloth_dense_tier_seeds = True,
+ _unsloth_peak_identity = ("flux.1", "dense:bfloat16", "off"),
+ _unsloth_measured_reserve = (
+ 4352,
+ "flux.1",
+ "off",
+ 22681,
+ None,
+ None,
+ None,
+ False,
+ True,
+ False,
+ True,
+ ),
+ )
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == 0
+ pipe._unsloth_peak_static = True
+ flat = dm.estimate_image_runtime_mib(width = 1024, height = 1024, batch_size = 1, family = "flux.1")
+ assert (
+ dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == max(0, int(flat) - 4352) > 0
+ )
diff --git a/studio/backend/tests/test_diffusion_small_host_planner.py b/studio/backend/tests/test_diffusion_small_host_planner.py
index 79981136207..dfef1cf4c6f 100644
--- a/studio/backend/tests/test_diffusion_small_host_planner.py
+++ b/studio/backend/tests/test_diffusion_small_host_planner.py
@@ -7,6 +7,7 @@ the dense measured activation peak scales with that compute dtype. CPU only."""
from __future__ import annotations
+import pytest
import torch
import core.inference.diffusion_memory as dm
@@ -33,14 +34,16 @@ def test_dense_measured_peak_scales_with_fp32_compute(monkeypatch):
assert fp16 == 5120 and fp32 >= 8480 * 1.15
-def test_dense_table_skips_bf16_denoisers(monkeypatch):
- """The dense eager table was measured on fp16 (and fp32-promoted) cards; a bf16 card keeps the flat plan."""
[email protected]("bf16_table", [True, False])
+def test_dense_table_places_bf16_only_from_its_own_measurements(monkeypatch, bf16_table):
+ """bf16 is priced only by its own table; with it off a bf16 card keeps the flat plan."""
import types
for name in (
dm.MEASURED_ACTIVATION_ENV,
"UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE",
dm.PARTIAL_RESIDENT_ENV,
+ dm.MEASURED_ACTIVATION_BF16_ENV,
):
monkeypatch.delenv(name, raising = False)
monkeypatch.setattr(
@@ -63,7 +66,14 @@ def test_dense_table_skips_bf16_denoisers(monkeypatch):
estimates = {"safe_device_budget_mib": 21500, "base_overhead_mib": 2048},
stream_text_encoders = True,
)
- for dtype, kept in ((torch.bfloat16, False), (torch.float16, True)):
- pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = dtype))
+ if not bf16_table:
+ monkeypatch.setenv(dm.MEASURED_ACTIVATION_BF16_ENV, "0")
+ for dtype, kept in ((torch.bfloat16, bf16_table), (torch.float16, True)):
+ pipe = types.SimpleNamespace(
+ transformer = types.SimpleNamespace(dtype = dtype), _unsloth_dense_tier_seeds = True
+ )
out = dm.refine_plan_from_loaded_weights(pipe, plan, family = "flux.1", speed_mode = "off")
assert bool(out.resident_transformer_mib) is kept, dtype
+ assert bool(out.estimates.get("measured_dense_bf16")) is (
+ dtype == torch.bfloat16 and bf16_table
+ )