Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: seed the dense bf16 placement from FLUX.1's measured peak (#15149)
Daniel Han68c1c159c768
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 1b188d0bda7..1f031033d9f 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -10444,6 +10444,7 @@ class DiffusionBackend:plan.estimates.get("measured_learned_peak_quadratic"),plan.estimates.get("measured_static_seed", True),plan.estimates.get("measured_learned_cfg_off", False),+ plan.estimates.get("measured_dense_bf16", False),)setattr(pipe, DECODE_LEND_ATTR, decode_lend_from_plan(plan))diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex e70f612f9b0..3baec144685 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -3330,19 +3330,52 @@ _MEASURED_DENSE_SPEED_MODES = ("off",)MEASURED_ACTIVATION_DENSE_TIERS_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE_TIERS"+# Dense bf16, eager tier: worst recorded whole-render peak at 1024x1024, all groups streamed (what a learned peak holds).+# family -> (peak MiB, largest loaded DiT MiB it covers). "0" = learned peak alone.+MEASURED_ACTIVATION_BF16_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16"+_MEASURED_DENSE_BF16_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {+ # FLUX.1-schnell, Colab L4, 4 steps: 3417 (4 loads), 3581 (1 load).+ "flux.1": (3581, 23800),+ # Not seeded: FLUX.2-klein-4B's 2446 used a tiled decode; Qwen-Image-2.1's 5486 was recorded resident, unstreamed.+}++def _env_off(name: str) -> bool:return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")+def dense_bf16_seeded(family: Optional[str]) -> bool:+ return not _env_off(MEASURED_ACTIVATION_BF16_ENV) and (+ str(family or "").lower() in _MEASURED_DENSE_BF16_EAGER_PEAK_MIB+ )++def _dense_seed_mib(- family: str, speed_mode: str, dense_mib: int, widen: int, tier_seeds: bool+ family: str,+ speed_mode: str,+ dense_mib: int,+ widen: int,+ tier_seeds: bool,+ bf16: bool = False,) -> Optional[int]:"""Seed MiB for a dense fp16 / fp32 denoiser (``widen`` = its element size), or None. Eager: the dense table. Default/ max: the larger of the dense table and the tier's worst phase (encoder, denoise, untiled decode), only for afamily the dense table measured at this DiT size. Families it lacks keep the flat estimate: on a T4, Z-Image's 11.5GiB DiT would then be kept partly resident, which ran slower per step than streaming it whole (3.8-4.4 vs 3.2-3.6s/step). ``tier_seeds``: the target is where the phase table was measured (NVIDIA with a sub-quadratic attention- kernel, ``diffusion._measured_target``); elsewhere (math-only attention, ROCm) the tiers keep the flat estimate."""+ kernel, ``diffusion._measured_target``); elsewhere (math-only attention, ROCm) the tiers keep the flat estimate.+ ``bf16``: the bf16 table, eager tier and ``tier_seeds`` targets only (all it was measured on)."""+ if bf16:+ entry = _MEASURED_DENSE_BF16_EAGER_PEAK_MIB.get(family)+ if (+ entry is None+ or not tier_seeds+ or _env_off(MEASURED_ACTIVATION_BF16_ENV)+ or speed_mode not in _MEASURED_DENSE_SPEED_MODES+ or not 0 < dense_mib <= entry[1]+ ):+ return None+ return entry[0]entry = _MEASURED_DENSE_EAGER_PEAK_MIB.get(family)if entry is None or not 0 < dense_mib <= entry[1] * widen // 2:return None@@ -3374,6 +3407,7 @@ def measured_image_runtime_mib(static_seed: bool = True,dense_tier_seeds: bool = False,phase: Optional[str] = None,+ dense_bf16: bool = False,) -> Optional[int]:"""Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate).``phase`` (``diffusion_measured_peaks.PHASES``) reads that phase's learned peak instead of the whole render's.@@ -3399,6 +3433,7 @@ def measured_image_runtime_mib(int(dense_transformer_mib),max(2, int(compute_bytes or 2)),dense_tier_seeds,+ bf16 = dense_bf16,)elif static_seed and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES:seed = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())@@ -4095,17 +4130,24 @@ def _refine_plan_from_loaded_weights(return plandense_mib = None if _pipe_denoisers_hold_torchao(pipe) else ditcompute_bytes = _denoiser_compute_bytes(pipe)- # The dense table covers fp16 / fp32 unpacked weights only; everything else uses a learned peak alone.- # Stored fp8 layers widen per forward, which no static table priced, even beside torchao layers.+ # Dense tables cover unpacked fp16 / fp32 / seeded bf16 only; stored fp8 widens per forward, never priced.+ packed = dense_mib is not None and _pipe_denoisers_hold_packed(pipe)+ fp8_stored = any(+ _holds_fp8_stored(getattr(pipe, name, None))+ for name in ("transformer", "transformer_2", "unconditional_transformer", "unet")+ )+ dense_bf16 = bool(+ dense_mib is not None+ and compute_bytes is None+ and not packed+ and not fp8_stored+ and dense_bf16_seeded(family)+ and _denoiser_is_dense_bf16(pipe)+ and bool(getattr(pipe, "_unsloth_dense_tier_seeds", False))+ )static_seed = not (- any(- _holds_fp8_stored(getattr(pipe, name, None))- for name in ("transformer", "transformer_2", "unconditional_transformer", "unet")- )- or (- dense_mib is not None- and (compute_bytes is None or _pipe_denoisers_hold_packed(pipe))- )+ fp8_stored+ or (dense_mib is not None and ((compute_bytes is None and not dense_bf16) or packed)))quant = learned_peak_label(pipe, attention, model, vae, te, cache, graphs)try:@@ -4122,6 +4164,7 @@ def _refine_plan_from_loaded_weights(phase_seeds = bool(getattr(pipe, "_unsloth_phase_seeds", False)),dense_tier_seeds = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False)),static_seed = static_seed,+ dense_bf16 = dense_bf16,)if headroom is None:return plan@@ -4141,6 +4184,7 @@ def _refine_plan_from_loaded_weights(measured_dense_transformer_mib = dense_mib,measured_compute_bytes = compute_bytes,measured_static_seed = static_seed,+ measured_dense_bf16 = dense_bf16,measured_learned_cfg_off = bool(cfg_off_only and learned),loaded_transformer_mib = dit,loaded_text_encoder_mib = encoders,@@ -4172,6 +4216,7 @@ def _refine_plan_from_loaded_weights(dense_tier_seeds = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False)),static_seed = static_seed,phase = phase,+ dense_bf16 = dense_bf16,)return value if value is not None and value < headroom else None@@ -5079,6 +5124,7 @@ def measured_request_extra_mib(sized["phase_seeds"] = bool(getattr(pipe, "_unsloth_phase_seeds", False))sized["dense_tier_seeds"] = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False))sized["static_seed"] = bool(reserve[8]) if len(reserve) > 8 and reserve[8] is not None else True+ sized["dense_bf16"] = bool(len(reserve) > 10 and reserve[10])need = measured_image_runtime_mib(family, speed_mode, **sized)learned = reserve[5] if len(reserve) > 5 else None# placed from a peak recorded without CFG: a request that runs it doubles the denoiser batch no measurement covers@@ -5089,7 +5135,8 @@ def measured_request_extra_mib(or (dense_mib is None and sized["phase_seeds"]))if (unpriced_references and tier_seeded) or (- (learned or need is None)+ # the bf16 seed is eager-only like a learned peak: same invalidation+ (learned or need is None or sized["dense_bf16"])and (getattr(pipe, "_unsloth_peak_static", False)or getattr(pipe, "_unsloth_loras", None)diff --git a/studio/backend/tests/test_diffusion_bf16_seed.py b/studio/backend/tests/test_diffusion_bf16_seed.pynew file mode 100644index 00000000000..5042cc53c8a--- /dev/null+++ b/studio/backend/tests/test_diffusion_bf16_seed.py@@ -0,0 +1,196 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A fresh bf16 load places like a returning one: the bf16 dense table seeds what only a learned peak did before.+Numbers from FLUX.1-schnell bf16 on a Colab L4 (the load's own estimates and its recorded peaks). CPU only."""++from __future__ import annotations++import types++import pytest++torch = pytest.importorskip("torch")++import core.inference.diffusion_memory as dm # noqa: E402++# unsloth/FLUX.1-schnell bf16, L4 load: diffusion.loaded estimates+FLUX_L4_LOADED = {+ "transformer": (22681, "dit"),+ "text_encoder_2": (9319, "text_encoder"),+ "vae": (160, "other"),+}+L4_BUDGET = 20321+ENVS = (+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION",+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE",+ "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16",+)+++def store(tmp_path, monkeypatch):+ import core.inference.diffusion_measured_peaks as mp++ for name in ENVS:+ monkeypatch.delenv(name, raising = False)+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(tmp_path / "peaks.json"))+ monkeypatch.delenv(mp.LEARNED_PEAKS_ENV, raising = False)+ mp._cache.update(path = None, mtime = None, data = None)+ monkeypatch.setattr(mp, "current_device_name", lambda: "NVIDIA L4", raising = False)+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(FLUX_L4_LOADED))+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: False)+ monkeypatch.setattr(dm, "denoiser_quant_label", lambda pipe: "dense:bfloat16")+ return mp+++def _pipe(dtype = torch.bfloat16, measured_target = True):+ return types.SimpleNamespace(+ transformer = types.SimpleNamespace(dtype = dtype),+ _unsloth_dense_tier_seeds = measured_target,+ )+++def _l4_plan():+ memory = dm.DeviceMemory("cuda", "cuda", "discrete_vram", 22000, 22563)+ return dm.MemoryPlan(+ requested_mode = dm.MEMORY_MODE_AUTO,+ offload_policy = dm.OFFLOAD_GROUP,+ vae_tiling = False,+ vae_slicing = False,+ device_memory = memory,+ estimates = {+ "safe_device_budget_mib": L4_BUDGET,+ "base_overhead_mib": 2048,+ "text_encoder_dense_mib": 9346,+ },+ stream_text_encoders = False,+ stream_transformer = True,+ )+++def _refine(pipe, plan):+ return dm.refine_plan_from_loaded_weights(pipe, plan, family = "flux.1", speed_mode = "off")+++def test_a_fresh_bf16_flux_on_an_l4_keeps_part_of_its_denoiser_resident(store):+ fresh = _refine(_pipe(), _l4_plan())+ # 3581 x 1.15 -> 4352: the worst measured bf16 peak+ assert fresh.estimates["measured_runtime_headroom_mib"] == 4352+ assert fresh.estimates["measured_dense_bf16"] is True+ assert fresh.estimates["measured_static_seed"] is True+ assert fresh.stream_text_encoders and not fresh.resident_text_encoder_mib+ # the returning load on the L4 kept 14017 MiB from its 3417 MiB peak (-> 4096); the seed is 256 MiB stricter+ assert fresh.resident_transformer_mib == 14017 - 256+++def test_the_fresh_plan_matches_a_returning_load_within_the_seed_margin(store, monkeypatch):+ fresh = _refine(_pipe(), _l4_plan())+ store.record(+ family = "flux.1",+ quant = "dense:bfloat16",+ speed_mode = "off",+ width = 1024,+ height = 1024,+ peak_mib = 3417,+ )+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16", "0")+ returning = _refine(_pipe(), _l4_plan())+ assert returning.resident_transformer_mib == 14017+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16")+ both = _refine(_pipe(), _l4_plan())+ assert both.resident_transformer_mib == fresh.resident_transformer_mib+++def test_the_bf16_kill_switch_restores_the_flat_plan(store, monkeypatch):+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_BF16", "0")+ plan = _l4_plan()+ assert _refine(_pipe(), plan) is plan+++def test_only_the_eager_tier_and_measured_sizes_are_seeded(store):+ m = dm.measured_image_runtime_mib+ assert (+ m("flux.1", "off", dense_transformer_mib = 22681, dense_bf16 = True, dense_tier_seeds = True)+ == 4352+ )+ assert m("flux.1", "off", dense_transformer_mib = 22681, dense_bf16 = True) is None+ assert (+ m("flux.1", "default", dense_transformer_mib = 22681, dense_bf16 = True, dense_tier_seeds = True)+ is None+ )+ assert (+ m("flux.1", "off", dense_transformer_mib = 30000, dense_bf16 = True, dense_tier_seeds = True)+ is None+ )+ assert (+ m("z-image", "off", dense_transformer_mib = 11000, dense_bf16 = True, dense_tier_seeds = True)+ is None+ )+ assert m("flux.1", "off", dense_transformer_mib = 22680) == 2816+++def test_math_only_attention_keeps_a_bf16_denoiser_learned_only(store):+ """The seed was measured with a sub-quadratic SDPA kernel; math-only attention / ROCm keep the streamed plan."""+ plan = _l4_plan()+ assert _refine(_pipe(measured_target = False), plan) is plan+++def test_packed_and_fp8_stored_bf16_denoisers_stay_learned_only(store, monkeypatch):+ plan = _l4_plan()+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_packed", lambda pipe: True)+ assert _refine(_pipe(), plan) is plan+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_packed", lambda pipe: False)+ monkeypatch.setattr(dm, "_holds_fp8_stored", lambda module: module is not None)+ assert _refine(_pipe(), plan) is plan+++def test_the_request_guard_reads_the_bf16_seed_from_the_reserve(store):+ pipe = types.SimpleNamespace(+ _unsloth_dense_tier_seeds = True,+ _unsloth_peak_identity = ("flux.1", "dense:bfloat16", "off"),+ _unsloth_measured_reserve = (+ 4352,+ "flux.1",+ "off",+ 22681,+ None,+ None,+ None,+ False,+ True,+ False,+ True,+ ),+ )+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == 0+ # 2 MP: 2 x 3581 x 1.15 = 8236 -> 8448+ assert dm.measured_request_extra_mib(pipe, width = 2048, height = 1024) == 8448 - 4352+++def test_the_deferred_profile_invalidates_the_bf16_seed(store):+ """Generation 3 engages the compiled tier the eager seed never measured: requests size from the flat estimate."""+ pipe = types.SimpleNamespace(+ _unsloth_dense_tier_seeds = True,+ _unsloth_peak_identity = ("flux.1", "dense:bfloat16", "off"),+ _unsloth_measured_reserve = (+ 4352,+ "flux.1",+ "off",+ 22681,+ None,+ None,+ None,+ False,+ True,+ False,+ True,+ ),+ )+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == 0+ pipe._unsloth_peak_static = True+ flat = dm.estimate_image_runtime_mib(width = 1024, height = 1024, batch_size = 1, family = "flux.1")+ assert (+ dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == max(0, int(flat) - 4352) > 0+ )diff --git a/studio/backend/tests/test_diffusion_small_host_planner.py b/studio/backend/tests/test_diffusion_small_host_planner.pyindex 79981136207..dfef1cf4c6f 100644--- a/studio/backend/tests/test_diffusion_small_host_planner.py+++ b/studio/backend/tests/test_diffusion_small_host_planner.py@@ -7,6 +7,7 @@ the dense measured activation peak scales with that compute dtype. CPU only."""from __future__ import annotations+import pytestimport torchimport core.inference.diffusion_memory as dm@@ -33,14 +34,16 @@ def test_dense_measured_peak_scales_with_fp32_compute(monkeypatch):assert fp16 == 5120 and fp32 >= 8480 * 1.15-def test_dense_table_skips_bf16_denoisers(monkeypatch):- """The dense eager table was measured on fp16 (and fp32-promoted) cards; a bf16 card keeps the flat plan."""[email protected]("bf16_table", [True, False])+def test_dense_table_places_bf16_only_from_its_own_measurements(monkeypatch, bf16_table):+ """bf16 is priced only by its own table; with it off a bf16 card keeps the flat plan."""import typesfor name in (dm.MEASURED_ACTIVATION_ENV,"UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE",dm.PARTIAL_RESIDENT_ENV,+ dm.MEASURED_ACTIVATION_BF16_ENV,):monkeypatch.delenv(name, raising = False)monkeypatch.setattr(@@ -63,7 +66,14 @@ def test_dense_table_skips_bf16_denoisers(monkeypatch):estimates = {"safe_device_budget_mib": 21500, "base_overhead_mib": 2048},stream_text_encoders = True,)- for dtype, kept in ((torch.bfloat16, False), (torch.float16, True)):- pipe = types.SimpleNamespace(transformer = types.SimpleNamespace(dtype = dtype))+ if not bf16_table:+ monkeypatch.setenv(dm.MEASURED_ACTIVATION_BF16_ENV, "0")+ for dtype, kept in ((torch.bfloat16, bf16_table), (torch.float16, True)):+ pipe = types.SimpleNamespace(+ transformer = types.SimpleNamespace(dtype = dtype), _unsloth_dense_tier_seeds = True+ )out = dm.refine_plan_from_loaded_weights(pipe, plan, family = "flux.1", speed_mode = "off")assert bool(out.resident_transformer_mib) is kept, dtype+ assert bool(out.estimates.get("measured_dense_bf16")) is (+ dtype == torch.bfloat16 and bf16_table+ )