Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: seed FLUX.2 klein residency from its measured peak on the first load (#14866)
Daniel Hana4e25b4762cd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 5b02f1c2c19..01739ba198e 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -2600,14 +2600,21 @@ def _measured_target(target: Any) -> bool:return False-def _calibrated_activation(fam: Any, target: Any) -> Any:- """Measured activations apply only where measured: NVIDIA with a sub-quadratic attention kernel."""+def _calibrated_activation(+ fam: Any,+ target: Any,+ *,+ unpriced_references_ok: bool = False,+) -> Any:+ """Measured activations apply only where measured: NVIDIA with a sub-quadratic attention kernel.+ ``unpriced_references_ok``: skip the reference rule; the caller sizes references via measured_request_extra_mib."""if getattr(target, "backend", None) != "cuda" or getattr(target, "vendor", None) != "nvidia":return None# Measured without input images: references the guard cannot size (no reference_resolutions) would overflow.# FLUX.3 can: every reference is resized to 1 MP and the request guard charges those pixels on top.if (- getattr(fam, "reference", False)+ not unpriced_references_ok+ and getattr(fam, "reference", False)and not tuple(getattr(fam, "reference_resolutions", ()) or ())and not is_flux3_family(fam)):@@ -9880,8 +9887,11 @@ class DiffusionBackend:"unknown" # adapters baked in at load: a build the key cannot name)try:- # the phase-table seed holds only where it was measured- pipe._unsloth_phase_seeds = _calibrated_activation(fam, target) is not None+ # the phase-table seed holds only where it was measured; unpriced references size flat+ pipe._unsloth_phase_seeds = (+ _calibrated_activation(fam, target, unpriced_references_ok = True)+ is not None+ )# the dense default / max tier seeds hold on the measured target onlypipe._unsloth_dense_tier_seeds = _measured_target(target)# a CFG-capable family may place from a peak recorded without CFG (``cfg:off``)diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 57883d7982b..aa324e918f6 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -4271,11 +4271,10 @@ def measured_request_extra_mib(learned = reserve[5] if len(reserve) > 5 else None# placed from a peak recorded without CFG: a request that runs it doubles the denoiser batch no measurement coverscfg_unmeasured = bool(len(reserve) > 9 and reserve[9]) and cfg is not False- # the dense tier seeds were measured on text-to-image: a reference the guard cannot size is not covered- tier_seeded = (- dense_mib is not None- and sized["dense_tier_seeds"]- and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES+ # tier and phase seeds were measured on text-to-image: an unsized reference is not covered+ tier_seeded = str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES and (+ (dense_mib is not None and sized["dense_tier_seeds"])+ or (dense_mib is None and sized["phase_seeds"]))if (unpriced_references and tier_seeded) or ((learned or need is None)diff --git a/studio/backend/tests/test_diffusion_learned_peaks.py b/studio/backend/tests/test_diffusion_learned_peaks.pyindex 75cc8543c81..be1a4c88bb8 100644--- a/studio/backend/tests/test_diffusion_learned_peaks.py+++ b/studio/backend/tests/test_diffusion_learned_peaks.py@@ -1337,7 +1337,11 @@ def test_phase_table_seed_needs_its_measured_configuration(klein_pipe):src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(encoding = "utf-8")- assert "pipe._unsloth_phase_seeds = _calibrated_activation(fam, target) is not None" in src+ at = src.index("pipe._unsloth_phase_seeds = (")+ window = " ".join(src[at : at + 300].split()) # the formatter may wrap the expression+ assert (+ "_calibrated_activation(fam, target, unpriced_references_ok = True) is not None" in window+ )def test_oom_inside_the_first_offload_apply_unwinds_its_hooks(monkeypatch):diff --git a/studio/backend/tests/test_diffusion_reference_phase_seeds.py b/studio/backend/tests/test_diffusion_reference_phase_seeds.pynew file mode 100644index 00000000000..cf0840d2f29--- /dev/null+++ b/studio/backend/tests/test_diffusion_reference_phase_seeds.py@@ -0,0 +1,119 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.2 Klein places from its measured phase seed on the first load; a reference request sizes from the flat estimate.++The phase seed was refused to every reference family the request guard cannot price (Klein's source image), so an+automatic int8 Klein on a 12 / 16 GB card had no headroom figure on its first load and streamed every block every+step (klein-9B at 16 GB: 0.18 s/step at a 6 GB peak, vs 0.041 resident). The guard already sizes an unpriced reference+from the flat estimate for the dense tier seeds; the phase seeds now take the same path. CPU only.+"""++from __future__ import annotations++import types+from pathlib import Path++import pytest++import core.inference.diffusion_memory as dm++ENVS = (+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION",+ "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_DENSE",+ "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",+ "UNSLOTH_DIFFUSION_RESIDENT_DIT",+ "UNSLOTH_DIFFUSION_STREAMED_RESIDENCY",+)+# klein-9B auto at a 16 GiB cap (outputs/low_vram_audit): int8 DiT, streamed Qwen3-8B encoder, bf16 VAE, MiB.+KLEIN9_INT8_LOADED = {+ "transformer": (8745, "dit"),+ "text_encoder": (10900, "text_encoder"),+ "vae": (160, "other"),+}++[email protected](autouse = True)+def _env(monkeypatch):+ for name in ENVS:+ monkeypatch.delenv(name, raising = False)+++def _klein_pipe(monkeypatch, phase_seeds: bool):+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(KLEIN9_INT8_LOADED))+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ monkeypatch.setattr(dm, "_holds_fp8_stored", lambda module: False)+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: 1884)+ return types.SimpleNamespace(_unsloth_phase_seeds = phase_seeds)+++def _plan_16gb():+ memory = dm.DeviceMemory("cuda", "cuda", "discrete_vram", 15872, 16384)+ return dm.MemoryPlan(+ requested_mode = dm.MEMORY_MODE_AUTO,+ offload_policy = dm.OFFLOAD_GROUP,+ vae_tiling = False,+ vae_slicing = True,+ device_memory = memory,+ estimates = {+ "safe_device_budget_mib": 13824,+ "base_overhead_mib": dm.DEFAULT_BASE_OVERHEAD_MIB,+ },+ stream_text_encoders = True,+ stream_transformer = True,+ )+++def test_calibrated_activation_skips_the_reference_rule_only_when_asked(monkeypatch):+ import core.inference.diffusion as d+ from core.inference.diffusion_families import detect_family++ monkeypatch.setattr(d, "sdpa_subquadratic_confirmed", lambda target: True)+ nvidia = types.SimpleNamespace(backend = "cuda", vendor = "nvidia")+ klein = detect_family("black-forest-labs/FLUX.2-klein-4B")+ assert klein.reference and not klein.reference_resolutions+ assert d._calibrated_activation(klein, nvidia) is None+ assert d._calibrated_activation(klein, nvidia, unpriced_references_ok = True) is not None+ rocm = types.SimpleNamespace(backend = "cuda", vendor = "amd")+ assert d._calibrated_activation(klein, rocm, unpriced_references_ok = True) is None+++def test_the_load_seeds_the_placement_with_the_reference_rule_skipped():+ src = (Path(dm.__file__).parent / "diffusion.py").read_text(encoding = "utf-8")+ at = src.index("pipe._unsloth_phase_seeds = (")+ assert (+ "_calibrated_activation(fam, target, unpriced_references_ok = True)" in src[at : at + 200]+ )+++def test_first_load_klein_int8_on_16gb_keeps_the_transformer_resident(monkeypatch):+ seeded = dm.refine_plan_from_loaded_weights(+ _klein_pipe(monkeypatch, True), _plan_16gb(), family = "flux.2-klein", speed_mode = "default"+ )+ assert seeded.estimates["measured_runtime_headroom_mib"] == 3072+ assert seeded.resident_transformer_mib == 8745+ # without the phase seed (main) there is no headroom figure: the plan streams every block+ plan = _plan_16gb()+ assert (+ dm.refine_plan_from_loaded_weights(+ _klein_pipe(monkeypatch, False), plan, family = "flux.2-klein", speed_mode = "default"+ )+ is plan+ )+++def test_unpriced_reference_on_a_phase_seeded_placement_sizes_from_the_flat_estimate():+ pipe = types.SimpleNamespace(+ _unsloth_measured_reserve = (3072, "flux.2-klein", "default", None, None),+ _unsloth_phase_seeds = True,+ )+ flat = dm.estimate_image_runtime_mib(width = 1024, height = 1024, family = "flux.2-klein")+ extra = dm.measured_request_extra_mib+ assert extra(pipe, width = 1024, height = 1024, unpriced_references = True) == flat - 3072 > 0+ assert extra(pipe, width = 1024, height = 1024) == 0+ # a placement without the phase seed (static / learned) is unchanged+ learned = types.SimpleNamespace(+ _unsloth_measured_reserve = (3072, "z-image", "default", None, None),+ _unsloth_phase_seeds = False,+ )+ assert extra(learned, width = 1024, height = 1024, unpriced_references = True) == 0