Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: seed the small-host int8 placement from its measured peak (FLUX.2-klein on a 12.7 GB T4: 30.4 to 10.6 s per image) (#15330)
Daniel Han2b116f339503
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 9154ca4755c..90d95554100 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -11105,6 +11105,7 @@ class DiffusionBackend:plan.estimates.get("measured_static_seed", True),plan.estimates.get("measured_learned_cfg_off", False),plan.estimates.get("measured_dense_bf16", False),+ plan.estimates.get("measured_small_host_int8", False),)setattr(pipe, DECODE_LEND_ATTR, decode_lend_from_plan(plan))diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 0a4e62491f1..f3b386f553d 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -3483,10 +3483,44 @@ _MEASURED_DENSE_BF16_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {}+# Small-host int8 DiT, eager: (peak MiB above resident weights at 1024^2, largest int8 DiT MiB covered).+# FLUX.2-klein-4B, Kaggle T4 fp16, 4 steps, sliced VAE. "0" = learned peak alone.+MEASURED_ACTIVATION_SMALL_HOST_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_SMALL_HOST"+_MEASURED_SMALL_HOST_INT8_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {+ "flux.2-klein": (2805, 3800),+}++def _env_off(name: str) -> bool:return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")+def _small_host_int8_seed_mib(+ family: Optional[str], speed_mode: Optional[str], int8_dit_mib: int, widen: int+) -> Optional[int]:+ """Seed MiB for a small-host int8 denoiser (``widen`` = the compute element size), or None."""+ if _env_off(MEASURED_ACTIVATION_SMALL_HOST_ENV):+ return None+ entry = _MEASURED_SMALL_HOST_INT8_EAGER_PEAK_MIB.get(str(family or "").lower())+ if entry is None or str(speed_mode or "") not in _MEASURED_DENSE_SPEED_MODES:+ return None+ if not 0 < int(int8_dit_mib) <= entry[1]:+ return None+ return entry[0] * max(2, int(widen)) // 2+++def pipe_small_host_int8(pipe: Any) -> bool:+ """Whether the small-host route stored this pipe's denoiser as int8 weights (diffusion_small_host.mark)."""+ info = getattr(pipe, "_unsloth_small_host", None)+ comps = info.get("components") if isinstance(info, dict) else None+ if not isinstance(comps, dict):+ return False+ return any(+ str(comps.get(name, "")).startswith("int8 weights")+ for name in ("transformer", "transformer_2", "unconditional_transformer")+ )++def dense_bf16_seeded(family: Optional[str]) -> bool:return not _env_off(MEASURED_ACTIVATION_BF16_ENV) and (str(family or "").lower() in _MEASURED_DENSE_BF16_EAGER_PEAK_MIB@@ -3551,6 +3585,7 @@ def measured_image_runtime_mib(dense_tier_seeds: bool = False,phase: Optional[str] = None,dense_bf16: bool = False,+ small_host_int8: bool = False,) -> Optional[int]:"""Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate).``phase`` (``diffusion_measured_peaks.PHASES``) reads that phase's learned peak instead of the whole render's.@@ -3578,6 +3613,10 @@ def measured_image_runtime_mib(dense_tier_seeds,bf16 = dense_bf16,)+ elif small_host_int8 and dense_transformer_mib is not None and phase is None:+ seed = _small_host_int8_seed_mib(+ family, speed_mode, int(dense_transformer_mib), max(2, int(compute_bytes or 2))+ )elif static_seed and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES:seed = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())if seed is None and phase_seeds:@@ -4597,6 +4636,9 @@ def _refine_plan_from_loaded_weights(fp8_storedor (dense_mib is not None and ((compute_bytes is None and not dense_bf16) or packed)))+ small_host_int8 = bool(+ packed and not fp8_stored and compute_bytes is not None and pipe_small_host_int8(pipe)+ )quant = learned_peak_label(pipe, attention, model, vae, te, cache, graphs)try:pipe._unsloth_peak_identity = (family, quant, speed_mode)@@ -4613,6 +4655,7 @@ def _refine_plan_from_loaded_weights(dense_tier_seeds = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False)),static_seed = static_seed,dense_bf16 = dense_bf16,+ small_host_int8 = small_host_int8,)if headroom is None:return plan@@ -4633,6 +4676,7 @@ def _refine_plan_from_loaded_weights(measured_compute_bytes = compute_bytes,measured_static_seed = static_seed,measured_dense_bf16 = dense_bf16,+ measured_small_host_int8 = small_host_int8,measured_learned_cfg_off = bool(cfg_off_only and learned),loaded_transformer_mib = dit,loaded_text_encoder_mib = encoders,@@ -5611,6 +5655,7 @@ def measured_request_extra_mib(sized["dense_tier_seeds"] = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False))sized["static_seed"] = bool(reserve[8]) if len(reserve) > 8 and reserve[8] is not None else Truesized["dense_bf16"] = bool(len(reserve) > 10 and reserve[10])+ sized["small_host_int8"] = bool(len(reserve) > 11 and reserve[11])need = measured_image_runtime_mib(family, speed_mode, **sized)learned = reserve[5] if len(reserve) > 5 else None# placed from a peak recorded without CFG: a request that runs it doubles the denoiser batch no measurement covers@@ -5623,8 +5668,8 @@ def measured_request_extra_mib(tier_moved = bool(getattr(pipe, TIER_MOVED_ATTR, False))tier_peak = int(tier_peak_mib) if tier_moved and tier_peak_mib else Noneif (unpriced_references and tier_seeded) or (- # the bf16 seed is eager-only like a learned peak: same invalidation- (learned or need is None or sized["dense_bf16"])+ # eager-only seeds: same invalidation as a learned peak+ (learned or need is None or sized["dense_bf16"] or sized["small_host_int8"])and (getattr(pipe, "_unsloth_peak_static", False)or (tier_moved and tier_peak is None)diff --git a/studio/backend/tests/test_diffusion_measured_placement_dense.py b/studio/backend/tests/test_diffusion_measured_placement_dense.pyindex 06d9da73dda..7833f9d4acf 100644--- a/studio/backend/tests/test_diffusion_measured_placement_dense.py+++ b/studio/backend/tests/test_diffusion_measured_placement_dense.py@@ -206,6 +206,81 @@ def test_small_host_int8_denoiser_keeps_the_flat_plan(klein_pipe):assert _refine(klein_pipe, plan) is plan+KLEIN_SMALL_HOST_LOADED = {+ "transformer": (3700, "dit"),+ "text_encoder": (5940, "text_encoder"),+ "vae": (160, "other"),+}+KLEIN_SMALL_HOST_HEADROOM = 3328 # 2805 x 1.15, rounded up to 256 MiB+SMALL_HOST_SEED_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_SMALL_HOST"+++def _small_host_int8_pipe(klein_pipe, monkeypatch):+ import torch++ import core.inference.diffusion_small_host as sh++ dit = torch.nn.Sequential(torch.nn.Linear(2048, 2048)).to(torch.bfloat16)+ sh.quantize_int8_weight_(dit, compute_dtype = torch.float16, work_device = "cpu")+ klein_pipe.transformer = types.SimpleNamespace(dtype = torch.float16, parameters = dit.parameters)+ sh.mark(+ klein_pipe,+ {+ "components": {+ "text_encoder": "memory-mapped, layerwise cast (6930 MiB)",+ "transformer": "int8 weights (3694 MiB, 106 linears)",+ }+ },+ )+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(KLEIN_SMALL_HOST_LOADED))+ monkeypatch.delenv(SMALL_HOST_SEED_ENV, raising = False)+ return klein_pipe+++def test_small_host_int8_klein_places_from_its_measured_seed(klein_pipe, monkeypatch):+ pipe = _small_host_int8_pipe(klein_pipe, monkeypatch)+ plan = _flat_plan(12758)+ new = _refine(pipe, plan)+ assert new is not plan+ assert new.estimates["measured_runtime_headroom_mib"] == KLEIN_SMALL_HOST_HEADROOM+ assert new.estimates["measured_small_host_int8"] is True+ assert new.resident_transformer_mib == 3700+ assert (new.resident_text_encoder_mib or 0) > 0+ resident = 3700 + (new.resident_text_encoder_mib or 0) + 160+ assert resident + KLEIN_SMALL_HOST_HEADROOM + dm.DEFAULT_BASE_OVERHEAD_MIB <= 12758+++def test_small_host_int8_seed_kill_switch_and_scope(klein_pipe, monkeypatch):+ pipe = _small_host_int8_pipe(klein_pipe, monkeypatch)+ plan = _flat_plan(12758)+ monkeypatch.setenv(SMALL_HOST_SEED_ENV, "0")+ assert _refine(pipe, plan) is plan+ monkeypatch.delenv(SMALL_HOST_SEED_ENV)+ assert _refine(pipe, plan, family = "flux.1") is plan+ assert dm._small_host_int8_seed_mib("flux.2-klein", "default", 3700, 2) is None+ assert dm._small_host_int8_seed_mib("flux.2-klein", "off", 9000, 2) is None+ assert dm._small_host_int8_seed_mib("flux.2-klein", "off", 3700, 4) == 5610+ pipe._unsloth_measured_reserve = (+ KLEIN_SMALL_HOST_HEADROOM,+ "flux.2-klein",+ "off",+ 3700,+ 2,+ None,+ None,+ None,+ False,+ False,+ False,+ True,+ )+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == 0+ assert dm.measured_request_extra_mib(pipe, width = 2048, height = 2048) > 0+ # eager-only seed: a moved tier falls back to the flat estimate+ setattr(pipe, dm.TIER_MOVED_ATTR, True)+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) > 0++# ------------------------------------------------------------------- learned-only placement (bf16 / packed denoisers)# The dense eager table was measured on fp16 / fp32 unpacked weights, so a bf16 denoiser (every Ampere+ card) and a# GGUF / int8 packed one used to return before the learned-peak lookup: their renders recorded peaks no load ever read.