Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: seed the small-host int8 placement from its measured peak (FLUX.2-klein on a 12.7 GB T4: 30.4 to 10.6 s per image) (#15330)

Daniel Han2b116f339503
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 9154ca4755c..90d95554100 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -11105,6 +11105,7 @@ class DiffusionBackend:
plan.estimates.get("measured_static_seed", True),
plan.estimates.get("measured_learned_cfg_off", False),
plan.estimates.get("measured_dense_bf16", False),
+ plan.estimates.get("measured_small_host_int8", False),
)
setattr(pipe, DECODE_LEND_ATTR, decode_lend_from_plan(plan))
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 0a4e62491f1..f3b386f553d 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -3483,10 +3483,44 @@ _MEASURED_DENSE_BF16_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {
}
+# Small-host int8 DiT, eager: (peak MiB above resident weights at 1024^2, largest int8 DiT MiB covered).
+# FLUX.2-klein-4B, Kaggle T4 fp16, 4 steps, sliced VAE. "0" = learned peak alone.
+MEASURED_ACTIVATION_SMALL_HOST_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_SMALL_HOST"
+_MEASURED_SMALL_HOST_INT8_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {
+ "flux.2-klein": (2805, 3800),
+}
+
+
def _env_off(name: str) -> bool:
return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")
+def _small_host_int8_seed_mib(
+ family: Optional[str], speed_mode: Optional[str], int8_dit_mib: int, widen: int
+) -> Optional[int]:
+ """Seed MiB for a small-host int8 denoiser (``widen`` = the compute element size), or None."""
+ if _env_off(MEASURED_ACTIVATION_SMALL_HOST_ENV):
+ return None
+ entry = _MEASURED_SMALL_HOST_INT8_EAGER_PEAK_MIB.get(str(family or "").lower())
+ if entry is None or str(speed_mode or "") not in _MEASURED_DENSE_SPEED_MODES:
+ return None
+ if not 0 < int(int8_dit_mib) <= entry[1]:
+ return None
+ return entry[0] * max(2, int(widen)) // 2
+
+
+def pipe_small_host_int8(pipe: Any) -> bool:
+ """Whether the small-host route stored this pipe's denoiser as int8 weights (diffusion_small_host.mark)."""
+ info = getattr(pipe, "_unsloth_small_host", None)
+ comps = info.get("components") if isinstance(info, dict) else None
+ if not isinstance(comps, dict):
+ return False
+ return any(
+ str(comps.get(name, "")).startswith("int8 weights")
+ for name in ("transformer", "transformer_2", "unconditional_transformer")
+ )
+
+
def dense_bf16_seeded(family: Optional[str]) -> bool:
return not _env_off(MEASURED_ACTIVATION_BF16_ENV) and (
str(family or "").lower() in _MEASURED_DENSE_BF16_EAGER_PEAK_MIB
@@ -3551,6 +3585,7 @@ def measured_image_runtime_mib(
dense_tier_seeds: bool = False,
phase: Optional[str] = None,
dense_bf16: bool = False,
+ small_host_int8: bool = False,
) -> Optional[int]:
"""Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate).
``phase`` (``diffusion_measured_peaks.PHASES``) reads that phase's learned peak instead of the whole render's.
@@ -3578,6 +3613,10 @@ def measured_image_runtime_mib(
dense_tier_seeds,
bf16 = dense_bf16,
)
+ elif small_host_int8 and dense_transformer_mib is not None and phase is None:
+ seed = _small_host_int8_seed_mib(
+ family, speed_mode, int(dense_transformer_mib), max(2, int(compute_bytes or 2))
+ )
elif static_seed and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES:
seed = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())
if seed is None and phase_seeds:
@@ -4597,6 +4636,9 @@ def _refine_plan_from_loaded_weights(
fp8_stored
or (dense_mib is not None and ((compute_bytes is None and not dense_bf16) or packed))
)
+ small_host_int8 = bool(
+ packed and not fp8_stored and compute_bytes is not None and pipe_small_host_int8(pipe)
+ )
quant = learned_peak_label(pipe, attention, model, vae, te, cache, graphs)
try:
pipe._unsloth_peak_identity = (family, quant, speed_mode)
@@ -4613,6 +4655,7 @@ def _refine_plan_from_loaded_weights(
dense_tier_seeds = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False)),
static_seed = static_seed,
dense_bf16 = dense_bf16,
+ small_host_int8 = small_host_int8,
)
if headroom is None:
return plan
@@ -4633,6 +4676,7 @@ def _refine_plan_from_loaded_weights(
measured_compute_bytes = compute_bytes,
measured_static_seed = static_seed,
measured_dense_bf16 = dense_bf16,
+ measured_small_host_int8 = small_host_int8,
measured_learned_cfg_off = bool(cfg_off_only and learned),
loaded_transformer_mib = dit,
loaded_text_encoder_mib = encoders,
@@ -5611,6 +5655,7 @@ def measured_request_extra_mib(
sized["dense_tier_seeds"] = bool(getattr(pipe, "_unsloth_dense_tier_seeds", False))
sized["static_seed"] = bool(reserve[8]) if len(reserve) > 8 and reserve[8] is not None else True
sized["dense_bf16"] = bool(len(reserve) > 10 and reserve[10])
+ sized["small_host_int8"] = bool(len(reserve) > 11 and reserve[11])
need = measured_image_runtime_mib(family, speed_mode, **sized)
learned = reserve[5] if len(reserve) > 5 else None
# placed from a peak recorded without CFG: a request that runs it doubles the denoiser batch no measurement covers
@@ -5623,8 +5668,8 @@ def measured_request_extra_mib(
tier_moved = bool(getattr(pipe, TIER_MOVED_ATTR, False))
tier_peak = int(tier_peak_mib) if tier_moved and tier_peak_mib else None
if (unpriced_references and tier_seeded) or (
- # the bf16 seed is eager-only like a learned peak: same invalidation
- (learned or need is None or sized["dense_bf16"])
+ # eager-only seeds: same invalidation as a learned peak
+ (learned or need is None or sized["dense_bf16"] or sized["small_host_int8"])
and (
getattr(pipe, "_unsloth_peak_static", False)
or (tier_moved and tier_peak is None)
diff --git a/studio/backend/tests/test_diffusion_measured_placement_dense.py b/studio/backend/tests/test_diffusion_measured_placement_dense.py
index 06d9da73dda..7833f9d4acf 100644
--- a/studio/backend/tests/test_diffusion_measured_placement_dense.py
+++ b/studio/backend/tests/test_diffusion_measured_placement_dense.py
@@ -206,6 +206,81 @@ def test_small_host_int8_denoiser_keeps_the_flat_plan(klein_pipe):
assert _refine(klein_pipe, plan) is plan
+KLEIN_SMALL_HOST_LOADED = {
+ "transformer": (3700, "dit"),
+ "text_encoder": (5940, "text_encoder"),
+ "vae": (160, "other"),
+}
+KLEIN_SMALL_HOST_HEADROOM = 3328 # 2805 x 1.15, rounded up to 256 MiB
+SMALL_HOST_SEED_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_SMALL_HOST"
+
+
+def _small_host_int8_pipe(klein_pipe, monkeypatch):
+ import torch
+
+ import core.inference.diffusion_small_host as sh
+
+ dit = torch.nn.Sequential(torch.nn.Linear(2048, 2048)).to(torch.bfloat16)
+ sh.quantize_int8_weight_(dit, compute_dtype = torch.float16, work_device = "cpu")
+ klein_pipe.transformer = types.SimpleNamespace(dtype = torch.float16, parameters = dit.parameters)
+ sh.mark(
+ klein_pipe,
+ {
+ "components": {
+ "text_encoder": "memory-mapped, layerwise cast (6930 MiB)",
+ "transformer": "int8 weights (3694 MiB, 106 linears)",
+ }
+ },
+ )
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(KLEIN_SMALL_HOST_LOADED))
+ monkeypatch.delenv(SMALL_HOST_SEED_ENV, raising = False)
+ return klein_pipe
+
+
+def test_small_host_int8_klein_places_from_its_measured_seed(klein_pipe, monkeypatch):
+ pipe = _small_host_int8_pipe(klein_pipe, monkeypatch)
+ plan = _flat_plan(12758)
+ new = _refine(pipe, plan)
+ assert new is not plan
+ assert new.estimates["measured_runtime_headroom_mib"] == KLEIN_SMALL_HOST_HEADROOM
+ assert new.estimates["measured_small_host_int8"] is True
+ assert new.resident_transformer_mib == 3700
+ assert (new.resident_text_encoder_mib or 0) > 0
+ resident = 3700 + (new.resident_text_encoder_mib or 0) + 160
+ assert resident + KLEIN_SMALL_HOST_HEADROOM + dm.DEFAULT_BASE_OVERHEAD_MIB <= 12758
+
+
+def test_small_host_int8_seed_kill_switch_and_scope(klein_pipe, monkeypatch):
+ pipe = _small_host_int8_pipe(klein_pipe, monkeypatch)
+ plan = _flat_plan(12758)
+ monkeypatch.setenv(SMALL_HOST_SEED_ENV, "0")
+ assert _refine(pipe, plan) is plan
+ monkeypatch.delenv(SMALL_HOST_SEED_ENV)
+ assert _refine(pipe, plan, family = "flux.1") is plan
+ assert dm._small_host_int8_seed_mib("flux.2-klein", "default", 3700, 2) is None
+ assert dm._small_host_int8_seed_mib("flux.2-klein", "off", 9000, 2) is None
+ assert dm._small_host_int8_seed_mib("flux.2-klein", "off", 3700, 4) == 5610
+ pipe._unsloth_measured_reserve = (
+ KLEIN_SMALL_HOST_HEADROOM,
+ "flux.2-klein",
+ "off",
+ 3700,
+ 2,
+ None,
+ None,
+ None,
+ False,
+ False,
+ False,
+ True,
+ )
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == 0
+ assert dm.measured_request_extra_mib(pipe, width = 2048, height = 2048) > 0
+ # eager-only seed: a moved tier falls back to the flat estimate
+ setattr(pipe, dm.TIER_MOVED_ATTR, True)
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) > 0
+
+
# ------------------------------------------------------------------- learned-only placement (bf16 / packed denoisers)
# The dense eager table was measured on fp16 / fp32 unpacked weights, so a bf16 denoiser (every Ampere+ card) and a
# GGUF / int8 packed one used to return before the learned-peak lookup: their renders recorded peaks no load ever read.