Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep more of a streamed FLUX.3 klein int8 transformer resident at small budgets (L4 6 GiB -4.2% per image, H2D 56.9 -> 43.9 GiB) (#17419)

Daniel Han77e48e379186
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 700cf1d7c4f..eaff93400d3 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -2581,6 +2581,30 @@ def _step_cache_dropped_by_layer_split(pipe: Any) -> bool:
return transformer is not None and not getattr(transformer, "_unsloth_step_cache", None)
+def _engage_streamed_expandable(pipe: Any) -> bool:
+ """Blocks sized from the denoise phase: run the load in expandable allocator segments (diffusion_decode_segments)."""
+ try:
+ from .diffusion_decode_segments import engage_streamed
+ from .diffusion_memory import _phase_placement
+ if not _phase_placement(pipe) or not engage_streamed():
+ return False
+ except Exception as exc: # noqa: BLE001 - fixed segments, as before
+ logger.debug("diffusion.memory: expandable segments not engaged (%s)", exc)
+ return False
+ logger.info(
+ "diffusion.memory: the denoise-phase placement runs in expandable allocator segments"
+ )
+ return True
+
+
+def _release_streamed_expandable() -> None:
+ try:
+ from .diffusion_decode_segments import release_streamed
+ release_streamed()
+ except Exception: # noqa: BLE001 - teardown is best effort
+ pass
+
+
def _drop_measured_reserve_for_layer_split(pipe: Any, plan: Any) -> bool:
"""A layer split replaced the one-card plan whose partial residency the measured reserve priced. Left in place, a
conditioned request past it would give the split denoiser on-demand offload hooks onto the primary
@@ -11349,6 +11373,7 @@ class DiffusionBackend:
)
# streams the whole-resident denoiser back to the flat room while the encoders run
install_encode_release(pipe, plan, logger)
+ _engage_streamed_expandable(pipe)
# Placed, not yet compiled or captured: a later LoRA weight / subset change then writes in place.
pinned_scales = _pin_baked_lora_scales(pipe, target.device)
if pinned_scales:
@@ -11673,6 +11698,7 @@ class DiffusionBackend:
load_bg_compile.close()
load_bg_compile = None
restore_backend_flags(backend_flags_before)
+ _release_streamed_expandable()
_release_rocm_tunableop(pipe)
compile_cache.restore(compile_ctx, logger = logger)
gguf_compile.uninstall_all() # idempotent
@@ -16037,6 +16063,7 @@ class DiffusionBackend:
pass
# Restore process-wide backend flags so the next `off` load is bit-identical; all calls idempotent.
restore_backend_flags(state.backend_flags_before)
+ _release_streamed_expandable()
_release_rocm_tunableop(state.pipe)
prebuild = sys.modules.get(f"{__package__}.diffusion_vae_prebuild")
if prebuild is not None:
diff --git a/studio/backend/core/inference/diffusion_decode_segments.py b/studio/backend/core/inference/diffusion_decode_segments.py
index 7dcbeafd421..c9b0e994d59 100644
--- a/studio/backend/core/inference/diffusion_decode_segments.py
+++ b/studio/backend/core/inference/diffusion_decode_segments.py
@@ -3,7 +3,11 @@
"""Run each image VAE decode in one expandable allocator segment (Linux CUDA, native allocator): its growing
activations and cuDNN workspaces otherwise each map a new fixed segment (SDXL 1024^2: 11.3 -> 10.1 GiB reserved).
-UNSLOTH_DIFFUSION_DECODE_EXPANDABLE=0 turns it off."""
+UNSLOTH_DIFFUSION_DECODE_EXPANDABLE=0 turns it off.
+
+Denoise-phase-sized streamed placements use expandable segments for the whole load (``engage_streamed``): fixed
+segments cached ~1.9 GiB of churn (FLUX.3 Klein int8, L4 6 GiB), evicting resident blocks every render.
+UNSLOTH_DIFFUSION_STREAMED_EXPANDABLE=0 turns it off."""
from __future__ import annotations
@@ -16,6 +20,7 @@ from typing import Any, Iterator, Optional
from .rocm_bf16 import is_rocm_torch
DECODE_EXPANDABLE_ENV = "UNSLOTH_DIFFUSION_DECODE_EXPANDABLE"
+STREAMED_EXPANDABLE_ENV = "UNSLOTH_DIFFUSION_STREAMED_EXPANDABLE"
ENABLED_ATTR = "_unsloth_decode_expandable"
_ALLOC_CONF_VARS = ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF")
@@ -23,7 +28,9 @@ _lock = threading.Lock()
_depth = 0
_usable: Optional[bool] = None
_restore: Optional[str] = None
-STATS = {"engaged": 0, "skipped": 0}
+# settings restored by release_streamed; None = not engaged
+_streamed: Optional[str] = None
+STATS = {"engaged": 0, "skipped": 0, "streamed": 0}
def _env_off() -> bool:
@@ -177,6 +184,57 @@ def expandable_decode(vae: Any, z: Any) -> Iterator[None]:
_restore = None
+def streamed_enabled() -> bool:
+ return os.environ.get(STREAMED_EXPANDABLE_ENV, "").strip().lower() not in (
+ "0",
+ "false",
+ "no",
+ "off",
+ )
+
+
+def engage_streamed() -> bool:
+ """Expandable segments until ``release_streamed`` (Linux CUDA, native allocator). True if engaged."""
+ global _usable, _streamed
+ if not streamed_enabled():
+ return False
+ torch = _torch()
+ if torch is None or _busy(torch):
+ return False
+ with _lock:
+ if _streamed is not None:
+ return True
+ if _usable is None:
+ _usable = _decide(torch)
+ if not _usable or _depth > 0:
+ return False
+ live = _live_settings(torch)
+ if live is None:
+ return False
+ try:
+ _setter(torch)("expandable_segments:True")
+ except Exception: # noqa: BLE001 - refused: never retry
+ _usable = False
+ return False
+ _streamed = live
+ STATS["streamed"] += 1
+ return True
+
+
+def release_streamed() -> bool:
+ global _usable, _streamed
+ torch = _torch()
+ with _lock:
+ if _streamed is None or torch is None or _depth > 0:
+ return False
+ try:
+ _put_back(torch, _streamed)
+ except Exception: # noqa: BLE001
+ _usable = False
+ _streamed = None
+ return True
+
+
def enable(vae: Any) -> bool:
if vae is None or _env_off():
return False
@@ -187,4 +245,14 @@ def enable(vae: Any) -> bool:
return True
-__all__ = ["DECODE_EXPANDABLE_ENV", "ENABLED_ATTR", "STATS", "enable", "expandable_decode"]
+__all__ = [
+ "DECODE_EXPANDABLE_ENV",
+ "ENABLED_ATTR",
+ "STATS",
+ "STREAMED_EXPANDABLE_ENV",
+ "enable",
+ "engage_streamed",
+ "expandable_decode",
+ "release_streamed",
+ "streamed_enabled",
+]
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index fc2f5e37041..8f9e170818e 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -4211,6 +4211,24 @@ def denoise_phase_residency_enabled() -> bool:
return not _env_off(DENOISE_PHASE_RESIDENCY_ENV)
+# (denoise, tiled decode) MiB, streamed int8, default tier, 1024^2, window excluded, no margin; measured on an L4 at
+# 6 GiB. The resident table (901 / 2437) left ~2 GiB unused. "0" = the resident table.
+STREAMED_PHASE_SEED_ENV = "UNSLOTH_DIFFUSION_STREAMED_PHASE_SEED"
+_MEASURED_STREAMED_PHASE_MIB: dict[str, tuple[int, int]] = {
+ "flux.3-klein": (308, 904),
+}
+
+
+def _streamed_phase_seed(
+ family: Optional[str], speed_mode: Optional[str], quant: Optional[str], tiled: bool
+) -> Optional[tuple[int, int]]:
+ if _env_off(STREAMED_PHASE_SEED_ENV) or not tiled or str(speed_mode or "") != "default":
+ return None
+ if "int8" not in str(quant or "").split("|", 1)[0].lower():
+ return None
+ return _MEASURED_STREAMED_PHASE_MIB.get(str(family or "").lower())
+
+
def _top_level_group_mib(pipe: Any) -> Optional[int]:
"""MiB of the largest denoiser's weights outside its block stacks (``ModuleList`` / ``Sequential`` children):
block-level group offload keeps them in one top-level group that is onloaded for the whole forward when it is not
@@ -4279,6 +4297,9 @@ def _denoise_phase_reserve(
and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES
):
row = phases[1 if speed_mode == "max" else 0]
+ streamed = _streamed_phase_seed(family, speed_mode, quant, tiled)
+ if streamed is not None:
+ row = (row[0], streamed[0], row[2], streamed[1], row[4])
denoise = int(math.ceil(row[_PHASE_DENOISE_COLUMN] * _MEASURED_PEAK_MARGIN))
decode = int(row[_PHASE_TILED_DECODE_COLUMN if tiled else _PHASE_UNTILED_DECODE_COLUMN])
if quant is not None:
diff --git a/studio/backend/tests/test_diffusion_flux3_streamed_residency.py b/studio/backend/tests/test_diffusion_flux3_streamed_residency.py
new file mode 100644
index 00000000000..942ba14c24c
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_flux3_streamed_residency.py
@@ -0,0 +1,163 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.3 Klein int8 at a 6 GiB budget keeps more of the streamed transformer resident; CPU only.
+
+Numbers from the Colab L4 load log: 5732 MiB free, 6144 MiB int8 DiT, 4619 MiB int8 text encoder, 161 MiB VAE,
+703 MiB prefetch window, 647 MiB top-level group, tiled VAE."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+import core.inference.diffusion_decode_segments as seg
+import core.inference.diffusion_measured_peaks as mp
+import core.inference.diffusion_memory as dm
+
+LOADED = {
+ "transformer": (6144, "dit"),
+ "text_encoder": (4619, "text_encoder"),
+ "vae": (161, "other"),
+}
+FREE_6G = 5732
+WINDOW = 703
+
+
+def pipe(tmp_path, monkeypatch):
+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(tmp_path / "peaks.json"))
+ for env in (
+ mp.LEARNED_PEAKS_ENV,
+ dm.MEASURED_ACTIVATION_ENV,
+ dm.DENOISE_PHASE_RESIDENCY_ENV,
+ dm.STREAMED_RESIDENCY_ENV,
+ "UNSLOTH_DIFFUSION_STREAMED_PHASE_SEED",
+ ):
+ monkeypatch.delenv(env, raising = False)
+ mp._cache.update(path = None, mtime = None, data = None)
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(LOADED))
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ monkeypatch.setattr(dm, "denoiser_quant_label", lambda pipe: "torchao:Int8Tensor")
+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: WINDOW)
+ monkeypatch.setattr(dm, "_top_level_group_mib", lambda pipe: 647)
+ monkeypatch.setattr(dm, "planned_vae_saver_tag", lambda pipe, plan: "tiled")
+ return types.SimpleNamespace(_unsloth_phase_seeds = True)
+
+
+def _place(pipe, speed = "default"):
+ memory = dm.DeviceMemory("cuda", "cuda", "discrete_vram", FREE_6G, 6144)
+ plan = dm.plan_diffusion_memory(
+ target = types.SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = memory,
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(
+ width = None, height = None, family = "flux.3-klein"
+ ),
+ requested_mode = None,
+ model_dense_mib = 11105,
+ companion_dense_mib = 5283,
+ text_encoder_dense_mib = 5092,
+ )
+ if plan.offload_policy == dm.OFFLOAD_MODEL:
+ plan = dm.torchao_streaming_plan(plan)
+ return dm.refine_plan_from_loaded_weights(pipe, plan, family = "flux.3-klein", speed_mode = speed)
+
+
+def test_6gib_int8_sizes_resident_blocks_from_the_streamed_phases(pipe):
+ new = _place(pipe)
+ # streamed denoise 308 MiB x 1.15, rounded up to 256; slack and window unchanged
+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 512
+ assert new.estimates["resident_dit_slack_mib"] == 1024
+ assert new.resident_transformer_mib == FREE_6G - 1024 - 161 - 512 - WINDOW == 3332
+ # the decode lends from the measured tiled decode, not the untiled 2437
+ assert new.estimates["measured_decode_peak_mib"] == 904
+
+
+def test_kill_switch_restores_the_resident_table(pipe, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_STREAMED_PHASE_SEED", "0")
+ new = _place(pipe)
+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 1280
+ assert new.estimates["measured_decode_peak_mib"] == 2437
+ assert new.resident_transformer_mib == 2564
+
+
+def test_unmeasured_cases_keep_the_resident_table():
+ assert dm._streamed_phase_seed("flux.3-klein", "default", "torchao:Int8Tensor", True) == (
+ 308,
+ 904,
+ )
+ assert dm._streamed_phase_seed("flux.3-klein", "max", "torchao:Int8Tensor", True) is None
+ assert dm._streamed_phase_seed("flux.3-klein", "default", "torchao:Int8Tensor", False) is None
+ # an int8 text encoder in the label does not make an fp8 denoiser int8
+ assert (
+ dm._streamed_phase_seed("flux.3-klein", "default", "torchao:Float8Tensor|te:int8", True)
+ is None
+ )
+ assert (
+ dm._streamed_phase_seed("flux.3-klein-base", "default", "torchao:Int8Tensor", True) is None
+ )
+ assert dm._streamed_phase_seed("qwen-image-2.1", "default", "torchao:Int8Tensor", True) is None
+
+
+def allocator(monkeypatch):
+ calls = []
+ fake = types.SimpleNamespace()
+ monkeypatch.setattr(seg, "_torch", lambda: fake)
+ monkeypatch.setattr(seg, "_busy", lambda torch: False)
+ monkeypatch.setattr(seg, "_decide", lambda torch: True)
+ monkeypatch.setattr(seg, "_live_settings", lambda torch: "")
+ monkeypatch.setattr(seg, "_setter", lambda torch: calls.append)
+ monkeypatch.setattr(seg, "_usable", None)
+ monkeypatch.setattr(seg, "_streamed", None, raising = False)
+ monkeypatch.setattr(seg, "_depth", 0)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_STREAMED_EXPANDABLE", raising = False)
+ return calls
+
+
+def test_streamed_placement_engages_expandable_segments_until_released(allocator):
+ assert seg.engage_streamed()
+ assert seg.engage_streamed()
+ assert allocator == ["expandable_segments:True"]
+ assert seg.release_streamed()
+ assert allocator == ["expandable_segments:True", "expandable_segments:False", ""]
+ assert not seg.release_streamed()
+
+
+def test_streamed_expandable_kill_switch(allocator, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_STREAMED_EXPANDABLE", "0")
+ assert not seg.engage_streamed()
+ assert allocator == []
+
+
+def test_streamed_expandable_leaves_a_configured_allocator(allocator, monkeypatch):
+ monkeypatch.setattr(seg, "_live_settings", lambda torch: None)
+ assert not seg.engage_streamed()
+ assert allocator == []
+
+
+def test_a_rolled_back_load_releases_streamed_expandable_segments():
+ import ast
+ from pathlib import Path
+
+ src = (Path(__file__).resolve().parents[1] / "core" / "inference" / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ rollbacks = [
+ n
+ for n in ast.walk(ast.parse(src))
+ if isinstance(n, ast.If)
+ and isinstance(n.test, ast.UnaryOp)
+ and isinstance(n.test.op, ast.Not)
+ and isinstance(n.test.operand, ast.Name)
+ and n.test.operand.id == "state_committed"
+ ]
+ assert rollbacks
+ for block in rollbacks:
+ called = {
+ c.func.id
+ for c in ast.walk(block)
+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)
+ }
+ assert "_release_streamed_expandable" in called