Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep more of a streamed FLUX.3 klein int8 transformer resident at small budgets (L4 6 GiB -4.2% per image, H2D 56.9 -> 43.9 GiB) (#17419)
Daniel Han77e48e379186
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 700cf1d7c4f..eaff93400d3 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -2581,6 +2581,30 @@ def _step_cache_dropped_by_layer_split(pipe: Any) -> bool:return transformer is not None and not getattr(transformer, "_unsloth_step_cache", None)+def _engage_streamed_expandable(pipe: Any) -> bool:+ """Blocks sized from the denoise phase: run the load in expandable allocator segments (diffusion_decode_segments)."""+ try:+ from .diffusion_decode_segments import engage_streamed+ from .diffusion_memory import _phase_placement+ if not _phase_placement(pipe) or not engage_streamed():+ return False+ except Exception as exc: # noqa: BLE001 - fixed segments, as before+ logger.debug("diffusion.memory: expandable segments not engaged (%s)", exc)+ return False+ logger.info(+ "diffusion.memory: the denoise-phase placement runs in expandable allocator segments"+ )+ return True+++def _release_streamed_expandable() -> None:+ try:+ from .diffusion_decode_segments import release_streamed+ release_streamed()+ except Exception: # noqa: BLE001 - teardown is best effort+ pass++def _drop_measured_reserve_for_layer_split(pipe: Any, plan: Any) -> bool:"""A layer split replaced the one-card plan whose partial residency the measured reserve priced. Left in place, aconditioned request past it would give the split denoiser on-demand offload hooks onto the primary@@ -11349,6 +11373,7 @@ class DiffusionBackend:)# streams the whole-resident denoiser back to the flat room while the encoders runinstall_encode_release(pipe, plan, logger)+ _engage_streamed_expandable(pipe)# Placed, not yet compiled or captured: a later LoRA weight / subset change then writes in place.pinned_scales = _pin_baked_lora_scales(pipe, target.device)if pinned_scales:@@ -11673,6 +11698,7 @@ class DiffusionBackend:load_bg_compile.close()load_bg_compile = Nonerestore_backend_flags(backend_flags_before)+ _release_streamed_expandable()_release_rocm_tunableop(pipe)compile_cache.restore(compile_ctx, logger = logger)gguf_compile.uninstall_all() # idempotent@@ -16037,6 +16063,7 @@ class DiffusionBackend:pass# Restore process-wide backend flags so the next `off` load is bit-identical; all calls idempotent.restore_backend_flags(state.backend_flags_before)+ _release_streamed_expandable()_release_rocm_tunableop(state.pipe)prebuild = sys.modules.get(f"{__package__}.diffusion_vae_prebuild")if prebuild is not None:diff --git a/studio/backend/core/inference/diffusion_decode_segments.py b/studio/backend/core/inference/diffusion_decode_segments.pyindex 7dcbeafd421..c9b0e994d59 100644--- a/studio/backend/core/inference/diffusion_decode_segments.py+++ b/studio/backend/core/inference/diffusion_decode_segments.py@@ -3,7 +3,11 @@"""Run each image VAE decode in one expandable allocator segment (Linux CUDA, native allocator): its growingactivations and cuDNN workspaces otherwise each map a new fixed segment (SDXL 1024^2: 11.3 -> 10.1 GiB reserved).-UNSLOTH_DIFFUSION_DECODE_EXPANDABLE=0 turns it off."""+UNSLOTH_DIFFUSION_DECODE_EXPANDABLE=0 turns it off.++Denoise-phase-sized streamed placements use expandable segments for the whole load (``engage_streamed``): fixed+segments cached ~1.9 GiB of churn (FLUX.3 Klein int8, L4 6 GiB), evicting resident blocks every render.+UNSLOTH_DIFFUSION_STREAMED_EXPANDABLE=0 turns it off."""from __future__ import annotations@@ -16,6 +20,7 @@ from typing import Any, Iterator, Optionalfrom .rocm_bf16 import is_rocm_torchDECODE_EXPANDABLE_ENV = "UNSLOTH_DIFFUSION_DECODE_EXPANDABLE"+STREAMED_EXPANDABLE_ENV = "UNSLOTH_DIFFUSION_STREAMED_EXPANDABLE"ENABLED_ATTR = "_unsloth_decode_expandable"_ALLOC_CONF_VARS = ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF")@@ -23,7 +28,9 @@ _lock = threading.Lock()_depth = 0_usable: Optional[bool] = None_restore: Optional[str] = None-STATS = {"engaged": 0, "skipped": 0}+# settings restored by release_streamed; None = not engaged+_streamed: Optional[str] = None+STATS = {"engaged": 0, "skipped": 0, "streamed": 0}def _env_off() -> bool:@@ -177,6 +184,57 @@ def expandable_decode(vae: Any, z: Any) -> Iterator[None]:_restore = None+def streamed_enabled() -> bool:+ return os.environ.get(STREAMED_EXPANDABLE_ENV, "").strip().lower() not in (+ "0",+ "false",+ "no",+ "off",+ )+++def engage_streamed() -> bool:+ """Expandable segments until ``release_streamed`` (Linux CUDA, native allocator). True if engaged."""+ global _usable, _streamed+ if not streamed_enabled():+ return False+ torch = _torch()+ if torch is None or _busy(torch):+ return False+ with _lock:+ if _streamed is not None:+ return True+ if _usable is None:+ _usable = _decide(torch)+ if not _usable or _depth > 0:+ return False+ live = _live_settings(torch)+ if live is None:+ return False+ try:+ _setter(torch)("expandable_segments:True")+ except Exception: # noqa: BLE001 - refused: never retry+ _usable = False+ return False+ _streamed = live+ STATS["streamed"] += 1+ return True+++def release_streamed() -> bool:+ global _usable, _streamed+ torch = _torch()+ with _lock:+ if _streamed is None or torch is None or _depth > 0:+ return False+ try:+ _put_back(torch, _streamed)+ except Exception: # noqa: BLE001+ _usable = False+ _streamed = None+ return True++def enable(vae: Any) -> bool:if vae is None or _env_off():return False@@ -187,4 +245,14 @@ def enable(vae: Any) -> bool:return True-__all__ = ["DECODE_EXPANDABLE_ENV", "ENABLED_ATTR", "STATS", "enable", "expandable_decode"]+__all__ = [+ "DECODE_EXPANDABLE_ENV",+ "ENABLED_ATTR",+ "STATS",+ "STREAMED_EXPANDABLE_ENV",+ "enable",+ "engage_streamed",+ "expandable_decode",+ "release_streamed",+ "streamed_enabled",+]diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex fc2f5e37041..8f9e170818e 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -4211,6 +4211,24 @@ def denoise_phase_residency_enabled() -> bool:return not _env_off(DENOISE_PHASE_RESIDENCY_ENV)+# (denoise, tiled decode) MiB, streamed int8, default tier, 1024^2, window excluded, no margin; measured on an L4 at+# 6 GiB. The resident table (901 / 2437) left ~2 GiB unused. "0" = the resident table.+STREAMED_PHASE_SEED_ENV = "UNSLOTH_DIFFUSION_STREAMED_PHASE_SEED"+_MEASURED_STREAMED_PHASE_MIB: dict[str, tuple[int, int]] = {+ "flux.3-klein": (308, 904),+}+++def _streamed_phase_seed(+ family: Optional[str], speed_mode: Optional[str], quant: Optional[str], tiled: bool+) -> Optional[tuple[int, int]]:+ if _env_off(STREAMED_PHASE_SEED_ENV) or not tiled or str(speed_mode or "") != "default":+ return None+ if "int8" not in str(quant or "").split("|", 1)[0].lower():+ return None+ return _MEASURED_STREAMED_PHASE_MIB.get(str(family or "").lower())++def _top_level_group_mib(pipe: Any) -> Optional[int]:"""MiB of the largest denoiser's weights outside its block stacks (``ModuleList`` / ``Sequential`` children):block-level group offload keeps them in one top-level group that is onloaded for the whole forward when it is not@@ -4279,6 +4297,9 @@ def _denoise_phase_reserve(and str(speed_mode or "") in _MEASURED_PEAK_SPEED_MODES):row = phases[1 if speed_mode == "max" else 0]+ streamed = _streamed_phase_seed(family, speed_mode, quant, tiled)+ if streamed is not None:+ row = (row[0], streamed[0], row[2], streamed[1], row[4])denoise = int(math.ceil(row[_PHASE_DENOISE_COLUMN] * _MEASURED_PEAK_MARGIN))decode = int(row[_PHASE_TILED_DECODE_COLUMN if tiled else _PHASE_UNTILED_DECODE_COLUMN])if quant is not None:diff --git a/studio/backend/tests/test_diffusion_flux3_streamed_residency.py b/studio/backend/tests/test_diffusion_flux3_streamed_residency.pynew file mode 100644index 00000000000..942ba14c24c--- /dev/null+++ b/studio/backend/tests/test_diffusion_flux3_streamed_residency.py@@ -0,0 +1,163 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.3 Klein int8 at a 6 GiB budget keeps more of the streamed transformer resident; CPU only.++Numbers from the Colab L4 load log: 5732 MiB free, 6144 MiB int8 DiT, 4619 MiB int8 text encoder, 161 MiB VAE,+703 MiB prefetch window, 647 MiB top-level group, tiled VAE."""++from __future__ import annotations++import types++import pytest++import core.inference.diffusion_decode_segments as seg+import core.inference.diffusion_measured_peaks as mp+import core.inference.diffusion_memory as dm++LOADED = {+ "transformer": (6144, "dit"),+ "text_encoder": (4619, "text_encoder"),+ "vae": (161, "other"),+}+FREE_6G = 5732+WINDOW = 703+++def pipe(tmp_path, monkeypatch):+ monkeypatch.setenv(mp.LEARNED_PEAKS_FILE_ENV, str(tmp_path / "peaks.json"))+ for env in (+ mp.LEARNED_PEAKS_ENV,+ dm.MEASURED_ACTIVATION_ENV,+ dm.DENOISE_PHASE_RESIDENCY_ENV,+ dm.STREAMED_RESIDENCY_ENV,+ "UNSLOTH_DIFFUSION_STREAMED_PHASE_SEED",+ ):+ monkeypatch.delenv(env, raising = False)+ mp._cache.update(path = None, mtime = None, data = None)+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(LOADED))+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ monkeypatch.setattr(dm, "denoiser_quant_label", lambda pipe: "torchao:Int8Tensor")+ monkeypatch.setattr(dm, "_stream_window_mib", lambda pipe: WINDOW)+ monkeypatch.setattr(dm, "_top_level_group_mib", lambda pipe: 647)+ monkeypatch.setattr(dm, "planned_vae_saver_tag", lambda pipe, plan: "tiled")+ return types.SimpleNamespace(_unsloth_phase_seeds = True)+++def _place(pipe, speed = "default"):+ memory = dm.DeviceMemory("cuda", "cuda", "discrete_vram", FREE_6G, 6144)+ plan = dm.plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = memory,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(+ width = None, height = None, family = "flux.3-klein"+ ),+ requested_mode = None,+ model_dense_mib = 11105,+ companion_dense_mib = 5283,+ text_encoder_dense_mib = 5092,+ )+ if plan.offload_policy == dm.OFFLOAD_MODEL:+ plan = dm.torchao_streaming_plan(plan)+ return dm.refine_plan_from_loaded_weights(pipe, plan, family = "flux.3-klein", speed_mode = speed)+++def test_6gib_int8_sizes_resident_blocks_from_the_streamed_phases(pipe):+ new = _place(pipe)+ # streamed denoise 308 MiB x 1.15, rounded up to 256; slack and window unchanged+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 512+ assert new.estimates["resident_dit_slack_mib"] == 1024+ assert new.resident_transformer_mib == FREE_6G - 1024 - 161 - 512 - WINDOW == 3332+ # the decode lends from the measured tiled decode, not the untiled 2437+ assert new.estimates["measured_decode_peak_mib"] == 904+++def test_kill_switch_restores_the_resident_table(pipe, monkeypatch):+ monkeypatch.setenv("UNSLOTH_DIFFUSION_STREAMED_PHASE_SEED", "0")+ new = _place(pipe)+ assert new.estimates["measured_denoise_phase_headroom_mib"] == 1280+ assert new.estimates["measured_decode_peak_mib"] == 2437+ assert new.resident_transformer_mib == 2564+++def test_unmeasured_cases_keep_the_resident_table():+ assert dm._streamed_phase_seed("flux.3-klein", "default", "torchao:Int8Tensor", True) == (+ 308,+ 904,+ )+ assert dm._streamed_phase_seed("flux.3-klein", "max", "torchao:Int8Tensor", True) is None+ assert dm._streamed_phase_seed("flux.3-klein", "default", "torchao:Int8Tensor", False) is None+ # an int8 text encoder in the label does not make an fp8 denoiser int8+ assert (+ dm._streamed_phase_seed("flux.3-klein", "default", "torchao:Float8Tensor|te:int8", True)+ is None+ )+ assert (+ dm._streamed_phase_seed("flux.3-klein-base", "default", "torchao:Int8Tensor", True) is None+ )+ assert dm._streamed_phase_seed("qwen-image-2.1", "default", "torchao:Int8Tensor", True) is None+++def allocator(monkeypatch):+ calls = []+ fake = types.SimpleNamespace()+ monkeypatch.setattr(seg, "_torch", lambda: fake)+ monkeypatch.setattr(seg, "_busy", lambda torch: False)+ monkeypatch.setattr(seg, "_decide", lambda torch: True)+ monkeypatch.setattr(seg, "_live_settings", lambda torch: "")+ monkeypatch.setattr(seg, "_setter", lambda torch: calls.append)+ monkeypatch.setattr(seg, "_usable", None)+ monkeypatch.setattr(seg, "_streamed", None, raising = False)+ monkeypatch.setattr(seg, "_depth", 0)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_STREAMED_EXPANDABLE", raising = False)+ return calls+++def test_streamed_placement_engages_expandable_segments_until_released(allocator):+ assert seg.engage_streamed()+ assert seg.engage_streamed()+ assert allocator == ["expandable_segments:True"]+ assert seg.release_streamed()+ assert allocator == ["expandable_segments:True", "expandable_segments:False", ""]+ assert not seg.release_streamed()+++def test_streamed_expandable_kill_switch(allocator, monkeypatch):+ monkeypatch.setenv("UNSLOTH_DIFFUSION_STREAMED_EXPANDABLE", "0")+ assert not seg.engage_streamed()+ assert allocator == []+++def test_streamed_expandable_leaves_a_configured_allocator(allocator, monkeypatch):+ monkeypatch.setattr(seg, "_live_settings", lambda torch: None)+ assert not seg.engage_streamed()+ assert allocator == []+++def test_a_rolled_back_load_releases_streamed_expandable_segments():+ import ast+ from pathlib import Path++ src = (Path(__file__).resolve().parents[1] / "core" / "inference" / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ rollbacks = [+ n+ for n in ast.walk(ast.parse(src))+ if isinstance(n, ast.If)+ and isinstance(n.test, ast.UnaryOp)+ and isinstance(n.test.op, ast.Not)+ and isinstance(n.test.operand, ast.Name)+ and n.test.operand.id == "state_committed"+ ]+ assert rollbacks+ for block in rollbacks:+ called = {+ c.func.id+ for c in ast.walk(block)+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)+ }+ assert "_release_streamed_expandable" in called