Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: skip settling for proven resident replans (#16400)
Daniel Haneb648c2047b7
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 9e7213949fc..172fb796204 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -128,6 +128,7 @@ from .diffusion_zimage_cfg_split import render as zimage_cfg_split_renderfrom .diffusion_prefix_kv import release_prefix_kv, track_prefix_kvfrom .diffusion_multigpu import (AUTOMATIC_ORDINALS,+ current_selection,end_cfg_render,release_cfg_prefix_kv,drop_displaced_cfg_router,@@ -187,6 +188,7 @@ from .diffusion_memory import (release_load_budget_fraction,vram_budget_scope,carry_load_budget,+ MEMORY_MODE_AUTO,MEMORY_MODE_BALANCED,MEMORY_MODE_LOW_VRAM,ACTIVATION_TILE,@@ -12439,9 +12441,23 @@ class DiffusionBackend:None # (fp8_storage, offload) -> MiB, set for a single-file DiT)single_file_stored_at: Any = None+ resident_replan = (+ kind == "pipeline"+ and transformer_resident_override_mib is not None+ and transformer_resident_override_mib > 0+ and companion_override_mib is not None+ and text_encoder_override_mib is not None+ and 0 <= text_encoder_override_mib <= companion_override_mib+ and getattr(target, "device", None) == "cuda"+ and getattr(target, "backend", None) == "cuda"+ and normalize_memory_mode(memory_mode) in (None, MEMORY_MODE_AUTO)+ and not cpu_offload+ and current_selection() is None+ and device_memory_override is None+ )device_memory = (device_memory_override- if device_memory_override is not None+ if device_memory_override is not None or resident_replanelse settled_snapshot_device_memory(target))load_dtype = getattr(target, "dtype", None)@@ -12683,19 +12699,62 @@ class DiffusionBackend:# Budgets the 1024x1024 default; generate() re-checks the real resolution.runtime_headroom = estimate_image_runtime_mib(width = None, height = None, family = variant_hint)- def _plan(model_mib: Optional[int]) -> Any:+ def _plan(+ model_mib: Optional[int],+ memory: Optional[DeviceMemory] = None,+ *,+ probe_calibration: bool = True,+ activation: Any = None,+ ) -> Any:return plan_diffusion_memory(target = target,- device_memory = device_memory,+ device_memory = device_memory if memory is None else memory,model_dense_mib = model_mib,companion_dense_mib = companion_mib,text_encoder_dense_mib = text_encoder_mib,runtime_headroom_mib = runtime_headroom,requested_mode = memory_mode,explicit_offload = cpu_offload,- calibrated_activation = _calibrated_activation(fam, target),+ calibrated_activation = _calibrated_activation(fam, target)+ if probe_calibration+ else activation,)+ if resident_replan:++ def can_stop(memory: DeviceMemory) -> bool:+ if (+ memory.memory_kind != "discrete_vram"+ or memory.free_mib is None+ or memory.total_mib is None+ ):+ return False+ # Check flat and both measured tiers without running the allocating SDPA probe during sampling.+ for activation in (+ None,+ calibrated_image_activation(fam.name, max_speed = False),+ calibrated_image_activation(fam.name, max_speed = True),+ ):+ candidate = _plan(+ model_dense_mib, memory, probe_calibration = False, activation = activation+ )+ if not (+ _plan_proves_resident(candidate)+ and not candidate.vae_tiling+ and not candidate.vae_slicing+ and not candidate.stream_text_encoders+ and candidate.resident_transformer_mib is None+ and candidate.resident_text_encoder_mib is None+ and candidate.secondary_ordinal is None+ and candidate.secondary_components is None+ and candidate.layer_split is None+ and candidate.fallback_plan is None+ ):+ return False+ return True++ device_memory = settled_snapshot_device_memory(target, can_stop = can_stop)+plan = _plan(model_dense_mib)if kind == "single_file" and single_file_path:self.__dict__.setdefault("_comfy_priced_offload", {})[str(single_file_path)] = Falsediff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex c20d520c15d..7fd84d13f32 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1337,6 +1337,8 @@ def settled_snapshot_device_memory(target: Any,attempts: int = 3,delay_s: float = 1.0,+ *,+ can_stop: Optional[Callable[[DeviceMemory], bool]] = None,) -> DeviceMemory:"""Max free over spaced reads (a transient only shrinks free); on mps, empty_cache first so freed pool is not budgeted as used memory."""device = getattr(target, "device", "cpu")@@ -1361,11 +1363,13 @@ def settled_snapshot_device_memory(passbest = snapshot_device_memory(target)delay_s = _settle_delay(delay_s)- for _ in range(max(0, attempts - 1)):+ for attempt in range(max(0, attempts - 1)):if best.free_mib is not None and best.total_mib is not None:# Free already within the reserve of total: nothing transient to wait out.if best.free_mib >= best.total_mib - max(2048, int(best.total_mib * 0.10)):break+ if attempt == 0 and can_stop is not None and can_stop(best):+ breaktry:import timetime.sleep(delay_s)diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex 44326f15b68..9bbda6a5a47 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -6952,8 +6952,7 @@ def test_plan_memory_dense_replan_does_not_double_count_prefetched_transformer(mdef test_plan_memory_pipeline_replan_prices_the_quantised_transformer(monkeypatch):- """Pipeline re-plans must price the quantised transformer, or offloaded runs never take the fast- path."""+ """Pipeline replans price the quantized transformer."""from core.inference import diffusion as dmodfrom core.inference.diffusion_memory import OFFLOAD_NONE, DeviceMemory@@ -6962,7 +6961,7 @@ def test_plan_memory_pipeline_replan_prices_the_quantised_transformer(monkeypatcmonkeypatch.setattr(dmod,"settled_snapshot_device_memory",- lambda t: DeviceMemory("cuda", "cuda", "discrete_vram", 40000, 40960),+ lambda t, **kwargs: DeviceMemory("cuda", "cuda", "discrete_vram", 40000, 40960),)monkeypatch.setattr(dmod, "estimate_image_runtime_mib", lambda **kw: 4000)monkeypatch.setattr(diff --git a/studio/backend/tests/test_diffusion_load_settle_reuse.py b/studio/backend/tests/test_diffusion_load_settle_reuse.pynew file mode 100644index 00000000000..becc6c63862--- /dev/null+++ b/studio/backend/tests/test_diffusion_load_settle_reuse.py@@ -0,0 +1,339 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import types++import pytest++from core.inference import diffusion as dmod+from core.inference import diffusion_memory as mem+from core.inference.diffusion_memory import DeviceMemory+from .test_diffusion_backend import fake_runtime, _stub_pipeline_dense_quant++CUDA = types.SimpleNamespace(device = "cuda", backend = "cuda", supports_model_cpu_offload = True)+FAMILY = types.SimpleNamespace(name = "z-image", base_repo = "Tongyi-MAI/Z-Image-Turbo")+++def _mock_cleanup(monkeypatch):+ import torch+ from core.inference import diffusion_cuda_graph++ monkeypatch.setattr(diffusion_cuda_graph, "synchronize_outside_capture", lambda: None)+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None, raising = False)+ mps = getattr(torch, "mps", None)+ if mps is not None:+ monkeypatch.setattr(mps, "empty_cache", lambda: None, raising = False)+++def card(monkeypatch):+ state = types.SimpleNamespace(+ free = 12_000, total = 24_000, reads = 0, sleeps = 0, kind = "discrete_vram"+ )++ def snapshot(target):+ state.reads += 1+ return DeviceMemory(target.backend, target.device, state.kind, state.free, state.total)++ def sleep(_s):+ state.sleeps += 1++ _mock_cleanup(monkeypatch)+ monkeypatch.setattr(mem, "snapshot_device_memory", snapshot)+ monkeypatch.setattr("time.sleep", sleep)+ monkeypatch.setattr(dmod, "_calibrated_activation", lambda *a, **k: None)+ monkeypatch.setattr(dmod, "estimate_image_runtime_mib", lambda **k: 1000)+ monkeypatch.setenv("UNSLOTH_SETTLE_DELAY_S", "1")+ return state+++def _replan(+ backend = None,+ target = CUDA,+ family = FAMILY,+ mode = None,+ cpu_offload = False,+ **overrides,+):+ kwargs = dict(+ kind = "pipeline",+ transformer_resident_override_mib = 3000,+ companion_override_mib = 1000,+ text_encoder_override_mib = 800,+ )+ kwargs.update(overrides)+ return (backend or dmod.DiffusionBackend())._plan_memory(+ target, None, "unused", family, mode, cpu_offload, **kwargs+ )+++def test_known_resident_replan_uses_one_fresh_read(card):+ result = _replan()+ assert dmod._plan_proves_resident(result)+ assert not result.vae_tiling and not result.vae_slicing+ assert result.device_memory.free_mib == 12_000+ assert 12_000 < card.total - max(2048, int(card.total * 0.10))+ assert (card.reads, card.sleeps) == (1, 0)+++def test_each_resident_replan_reads_current_memory(card):+ backend = dmod.DiffusionBackend()+ first = _replan(backend)+ card.free = 14_000+ second = _replan(backend)+ assert first.device_memory.free_mib == 12_000+ assert second.device_memory.free_mib == 14_000+ assert (card.reads, card.sleeps) == (2, 0)+++def test_initial_known_pipeline_plan_keeps_settling(card, tmp_path, monkeypatch):+ backend = dmod.DiffusionBackend()+ monkeypatch.setattr(backend, "_local_dir_weight_bytes", lambda *a, **k: 4000 * 2**20)+ monkeypatch.setattr(backend, "_cache_bytes", lambda *a, **k: 0)+ monkeypatch.setattr(backend, "_companion_cache_bytes", lambda *a, **k: 1000 * 2**20)+ monkeypatch.setattr(backend, "_text_encoder_cache_bytes", lambda *a, **k: 800 * 2**20)+ result = _replan(+ backend,+ transformer_resident_override_mib = None,+ repo_id = str(tmp_path),+ base_local_dir = str(tmp_path),+ )+ assert result.estimates["model_dense_mib"] == 4000+ assert dmod._plan_proves_resident(result)+ assert (card.reads, card.sleeps) == (3, 2)+++def test_active_multigpu_selection_keeps_settling(card, monkeypatch):+ from core.inference.diffusion_multigpu import (+ CFG_PARALLEL_ENV,+ MULTIGPU_ENV,+ MultiGpuSelection,+ current_selection,+ selection_scope,+ )++ monkeypatch.setenv(MULTIGPU_ENV, "auto")+ monkeypatch.setenv(CFG_PARALLEL_ENV, "auto")+ selection = MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)+ target = types.SimpleNamespace(**vars(CUDA), ordinal = 0)+ with selection_scope(selection):+ assert current_selection() is selection+ result = _replan(target = target)+ assert current_selection() is None+ assert dmod._plan_proves_resident(result)+ assert (card.reads, card.sleeps) == (3, 2)+++def test_explicit_cpu_offload_keeps_settling(card):+ result = _replan(cpu_offload = True)+ assert result.offload_policy == mem.OFFLOAD_MODEL+ assert result.estimates["explicit_cpu_offload"] == 1+ assert (card.reads, card.sleeps) == (3, 2)+++def test_supplied_device_memory_bypasses_sampling(card):+ supplied = DeviceMemory("cuda", "cuda", "discrete_vram", 14_000, card.total)+ card.free = 7000+ result = _replan(device_memory_override = supplied)+ assert result.device_memory is supplied+ assert dmod._plan_proves_resident(result)+ assert (card.reads, card.sleeps) == (0, 0)++[email protected]("mode", ["fast", "balanced", "low_vram"])+def test_explicit_memory_modes_keep_settling(card, mode):+ _replan(mode = mode)+ assert (card.reads, card.sleeps) == (3, 2)++[email protected]("field", ["companion_override_mib", "text_encoder_override_mib"])+def test_unknown_component_sizes_keep_settling(card, field):+ _replan(**{field: None})+ assert (card.reads, card.sleeps) == (3, 2)++[email protected]("field", ["free", "total"])+def test_unknown_memory_keeps_settling(card, field):+ setattr(card, field, None)+ _replan()+ assert (card.reads, card.sleeps) == (3, 2)++[email protected]("backend,kind", [("rocm", "discrete_vram"), ("cuda", "unified_memory")])+def test_other_cuda_memory_kinds_keep_settling(card, backend, kind):+ card.kind = kind+ target = types.SimpleNamespace(device = "cuda", backend = backend, supports_model_cpu_offload = True)+ _replan(target = target)+ assert (card.reads, card.sleeps) == (3, 2)++[email protected]("device", ["mps", "xpu", "cpu"])+def test_other_devices_read_fresh_without_cuda_waits(card, device):+ target = types.SimpleNamespace(device = device, backend = device, supports_model_cpu_offload = False)+ _replan(target = target)+ card.free = 14_000+ result = _replan(target = target)+ assert result.device_memory.free_mib == 14_000+ assert (card.reads, card.sleeps) == (2, 0)+++def test_calibration_probe_stays_after_sampling(card, monkeypatch):+ observed = []+ monkeypatch.setattr(dmod, "_calibrated_activation", lambda *a, **k: observed.append(card.reads))+ _replan()+ assert observed == [1]+ card.free = 8000+ card.reads = card.sleeps = 0+ _replan()+ assert observed == [1, 3]+++def test_flat_residency_does_not_override_calibrated_demotion(card, monkeypatch):+ card.free, card.total = 17_400, 32_000+ family = types.SimpleNamespace(name = "qwen-image-2.1", base_repo = "Qwen/Qwen-Image-2.1")+ activation = mem.calibrated_image_activation(family.name, max_speed = True)+ monkeypatch.setattr(dmod, "estimate_image_runtime_mib", lambda **k: 6963)+ monkeypatch.setattr(dmod, "_calibrated_activation", lambda *a, **k: activation)++ def sleep(_s):+ card.sleeps += 1+ card.free = 18_000++ monkeypatch.setattr("time.sleep", sleep)+ result = _replan(+ family = family,+ transformer_resident_override_mib = 2000,+ companion_override_mib = 2000,+ text_encoder_override_mib = 1000,+ )+ assert result.offload_policy == mem.OFFLOAD_NONE+ assert result.device_memory.free_mib == 18_000+ assert (card.reads, card.sleeps) == (3, 2)+++def test_constrained_replan_observes_delayed_release(card, monkeypatch):+ card.free = 10_000+ monkeypatch.setattr(dmod, "estimate_image_runtime_mib", lambda **k: 6000)++ def sleep(_s):+ card.sleeps += 1+ card.free = 20_000++ monkeypatch.setattr("time.sleep", sleep)+ result = _replan(+ transformer_resident_override_mib = 3000,+ companion_override_mib = 8000,+ text_encoder_override_mib = 6000,+ )+ assert result.offload_policy == mem.OFFLOAD_GROUP+ assert result.device_memory.free_mib == 20_000+ assert (card.reads, card.sleeps) == (3, 2)+++def test_original_sampler_still_waits_without_a_planner_proof(card):+ first = mem.settled_snapshot_device_memory(CUDA)+ second = mem.settled_snapshot_device_memory(CUDA)+ assert first == second+ assert (card.reads, card.sleeps) == (6, 4)+++def test_pipeline_quantized_resident_replan_stops_after_its_fresh_read(+ fake_runtime, card, tmp_path, monkeypatch+):+ card.free, card.total = 26_000, 48_000+ monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, "1")+ monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: (0, 39))+ monkeypatch.setattr(mem, "_installed_torchao_version", lambda: (0, 18))+ monkeypatch.setattr(dmod, "compile_eligible", lambda *a, **k: True)+ backend = dmod.DiffusionBackend()+ quantizations = _stub_pipeline_dense_quant(backend, monkeypatch)+ monkeypatch.setattr(backend, "_local_dir_weight_bytes", lambda *a, **k: 22_000 * 2**20)+ monkeypatch.setattr(backend, "_cache_bytes", lambda *a, **k: 0)+ monkeypatch.setattr(backend, "_companion_cache_bytes", lambda *a, **k: 8000 * 2**20)+ monkeypatch.setattr(backend, "_text_encoder_cache_bytes", lambda *a, **k: 7800 * 2**20)+ plans, accepted = [], []+ real_plan = dmod.DiffusionBackend._plan_memory+ real_sample = dmod.settled_snapshot_device_memory++ def plan(self, *args, **kwargs):+ result = real_plan(self, *args, **kwargs)+ plans.append(result)+ return result++ def sample(target, *args, **kwargs):+ predicate = kwargs.get("can_stop")+ if predicate is not None:++ def record(memory):+ result = predicate(memory)+ accepted.append(result)+ return result++ kwargs["can_stop"] = record+ return real_sample(target, *args, **kwargs)++ monkeypatch.setattr(dmod.DiffusionBackend, "_plan_memory", plan)+ monkeypatch.setattr(dmod, "settled_snapshot_device_memory", sample)+ monkeypatch.setattr(+ dmod, "apply_memory_plan", lambda pipe, plan, **kwargs: (plan.offload_policy, False)+ )+ status = backend.load_pipeline(+ "Tongyi-MAI/Z-Image-Turbo", model_kind = "pipeline", _base_local_dir = str(tmp_path)+ )+ assert len(quantizations) == 1+ assert len(plans) >= 2+ assert plans[0].offload_policy != mem.OFFLOAD_NONE+ assert dmod._plan_proves_resident(plans[-1])+ assert accepted == [True]+ assert (card.reads, card.sleeps) == (4, 2)+ assert status["transformer_quant"] == "fp8"+ assert status["offload_policy"] == mem.OFFLOAD_NONE+ assert status["resolved"]["transformer_quant"]["status"] == "applied"+ assert status["speed_mode"] == "default"++[email protected]("enabled", [None, "0", "false", "1"])+def test_delayed_release_preserves_resident_denoiser(fake_runtime, tmp_path, monkeypatch, enabled):+ if enabled is None:+ monkeypatch.delenv("UNSLOTH_DIFFUSION_REUSE_SETTLED_MEMORY", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_DIFFUSION_REUSE_SETTLED_MEMORY", enabled)+ monkeypatch.setenv("UNSLOTH_SETTLE_DELAY_S", "1")+ monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, "1")+ monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: (0, 39))+ monkeypatch.setattr(mem, "_installed_torchao_version", lambda: (0, 18))+ monkeypatch.setattr(dmod, "compile_eligible", lambda *a, **k: True)+ _mock_cleanup(monkeypatch)+ backend = dmod.DiffusionBackend()+ quantizations = _stub_pipeline_dense_quant(backend, monkeypatch)+ clock = [0]+ plans = []+ real_plan = dmod.DiffusionBackend._plan_memory++ def sleep(delay):+ clock[0] += delay++ def snapshot(target):+ return DeviceMemory(+ "cuda", "cuda", "discrete_vram", 10_000 if clock[0] < 3 else 20_000, 24_000+ )++ def plan(self, *args, **kwargs):+ result = real_plan(self, *args, **kwargs)+ plans.append(result)+ return result++ monkeypatch.setattr("time.sleep", sleep)+ monkeypatch.setattr(mem, "snapshot_device_memory", snapshot)+ monkeypatch.setattr(dmod.DiffusionBackend, "_plan_memory", plan)+ monkeypatch.setattr(+ dmod, "apply_memory_plan", lambda pipe, plan, **kwargs: (plan.offload_policy, False)+ )+ status = backend.load_pipeline(+ "Tongyi-MAI/Z-Image-Turbo", model_kind = "pipeline", _base_local_dir = str(tmp_path)+ )+ assert len(quantizations) == 1+ assert len(plans) == 2+ assert [p.device_memory.free_mib for p in plans] == [10_000, 20_000]+ assert status["transformer_quant"] == "fp8"+ assert status["offload_policy"] == "group"