Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: fixed 2 GiB discrete-card reserve (ComfyUI-style fixed reserve plus activation estimate) (#14951)
Daniel Hanabe78a0530a1
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 031839c9658..01baa29f897 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1507,7 +1507,30 @@ def calibrated_image_activation()-DISCRETE_RESERVE_CAP_MIB = 4096+# ComfyUI's minimum_inference_memory() (comfy/model_management.py); activations are priced separately here.+COMFY_MIN_INFERENCE_MIB = 800+COMFY_EXTRA_RESERVED_MIB = 400+COMFY_EXTRA_RESERVED_WINDOWS_MIB = 600+COMFY_EXTRA_RESERVED_WINDOWS_BIG_CARD_MIB = 100+# Placements measured on T4 / 8-16 GB cards are tuned against 2 GiB; ComfyUI's 1.2 GiB alone flips them.+DISCRETE_RESERVE_FLOOR_MIB = 2048+++def comfy_min_inference_mib(total_mib: Optional[int] = None) -> int:+ """ComfyUI's ``minimum_inference_memory()`` in MiB: 1200 on Linux, 1400 / 1500 (card above 15 GiB) on Windows."""+ if sys.platform != "win32":+ return COMFY_MIN_INFERENCE_MIB + COMFY_EXTRA_RESERVED_MIB+ extra = COMFY_EXTRA_RESERVED_WINDOWS_MIB+ if int(total_mib or 0) > 15 * 1024:+ extra += COMFY_EXTRA_RESERVED_WINDOWS_BIG_CARD_MIB+ return COMFY_MIN_INFERENCE_MIB + extra+++def discrete_reserve_mib(total_mib: Optional[int] = None) -> int:+ """A discrete card's fixed reserve: ComfyUI's, floored at 2 GiB."""+ return max(DISCRETE_RESERVE_FLOOR_MIB, comfy_min_inference_mib(total_mib))++_ROCM_APU_POOL = "rocm_apu_pool"@@ -1524,11 +1547,8 @@ def _reserve_mib(memory_kind: str, base: int) -> int:# The amdgpu GTT pool of an APU whose host RAM already covers the OS: still shared with the CPU side, so it# keeps the uncapped 10%.return max(2048, int(base * 0.10))- # A discrete card's reserve covers fragmentation and other tenants' growth, not the load's own activations (the- # runtime headroom prices those). 10% of the TOTAL grew with the card: 17.8 GiB on a 178 GiB card, which refused- # a resident or companion-split plan on a partly used big card with plenty free. Capped near ComfyUI's fixed- # reserve (~1.2 GiB plus its activation estimate); cards up to 40 GiB keep 10% (the 2 GiB floor below 20 GiB).- return max(2048, min(int(base * 0.10), DISCRETE_RESERVE_CAP_MIB))+ # Fixed, not a share: a capped 10% kept 2.4-4 GiB idle on 24-40 GiB cards.+ return discrete_reserve_mib(base)def _rocm_linux_apu_os_room_outside_pool(memory: DeviceMemory) -> bool:@@ -2653,7 +2673,9 @@ def _calibrated_faster_tier(stream_transformer: bool,) -> Optional[tuple[str, bool, bool, bool, str]]:"""Strictly faster tier than the flat pick, else None; resident tiers must fit the 2048 denoise (it cannot tile).- Every check grows with the budget, so more VRAM never picks a slower tier."""+ A flat resident-transformer pick whose measured 2048 denoise does not fit beside the weights (the flat runtime+ headroom undercounts it) is demoted to the fastest tier that does fit, as ComfyUI offloads weights against its+ activation estimate. Every check grows with the budget, so more VRAM never picks a slower tier."""if budget is None or free_mib is None or model_dense_mib is None or companion_dense_mib is None:return Nonete = max(0, int(text_encoder_dense_mib or 0))@@ -2732,6 +2754,24 @@ def _calibrated_faster_tier(),),)+ # Only the 2048 denoise is checked here: it cannot tile, while an oversized decode is tiled by the generation guard.+ denoise_fits = (+ transformer + te + others <= room+ if flat_rank == 0+ else te > 0 and transformer + others <= room+ )+ if flat_rank <= 1 and not denoise_fits:+ for fits, (tier_policy, stream_te, stream_tr, tile, reason) in candidates[flat_rank + 1 :]:+ if fits:+ return (+ tier_policy,+ stream_te,+ stream_tr,+ tile,+ "the measured 2048 denoise does not fit beside the flat resident pick; "+ + reason,+ )+ return Nonefor rank, (fits, tier) in enumerate(candidates):if rank >= flat_rank:return Nonediff --git a/studio/backend/tests/test_diffusion_auto_placement.py b/studio/backend/tests/test_diffusion_auto_placement.pyindex b1fa8956a78..835ec3f4f44 100644--- a/studio/backend/tests/test_diffusion_auto_placement.py+++ b/studio/backend/tests/test_diffusion_auto_placement.py@@ -462,12 +462,9 @@ def test_encoder_card_pricing_never_admits_an_encoder_that_does_not_fit_the_free# The discrete reserve: a partly used big card.- "total, expected",- [(15360, 2048), (24576, 2457), (40960, 4096), (81920, 4096), (182272, 4096)],-)-def test_discrete_reserve_is_capped(total, expected):- assert dm._reserve_mib("discrete_vram", total) == expected[email protected]("total", [15360, 24576, 40960, 81920, 182272])+def test_discrete_reserve_is_fixed(total):+ assert dm._reserve_mib("discrete_vram", total) == 2048def test_partly_used_big_cards_get_the_companion_split(monkeypatch):@@ -484,6 +481,20 @@ def test_partly_used_big_cards_get_the_companion_split(monkeypatch):assert d.plan.secondary_ordinal == 1 and d.plan.offload_policy == dm.OFFLOAD_NONE+def test_partly_used_big_card_places_the_companion_split_with_the_fixed_reserve(monkeypatch):+ # 2x B200, 20 / 12 GiB free: the old 4 GiB reserve offloaded; 2 GiB puts the encoder on cuda:1.+ b200 = 182272+ host = Host(+ [+ Card(0, b200, b200 - 20 * GIB, cap = (10, 0), name = "B200"),+ Card(1, b200, b200 - 12 * GIB, cap = (10, 0), name = "B200"),+ ]+ )+ d = decide(monkeypatch, host, _sizes(dit = 12000, te = 9000), headroom = 3000)+ assert d.plan.secondary_ordinal == 1 and d.plan.offload_policy == dm.OFFLOAD_NONE+ assert _placement(d)["chosen"] == "companion_split"++def test_the_apu_pool_keeps_the_uncapped_reserve(monkeypatch):memory = dm.DeviceMemory("cuda", "cuda:0", "unified_memory", 31 * GIB, 96 * GIB)monkeypatch.setattr(dm, "_rocm_linux_apu_os_room_outside_pool", lambda m: True)diff --git a/studio/backend/tests/test_diffusion_discrete_reserve.py b/studio/backend/tests/test_diffusion_discrete_reserve.pynew file mode 100644index 00000000000..20c4e57b04e--- /dev/null+++ b/studio/backend/tests/test_diffusion_discrete_reserve.py@@ -0,0 +1,134 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The discrete VRAM reserve is fixed (ComfyUI's ``minimum_inference_memory``, floored at 2 GiB), not a share of the card.++Cards up to 20 GiB already sat on the old 2 GiB floor, so their plans are unchanged. CPU only.+"""++from __future__ import annotations++import sys+import types++import pytest++import core.inference.diffusion_memory as dm++GIB = 1024+CARD_TOTALS_MIB = (+ 8 * GIB,+ 10 * GIB,+ 12 * GIB,+ 15095, # T4 as torch reports it+ 16 * GIB,+ 20 * GIB,+ 24 * GIB,+ 32 * GIB,+ 40 * GIB,+ 48 * GIB,+ 80 * GIB,+ 96 * GIB,+ 141 * GIB,+ 182272, # B200+)+FREE_FRACTIONS = (1.0, 0.9, 0.75, 0.5, 0.25, 0.12)+++def _old_reserve_mib(total_mib: int) -> int:+ return max(2048, min(int(total_mib * 0.10), 4096))++[email protected]("platform", ["linux", "win32"])[email protected]("total", CARD_TOTALS_MIB)+def test_reserve_matrix_never_grows_and_small_cards_keep_theirs(monkeypatch, platform, total):+ monkeypatch.setattr(sys, "platform", platform)+ new = dm._reserve_mib("discrete_vram", total)+ old = _old_reserve_mib(total)+ assert new == 2048+ assert new <= old+ if total <= 20 * GIB:+ assert new == old+ for frac in FREE_FRACTIONS:+ free = int(total * frac)+ memory = dm.DeviceMemory("cuda", "cuda:0", "discrete_vram", free, total)+ budget = dm._safe_device_budget_mib(memory)+ assert budget == max(0, free - new)+ assert budget >= max(0, free - old)+ # an explicit fast load keeps its own 2 GiB minimum+ assert dm._fast_device_budget_mib(memory) == max(0, free - 2048)+++def test_comfy_minimum_inference_memory(monkeypatch):+ monkeypatch.setattr(sys, "platform", "linux")+ assert dm.comfy_min_inference_mib(8 * GIB) == 1200+ monkeypatch.setattr(sys, "platform", "win32")+ assert dm.comfy_min_inference_mib(8 * GIB) == 1400+ assert dm.comfy_min_inference_mib(16 * GIB) == 1500+ assert dm.discrete_reserve_mib(16 * GIB) == 2048+++# (transformer, text encoder, other companions) MiB: klein-4B, Z-Image, Q21 int8, FLUX.1 fp8, SDXL-sized+_LOADS = (+ (7393, 7673, 160),+ (11739, 7672, 160),+ (9800, 9100, 480),+ (11500, 9500, 160),+ (4900, 1600, 160),+)+++def _plan(total: int, free: int, dit: int, te: int, other: int, mode: str):+ return dm.plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = dm.DeviceMemory("cuda", "cuda", "discrete_vram", free, total),+ requested_mode = mode,+ model_dense_mib = dit + te + other,+ companion_dense_mib = te + other,+ text_encoder_dense_mib = te,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(width = None, height = None),+ )++[email protected]("total", [t for t in CARD_TOTALS_MIB if t <= 20 * GIB])[email protected]("mode", ["auto", "fast"])+def test_small_card_plans_are_identical_to_the_old_reserve(monkeypatch, total, mode):+ for frac in FREE_FRACTIONS:+ free = int(total * frac)+ for dit, te, other in _LOADS:+ new = _plan(total, free, dit, te, other, mode)+ with monkeypatch.context() as m:+ m.setattr(+ dm, "discrete_reserve_mib", lambda total_mib = None: _old_reserve_mib(total_mib)+ )+ old = _plan(total, free, dit, te, other, mode)+ assert new == old, (total, free, dit, te, mode)++[email protected]("total", [t for t in CARD_TOTALS_MIB if t > 20 * GIB])+def test_big_card_plans_only_move_toward_resident(monkeypatch, total):+ order = {+ dm.OFFLOAD_NONE: 0,+ "group": 1,+ "model": 1,+ dm.OFFLOAD_STREAMING: 2,+ "sequential": 3,+ }+ for frac in FREE_FRACTIONS:+ free = int(total * frac)+ for dit, te, other in _LOADS:+ new = _plan(total, free, dit, te, other, "auto")+ with monkeypatch.context() as m:+ m.setattr(+ dm, "discrete_reserve_mib", lambda total_mib = None: _old_reserve_mib(total_mib)+ )+ old = _plan(total, free, dit, te, other, "auto")+ assert (+ new.estimates["safe_device_budget_mib"] >= old.estimates["safe_device_budget_mib"]+ )+ assert order.get(new.offload_policy, 9) <= order.get(old.offload_policy, 9), (+ total,+ free,+ dit,+ te,+ )diff --git a/studio/backend/tests/test_diffusion_memory_rocm_apu_reserve.py b/studio/backend/tests/test_diffusion_memory_rocm_apu_reserve.pyindex d5b47855559..5cf8bdcce47 100644--- a/studio/backend/tests/test_diffusion_memory_rocm_apu_reserve.py+++ b/studio/backend/tests/test_diffusion_memory_rocm_apu_reserve.py@@ -114,10 +114,10 @@ def test_nvidia_unified_memory_unchanged(host):def test_discrete_vram_keeps_the_discrete_reserve(host):- # a real discrete card: 10% of the total, capped (diffusion_memory.DISCRETE_RESERVE_CAP_MIB)- assert _plan(kind = "discrete_vram").estimates["safe_device_budget_mib"] == POOL_FREE - min(- int(POOL_TOTAL * 0.10), dm.DISCRETE_RESERVE_CAP_MIB- )+ # a real discrete card: the fixed discrete reserve (diffusion_memory.discrete_reserve_mib)+ assert _plan(kind = "discrete_vram").estimates[+ "safe_device_budget_mib"+ ] == POOL_FREE - dm.discrete_reserve_mib(POOL_TOTAL)def test_fast_budget_follows_the_same_reserve(host):diff --git a/studio/backend/tests/test_video_ltx23_memory_plan.py b/studio/backend/tests/test_video_ltx23_memory_plan.pyindex 6b322952a05..42e9300cf40 100644--- a/studio/backend/tests/test_video_ltx23_memory_plan.py+++ b/studio/backend/tests/test_video_ltx23_memory_plan.py@@ -235,12 +235,14 @@ def test_fast_that_offloaded_is_recorded_as_a_fallback_naming_the_policy():assert (entry["status"], entry["value"]) == ("applied", "fast")-def test_resident_proof_uses_the_budget_fast_was_placed_with():+def test_resident_proof_uses_the_budget_fast_was_placed_with(monkeypatch):"""Auto precision keeps bf16 only on a proven fit; a fast plan placed resident under its own budget is one."""+ import core.inference.diffusion_memory as dmfrom core.inference.diffusion import _plan_proves_resident- # Free memory between the auto and the fast budget (the discrete reserve is capped at 4 GiB, fast keeps 2 GiB).- fast = _plan(79_500, LTX23_DIT_MIB)+ # Widen the 2 GiB reserve to 8 GiB (fast keeps half) so auto and fast budgets differ.+ monkeypatch.setattr(dm, "_reserve_mib", lambda kind, base: 8192)+ fast = _plan(81_500, LTX23_DIT_MIB)assert fast.offload_policy == OFFLOAD_NONEassert fast.estimates["resident_required_mib"] > fast.estimates["safe_device_budget_mib"]assert _plan_proves_resident(fast)