Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: restore a kept media model's price after a failed replacement (#14826)

Daniel Han3e35cbb4965d
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index ce11e993c2d..0d4a817796a 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -5194,10 +5194,11 @@ class DiffusionBackend:
# sat unreproducible for want of the frames this logs.
logger.error("diffusion.load_failed: %s", exc, exc_info = True)
if self._state is not None:
- from .gpu_arbiter import DIFFUSION, restore_owner_account
+ from .gpu_arbiter import DIFFUSION, restore_media_need, restore_owner_account
from hub.services.models.account_access import restore_resident_metadata
restore_owner_account(DIFFUSION)
+ restore_media_need(DIFFUSION)
restore_resident_metadata(DIFFUSION)
try:
clear_gpu_cache()
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index 69208750416..ab943f575fc 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -50,6 +50,9 @@ _residents: dict[str, Optional[frozenset[int]]] = {}
_accounts: dict[str, tuple[Optional[str], Optional[str]]] = {}
# What each image or video resident was admitted with under When they fit / Always (gpu_residency.MediaNeed).
_media_needs: dict[str, dict] = {}
+# The price each media owner had before its newest load started (None: unpriced), put back by
+# restore_media_need when that load fails and the model it was replacing stays loaded.
+_media_need_rollback: dict[str, Optional[dict]] = {}
# Phrases the refusal copy always contains, for readers that lose the X-Unsloth-Refusal header.
MULTI_MODEL_REFUSAL_MARKERS = ("is set to Always", "is set to keep loaded")
@@ -430,6 +433,9 @@ def begin_media_load(
) -> Any:
"""``guarded_media_begin`` for a load that takes no GPU claim, under ``_lock`` all the same."""
register = guarded_media_begin(owner, policy, requested, begin)
+ with _lock:
+ # This load reprices nothing, so an older load's rollback must not apply to its failure.
+ _media_need_rollback.pop(owner, None)
if register is begin:
return begin()
with _lock:
@@ -545,6 +551,7 @@ def _drop_resident(owner: str) -> None:
return
del _residents[owner]
_media_needs.pop(owner, None)
+ _media_need_rollback.pop(owner, None)
if owner == _owner:
if _residents:
_owner = next(reversed(_residents))
@@ -786,6 +793,7 @@ def acquire_for(
raise
if owner in (DIFFUSION, VIDEO) and register is not None:
# The price media_held_mib() reports for this resident; unpriced residents are evicted to make room.
+ _media_need_rollback[owner] = _media_needs.get(owner)
if coreside and need_mib:
_media_needs[owner] = need_mib
else:
@@ -887,6 +895,19 @@ def restore_owner_account(owner: str, account_id: Optional[str] = None) -> bool:
return True
+def restore_media_need(owner: str) -> None:
+ """Put back the price the newest load of ``owner`` overwrote, when that load failed and the
+ model it was replacing stays loaded; otherwise the kept model is priced as the failed one."""
+ with _lock:
+ if owner not in _media_need_rollback:
+ return
+ previous = _media_need_rollback.pop(owner)
+ if previous is None:
+ _media_needs.pop(owner, None)
+ else:
+ _media_needs[owner] = previous
+
+
def release(owner: str) -> None:
"""Drop ``owner``'s residency (no-op if it isn't resident)."""
with _lock:
diff --git a/studio/backend/core/inference/mlx_diffusion_backend.py b/studio/backend/core/inference/mlx_diffusion_backend.py
index 18ba28e3844..cd83775ad08 100644
--- a/studio/backend/core/inference/mlx_diffusion_backend.py
+++ b/studio/backend/core/inference/mlx_diffusion_backend.py
@@ -1132,10 +1132,11 @@ class MlxDiffusionBackend:
return
logger.error("mlx_diffusion.load_failed: %s", exc, exc_info = True)
if self._state is not None:
- from .gpu_arbiter import DIFFUSION, restore_owner_account
+ from .gpu_arbiter import DIFFUSION, restore_media_need, restore_owner_account
from hub.services.models.account_access import restore_resident_metadata
restore_owner_account(DIFFUSION)
+ restore_media_need(DIFFUSION)
restore_resident_metadata("diffusion")
from utils.native_path_leases import redact_native_paths
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index 11cacf0dc6e..b1caf97e60b 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -3868,10 +3868,11 @@ class SdCppDiffusionBackend:
logger.error("sd_cpp.load_failed: %s", exc)
if self._state is not None:
# Displaced pipeline: give its account back GPU residency, drop its records.
- from .gpu_arbiter import DIFFUSION, restore_owner_account
+ from .gpu_arbiter import DIFFUSION, restore_media_need, restore_owner_account
from hub.services.models.account_access import restore_resident_metadata
restore_owner_account(DIFFUSION)
+ restore_media_need(DIFFUSION)
restore_resident_metadata("diffusion")
# Redact filesystem paths before this reaches /images/load-progress (as diffusers does).
from utils.native_path_leases import redact_native_paths
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index f9374fdb9c2..f777a5f3923 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -4995,10 +4995,11 @@ class VideoBackend:
return
logger.error("video.load_failed: %s", exc)
if self._state is not None:
- from .gpu_arbiter import VIDEO, restore_owner_account
+ from .gpu_arbiter import VIDEO, restore_media_need, restore_owner_account
from hub.services.models.account_access import restore_resident_metadata
restore_owner_account(VIDEO)
+ restore_media_need(VIDEO)
restore_resident_metadata(VIDEO)
# Free the debris of a failed construction: nothing was committed, so nothing else releases the VRAM.
# NVFP4 caches pin the denoiser, but a failed replacement keeps the old model, whose graph still uses them.
diff --git a/studio/backend/tests/test_account_native_diffusion_load_rollback.py b/studio/backend/tests/test_account_native_diffusion_load_rollback.py
index 9e5d2f891a3..b441f298175 100644
--- a/studio/backend/tests/test_account_native_diffusion_load_rollback.py
+++ b/studio/backend/tests/test_account_native_diffusion_load_rollback.py
@@ -90,3 +90,36 @@ def test_failed_native_cpu_load_leaves_a_released_arbiter_alone(monkeypatch):
)
assert gpu_arbiter.current_owner() is None
assert gpu_arbiter.owner_account() is None
+
+
+def test_failed_native_replacement_keeps_the_resident_priced_as_itself(monkeypatch):
+ backend = _failing_backend(monkeypatch)
+ old, new = {0: 8000}, {0: 1000}
+ monkeypatch.setattr(gpu_arbiter, "_residents", {"diffusion": frozenset({0})})
+ monkeypatch.setattr(gpu_arbiter, "_media_needs", {"diffusion": old})
+ monkeypatch.setattr(gpu_arbiter, "_media_need_rollback", {}, raising = False)
+ monkeypatch.setattr(gpu_arbiter, "physical_gpu_ids", lambda ids: None)
+ run_as(
+ BOB,
+ gpu_arbiter.acquire_for,
+ "diffusion",
+ lambda: None,
+ policy = "fit",
+ need_mib = new,
+ )
+ assert gpu_arbiter._media_needs["diffusion"] == new
+
+ run_as(
+ BOB,
+ lambda: backend._run_load(
+ repo_id = "org/bob-model",
+ gguf_filename = "flux1-dev-Q4_K_M.gguf",
+ base = "",
+ fam = None,
+ hf_token = None,
+ _load_token = backend._load_token,
+ ),
+ )
+
+ assert backend._loading.error and backend._state is not None
+ assert gpu_arbiter._media_needs["diffusion"] == old
diff --git a/studio/backend/tests/test_gpu_arbiter.py b/studio/backend/tests/test_gpu_arbiter.py
index 1f700d513f6..4bcd94ea355 100644
--- a/studio/backend/tests/test_gpu_arbiter.py
+++ b/studio/backend/tests/test_gpu_arbiter.py
@@ -1090,3 +1090,35 @@ def test_a_respawning_chat_server_counts_as_occupied(monkeypatch):
kept = types.SimpleNamespace(llama = types.SimpleNamespace(is_active = True, is_loaded = False))
monkeypatch.setattr(model_slots, "slots", [kept])
assert arb._chat_occupied() is True
+
+
+def test_a_failed_replacement_gives_the_kept_model_back_its_price(cores, monkeypatch):
+ # The new price is written as the load starts; its failure keeps the old model, which must
+ # not stay priced as the smaller model that never loaded.
+ monkeypatch.setattr(arb, "_media_need_rollback", {}, raising = False)
+ old, new = {0: 8000}, {0: 1000}
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = old)
+ arb.acquire_for(arb.DIFFUSION, lambda: {"loading": True}, policy = "fit", need_mib = new)
+ assert arb._media_needs[arb.DIFFUSION] == new
+
+ arb.restore_media_need(arb.DIFFUSION)
+ assert arb._media_needs[arb.DIFFUSION] == old
+ # Applied once: a repeated restore changes nothing.
+ arb.restore_media_need(arb.DIFFUSION)
+ assert arb._media_needs[arb.DIFFUSION] == old
+
+
+def test_a_failed_first_load_and_an_unpriced_one_restore_nothing_stale(cores, monkeypatch):
+ monkeypatch.setattr(arb, "_media_need_rollback", {}, raising = False)
+ # A first load had no price before it; its rollback is to none.
+ arb.acquire_for(arb.VIDEO, lambda: None, policy = "fit", need_mib = {0: 4096})
+ arb.restore_media_need(arb.VIDEO)
+ assert arb.VIDEO not in arb._media_needs
+
+ # A load that takes no GPU claim reprices nothing, so its failure must not reach back to an
+ # older load's rollback.
+ arb.acquire_for(arb.VIDEO, lambda: None, policy = "fit", need_mib = {0: 4096})
+ arb.acquire_for(arb.VIDEO, lambda: None, policy = "fit", need_mib = {0: 2048})
+ arb.begin_media_load(arb.VIDEO, "fit", ("org/video",), lambda: None)
+ arb.restore_media_need(arb.VIDEO)
+ assert arb._media_needs[arb.VIDEO] == {0: 2048}