Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: report the measured VRAM shortfall instead of blaming the previous worker (#16453)

Yakshitha4becf86d2ff
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 1db58f9b4e3..5b1e2806090 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -996,6 +996,41 @@ class InferenceOrchestrator:
previous = current
return not expected_free_gb
+ @staticmethod
+ def _describe_vram_shortfall(expected_free_gb: dict[int, float]) -> str:
+ """Report measured free VRAM, or nothing when telemetry is unavailable."""
+ try:
+ from utils.hardware import get_visible_gpu_utilization, gpu_query
+ with gpu_query.fresh_reads():
+ devices = get_visible_gpu_utilization().get("devices", [])
+ except Exception:
+ return ""
+ parts = []
+ for device in devices:
+ try:
+ index = int(device["index"])
+ needed = expected_free_gb.get(index)
+ if needed is None:
+ continue
+ free = max(float(device["vram_total_gb"]) - float(device["vram_used_gb"]), 0.0)
+ parts.append(
+ f"free {free:.1f} GiB of the {float(needed):.1f} GiB needed on cuda:{index}"
+ )
+ except (KeyError, TypeError, ValueError):
+ continue
+ return "; ".join(parts)
+
+ @staticmethod
+ def _vram_wait_message(expected_free_gb: dict[int, float]) -> str:
+ """The settle check cannot identify which process holds VRAM (#16436)."""
+ shortfall = InferenceOrchestrator._describe_vram_shortfall(expected_free_gb)
+ detail = f" ({shortfall})" if shortfall else ""
+ return (
+ "GPU memory has not freed up yet; not starting the replacement model"
+ f"{detail}. A reply may still be finishing, or another app may be "
+ "holding the GPU. Retry shortly."
+ )
+
def _reset_worker_scoped_state(self) -> None:
"""Drop bookkeeping that only means anything for the worker that just died."""
with self._active_cancel_lock:
@@ -2181,10 +2216,7 @@ class InferenceOrchestrator:
expected_free_gb = expected_free_gb,
)
if expected_free_gb is not None and not settled:
- raise RuntimeError(
- "GPU memory from the previous inference worker was not released; "
- "not starting the replacement model. Retry shortly."
- )
+ raise RuntimeError(self._vram_wait_message(expected_free_gb))
# Previous worker gone, VRAM back: the last moment before a long download.
if on_prior_worker_released is not None:
diff --git a/studio/backend/tests/test_audio_device_guards.py b/studio/backend/tests/test_audio_device_guards.py
index 791ed1e5fbf..2d439dfc8c7 100644
--- a/studio/backend/tests/test_audio_device_guards.py
+++ b/studio/backend/tests/test_audio_device_guards.py
@@ -496,7 +496,7 @@ def test_the_chat_claim_outlives_the_worker_that_earned_it():
load_src = inspect.getsource(InferenceOrchestrator.load_model)
hook = load_src.index("on_prior_worker_released()")
assert load_src.index("did not exit and still holds GPU") < hook
- assert load_src.index("was not released; ") < hook
+ assert load_src.index("if expected_free_gb is not None and not settled:") < hook
def test_whisper_cpp_publishes_the_load_before_choosing_its_placement():
diff --git a/studio/backend/tests/test_orchestrator_vram_shortfall_message.py b/studio/backend/tests/test_orchestrator_vram_shortfall_message.py
new file mode 100644
index 00000000000..a06ad57087c
--- /dev/null
+++ b/studio/backend/tests/test_orchestrator_vram_shortfall_message.py
@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from contextlib import nullcontext
+import importlib.util
+from pathlib import Path
+import sys
+import threading
+from types import SimpleNamespace
+
+import pytest
+
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+
+def _orchestrator_class():
+ spec = importlib.util.spec_from_file_location(
+ "inference_orchestrator_shortfall_under_test",
+ Path(__file__).resolve().parent.parent / "core/inference/orchestrator.py",
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module.InferenceOrchestrator
+
+
+def _devices(monkeypatch, devices):
+ import utils.hardware as hardware
+ from utils.hardware import gpu_query
+
+ monkeypatch.setattr(hardware, "get_visible_gpu_utilization", lambda: {"devices": devices})
+ monkeypatch.setattr(gpu_query, "fresh_reads", lambda: nullcontext())
+
+
+def test_shortfall_reports_free_and_needed(monkeypatch):
+ _devices(monkeypatch, [{"index": 0, "vram_total_gb": 7.93, "vram_used_gb": 6.2}])
+
+ text = _orchestrator_class()._describe_vram_shortfall({0: 3.9})
+
+ assert "free 1.7 GiB" in text
+ assert "3.9 GiB needed on cuda:0" in text
+
+
+def test_shortfall_covers_every_gpu_that_was_expected(monkeypatch):
+ _devices(
+ monkeypatch,
+ [
+ {"index": 0, "vram_total_gb": 8.0, "vram_used_gb": 6.0},
+ {"index": 1, "vram_total_gb": 8.0, "vram_used_gb": 1.0},
+ ],
+ )
+
+ text = _orchestrator_class()._describe_vram_shortfall({0: 3.9, 1: 3.9})
+
+ assert "cuda:0" in text and "cuda:1" in text
+ assert text.count("GiB needed") == 2
+
+
+def test_shortfall_is_empty_when_telemetry_raises(monkeypatch):
+ import utils.hardware as hardware
+
+ def boom():
+ raise RuntimeError("no GPU telemetry here")
+
+ monkeypatch.setattr(hardware, "get_visible_gpu_utilization", boom)
+
+ assert _orchestrator_class()._describe_vram_shortfall({0: 3.9}) == ""
+
+
+def test_shortfall_ignores_gpus_that_were_not_expected(monkeypatch):
+ _devices(monkeypatch, [{"index": 2, "vram_total_gb": 8.0, "vram_used_gb": 7.0}])
+
+ assert _orchestrator_class()._describe_vram_shortfall({0: 3.9}) == ""
+
+
+def test_message_names_both_holders_and_carries_the_measurement(monkeypatch):
+ _devices(monkeypatch, [{"index": 0, "vram_total_gb": 7.93, "vram_used_gb": 6.2}])
+
+ text = _orchestrator_class()._vram_wait_message({0: 3.9})
+
+ assert text.startswith("GPU memory has not freed up yet")
+ assert "free 1.7 GiB of the 3.9 GiB needed on cuda:0" in text
+ assert "A reply may still be finishing" in text
+ assert "another app" in text
+ assert "Retry shortly." in text
+ assert "previous inference worker" not in text
+
+
+def test_message_still_reads_well_without_telemetry(monkeypatch):
+ import utils.hardware as hardware
+
+ def boom():
+ raise RuntimeError("no GPU telemetry here")
+
+ monkeypatch.setattr(hardware, "get_visible_gpu_utilization", boom)
+
+ text = _orchestrator_class()._vram_wait_message({0: 3.9})
+
+ assert text.startswith("GPU memory has not freed up yet")
+ assert "(" not in text.split("model")[0]
+ assert "Retry shortly." in text
+
+
[email protected]("settled", [False, True])
[email protected]("telemetry_available", [False, True])
+def test_load_uses_shortfall_message_only_after_failed_settle(
+ monkeypatch, settled, telemetry_available
+):
+ from core.inference import orchestrator as module
+ from utils import hardware, transformers_version
+ from utils.hardware import gpu_query
+
+ cls = module.InferenceOrchestrator
+ backend = cls.__new__(cls)
+ backend.__dict__.update(
+ loading_models = set(),
+ models = {},
+ active_model_name = None,
+ _proc = None,
+ _subprocess_shutdown_lock = threading.RLock(),
+ _load_download_keys = [],
+ )
+ events = []
+
+ def telemetry():
+ events.append("telemetry")
+ if not telemetry_available:
+ raise RuntimeError("telemetry unavailable")
+ return {"devices": [{"index": 0, "vram_total_gb": 7.93, "vram_used_gb": 6.2}]}
+
+ def wait(_since, **kwargs):
+ assert kwargs["expected_free_gb"] == {0: 3.9}
+ events.append("settle")
+ return settled
+
+ monkeypatch.setattr(hardware, "get_visible_gpu_utilization", telemetry)
+ monkeypatch.setattr(gpu_query, "fresh_reads", nullcontext)
+ monkeypatch.setattr(module, "get_device", lambda: SimpleNamespace(value = "cpu"))
+ monkeypatch.setattr(module, "prepare_gpu_selection", lambda *a, **k: ([0], {}))
+ monkeypatch.setattr(transformers_version, "needs_transformers_5", lambda _: False)
+ monkeypatch.setattr(transformers_version, "sidecar_swap_kind", lambda: None)
+ monkeypatch.setattr(backend, "_ensure_subprocess_alive", lambda: False)
+ monkeypatch.setattr(backend, "_shutdown_subprocess", lambda **k: True)
+ monkeypatch.setattr(backend, "_wait_for_worker_vram_settle", wait)
+ monkeypatch.setattr(backend, "_spawn_subprocess", lambda _: events.append("spawn"))
+ monkeypatch.setattr(backend, "_wait_response", lambda *a: {"success": True})
+
+ def load():
+ return backend.load_model(
+ SimpleNamespace(identifier = "test/replacement"),
+ post_handoff_expected_free_gb = {0: 3.9},
+ on_prior_worker_released = lambda: events.append("released"),
+ )
+
+ if settled:
+ assert load() is True
+ assert events == ["settle", "released", "spawn"]
+ assert backend.active_model_name == "test/replacement"
+ else:
+ with pytest.raises(RuntimeError) as error:
+ load()
+ text = str(error.value)
+ assert "previous inference worker" not in text
+ assert "A reply may still be finishing" in text
+ if telemetry_available:
+ assert "free 1.7 GiB of the 3.9 GiB needed on cuda:0" in text
+ assert events == ["settle", "telemetry"]
+ assert backend.active_model_name is None
+ assert backend.models == {}