Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: report the measured VRAM shortfall instead of blaming the previous worker (#16453)
Yakshitha4becf86d2ff
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 1db58f9b4e3..5b1e2806090 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -996,6 +996,41 @@ class InferenceOrchestrator:previous = currentreturn not expected_free_gb+ @staticmethod+ def _describe_vram_shortfall(expected_free_gb: dict[int, float]) -> str:+ """Report measured free VRAM, or nothing when telemetry is unavailable."""+ try:+ from utils.hardware import get_visible_gpu_utilization, gpu_query+ with gpu_query.fresh_reads():+ devices = get_visible_gpu_utilization().get("devices", [])+ except Exception:+ return ""+ parts = []+ for device in devices:+ try:+ index = int(device["index"])+ needed = expected_free_gb.get(index)+ if needed is None:+ continue+ free = max(float(device["vram_total_gb"]) - float(device["vram_used_gb"]), 0.0)+ parts.append(+ f"free {free:.1f} GiB of the {float(needed):.1f} GiB needed on cuda:{index}"+ )+ except (KeyError, TypeError, ValueError):+ continue+ return "; ".join(parts)++ @staticmethod+ def _vram_wait_message(expected_free_gb: dict[int, float]) -> str:+ """The settle check cannot identify which process holds VRAM (#16436)."""+ shortfall = InferenceOrchestrator._describe_vram_shortfall(expected_free_gb)+ detail = f" ({shortfall})" if shortfall else ""+ return (+ "GPU memory has not freed up yet; not starting the replacement model"+ f"{detail}. A reply may still be finishing, or another app may be "+ "holding the GPU. Retry shortly."+ )+def _reset_worker_scoped_state(self) -> None:"""Drop bookkeeping that only means anything for the worker that just died."""with self._active_cancel_lock:@@ -2181,10 +2216,7 @@ class InferenceOrchestrator:expected_free_gb = expected_free_gb,)if expected_free_gb is not None and not settled:- raise RuntimeError(- "GPU memory from the previous inference worker was not released; "- "not starting the replacement model. Retry shortly."- )+ raise RuntimeError(self._vram_wait_message(expected_free_gb))# Previous worker gone, VRAM back: the last moment before a long download.if on_prior_worker_released is not None:diff --git a/studio/backend/tests/test_audio_device_guards.py b/studio/backend/tests/test_audio_device_guards.pyindex 791ed1e5fbf..2d439dfc8c7 100644--- a/studio/backend/tests/test_audio_device_guards.py+++ b/studio/backend/tests/test_audio_device_guards.py@@ -496,7 +496,7 @@ def test_the_chat_claim_outlives_the_worker_that_earned_it():load_src = inspect.getsource(InferenceOrchestrator.load_model)hook = load_src.index("on_prior_worker_released()")assert load_src.index("did not exit and still holds GPU") < hook- assert load_src.index("was not released; ") < hook+ assert load_src.index("if expected_free_gb is not None and not settled:") < hookdef test_whisper_cpp_publishes_the_load_before_choosing_its_placement():diff --git a/studio/backend/tests/test_orchestrator_vram_shortfall_message.py b/studio/backend/tests/test_orchestrator_vram_shortfall_message.pynew file mode 100644index 00000000000..a06ad57087c--- /dev/null+++ b/studio/backend/tests/test_orchestrator_vram_shortfall_message.py@@ -0,0 +1,171 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from contextlib import nullcontext+import importlib.util+from pathlib import Path+import sys+import threading+from types import SimpleNamespace++import pytest+++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)+++def _orchestrator_class():+ spec = importlib.util.spec_from_file_location(+ "inference_orchestrator_shortfall_under_test",+ Path(__file__).resolve().parent.parent / "core/inference/orchestrator.py",+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module.InferenceOrchestrator+++def _devices(monkeypatch, devices):+ import utils.hardware as hardware+ from utils.hardware import gpu_query++ monkeypatch.setattr(hardware, "get_visible_gpu_utilization", lambda: {"devices": devices})+ monkeypatch.setattr(gpu_query, "fresh_reads", lambda: nullcontext())+++def test_shortfall_reports_free_and_needed(monkeypatch):+ _devices(monkeypatch, [{"index": 0, "vram_total_gb": 7.93, "vram_used_gb": 6.2}])++ text = _orchestrator_class()._describe_vram_shortfall({0: 3.9})++ assert "free 1.7 GiB" in text+ assert "3.9 GiB needed on cuda:0" in text+++def test_shortfall_covers_every_gpu_that_was_expected(monkeypatch):+ _devices(+ monkeypatch,+ [+ {"index": 0, "vram_total_gb": 8.0, "vram_used_gb": 6.0},+ {"index": 1, "vram_total_gb": 8.0, "vram_used_gb": 1.0},+ ],+ )++ text = _orchestrator_class()._describe_vram_shortfall({0: 3.9, 1: 3.9})++ assert "cuda:0" in text and "cuda:1" in text+ assert text.count("GiB needed") == 2+++def test_shortfall_is_empty_when_telemetry_raises(monkeypatch):+ import utils.hardware as hardware++ def boom():+ raise RuntimeError("no GPU telemetry here")++ monkeypatch.setattr(hardware, "get_visible_gpu_utilization", boom)++ assert _orchestrator_class()._describe_vram_shortfall({0: 3.9}) == ""+++def test_shortfall_ignores_gpus_that_were_not_expected(monkeypatch):+ _devices(monkeypatch, [{"index": 2, "vram_total_gb": 8.0, "vram_used_gb": 7.0}])++ assert _orchestrator_class()._describe_vram_shortfall({0: 3.9}) == ""+++def test_message_names_both_holders_and_carries_the_measurement(monkeypatch):+ _devices(monkeypatch, [{"index": 0, "vram_total_gb": 7.93, "vram_used_gb": 6.2}])++ text = _orchestrator_class()._vram_wait_message({0: 3.9})++ assert text.startswith("GPU memory has not freed up yet")+ assert "free 1.7 GiB of the 3.9 GiB needed on cuda:0" in text+ assert "A reply may still be finishing" in text+ assert "another app" in text+ assert "Retry shortly." in text+ assert "previous inference worker" not in text+++def test_message_still_reads_well_without_telemetry(monkeypatch):+ import utils.hardware as hardware++ def boom():+ raise RuntimeError("no GPU telemetry here")++ monkeypatch.setattr(hardware, "get_visible_gpu_utilization", boom)++ text = _orchestrator_class()._vram_wait_message({0: 3.9})++ assert text.startswith("GPU memory has not freed up yet")+ assert "(" not in text.split("model")[0]+ assert "Retry shortly." in text++[email protected]("settled", [False, True])[email protected]("telemetry_available", [False, True])+def test_load_uses_shortfall_message_only_after_failed_settle(+ monkeypatch, settled, telemetry_available+):+ from core.inference import orchestrator as module+ from utils import hardware, transformers_version+ from utils.hardware import gpu_query++ cls = module.InferenceOrchestrator+ backend = cls.__new__(cls)+ backend.__dict__.update(+ loading_models = set(),+ models = {},+ active_model_name = None,+ _proc = None,+ _subprocess_shutdown_lock = threading.RLock(),+ _load_download_keys = [],+ )+ events = []++ def telemetry():+ events.append("telemetry")+ if not telemetry_available:+ raise RuntimeError("telemetry unavailable")+ return {"devices": [{"index": 0, "vram_total_gb": 7.93, "vram_used_gb": 6.2}]}++ def wait(_since, **kwargs):+ assert kwargs["expected_free_gb"] == {0: 3.9}+ events.append("settle")+ return settled++ monkeypatch.setattr(hardware, "get_visible_gpu_utilization", telemetry)+ monkeypatch.setattr(gpu_query, "fresh_reads", nullcontext)+ monkeypatch.setattr(module, "get_device", lambda: SimpleNamespace(value = "cpu"))+ monkeypatch.setattr(module, "prepare_gpu_selection", lambda *a, **k: ([0], {}))+ monkeypatch.setattr(transformers_version, "needs_transformers_5", lambda _: False)+ monkeypatch.setattr(transformers_version, "sidecar_swap_kind", lambda: None)+ monkeypatch.setattr(backend, "_ensure_subprocess_alive", lambda: False)+ monkeypatch.setattr(backend, "_shutdown_subprocess", lambda **k: True)+ monkeypatch.setattr(backend, "_wait_for_worker_vram_settle", wait)+ monkeypatch.setattr(backend, "_spawn_subprocess", lambda _: events.append("spawn"))+ monkeypatch.setattr(backend, "_wait_response", lambda *a: {"success": True})++ def load():+ return backend.load_model(+ SimpleNamespace(identifier = "test/replacement"),+ post_handoff_expected_free_gb = {0: 3.9},+ on_prior_worker_released = lambda: events.append("released"),+ )++ if settled:+ assert load() is True+ assert events == ["settle", "released", "spawn"]+ assert backend.active_model_name == "test/replacement"+ else:+ with pytest.raises(RuntimeError) as error:+ load()+ text = str(error.value)+ assert "previous inference worker" not in text+ assert "A reply may still be finishing" in text+ if telemetry_available:+ assert "free 1.7 GiB of the 3.9 GiB needed on cuda:0" in text+ assert events == ["settle", "telemetry"]+ assert backend.active_model_name is None+ assert backend.models == {}