Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: recognise a worker VRAM release that finished before the first settle poll (#16404)
Daniel Handfad9c534b24
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 5c79b13a1f2..3e715786b6f 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -1061,6 +1061,26 @@ class InferenceOrchestrator:seen = self._replies_readidle_deadline = now + _SHUTDOWN_READER_IDLE_S+ @staticmethod+ def _free_vram_mib() -> Optional[dict[int, int]]:+ """Free MiB per visible device from fresh live telemetry, or None when unreadable."""+ from utils.hardware import get_visible_gpu_utilization+ try:+ from utils.hardware import gpu_query++ result: dict[int, int] = {}+ # Consecutive samples are compared: a cached one would read as settled.+ with gpu_query.fresh_reads():+ devices = get_visible_gpu_utilization().get("devices", [])+ for device in devices:+ index = int(device["index"])+ total = float(device["vram_total_gb"])+ used = float(device["vram_used_gb"])+ result[index] = max(int((total - used) * 1024), 0)+ return result or None+ except (KeyError, TypeError, ValueError, OverflowError):+ return None+@staticmethoddef _wait_for_worker_vram_settle(since_kill: float,@@ -1069,10 +1089,9 @@ class InferenceOrchestrator:max_wait: float = 2.0,interval: float = 0.25,tolerance_mib: int = 256,+ before_kill_free_mib: Optional[dict[int, int]] = None,) -> bool:"""Poll cross-platform live telemetry until dead-worker VRAM stabilises."""- from utils.hardware import get_visible_gpu_utilization-if since_kill <= 0:return not expected_free_gbdeadline = time.monotonic() + max(max_wait, 0.0)@@ -1080,21 +1099,14 @@ class InferenceOrchestrator:def _probe() -> Optional[dict[int, int]]:if time.monotonic() >= deadline:return None- try:- from utils.hardware import gpu_query+ return InferenceOrchestrator._free_vram_mib()- result: dict[int, int] = {}- # Consecutive samples are compared: a cached one would read as settled.- with gpu_query.fresh_reads():- devices = get_visible_gpu_utilization().get("devices", [])- for device in devices:- index = int(device["index"])- total = float(device["vram_total_gb"])- used = float(device["vram_used_gb"])- result[index] = max(int((total - used) * 1024), 0)- return result or None- except (KeyError, TypeError, ValueError, OverflowError):- return None+ def _released(after: dict[int, int], before: dict[int, int]) -> bool:+ return any(+ after[index] - before[index]+ >= max(tolerance_mib, int(max(after[index], before[index]) * 0.02))+ for index in after+ )previous = _probe()if not previous:@@ -1111,7 +1123,16 @@ class InferenceOrchestrator:if _threshold_reached(previous):return True- observed_reclaim = False+ # A release that landed before the first poll is still a release: without the pre-kill+ # sample it read as stable-but-unreleased and the wait ran to its deadline.+ observed_reclaim = bool(+ before_kill_free_mib+ and before_kill_free_mib.keys() == previous.keys()+ and _released(previous, before_kill_free_mib)+ )+ # A release seen only against the pre-kill sample may be partial: ask for two stable polls.+ stable_needed = 2 if observed_reclaim else 1+ stable_run = 0while time.monotonic() < deadline:remaining = deadline - time.monotonic()if remaining <= 0:@@ -1120,12 +1141,9 @@ class InferenceOrchestrator:current = _probe()if not current or current.keys() != previous.keys():return not expected_free_gb- if any(- current[index] - previous[index]- >= max(tolerance_mib, int(max(current[index], previous[index]) * 0.02))- for index in current- ):+ if _released(current, previous):observed_reclaim = True+ stable_needed = 1if _threshold_reached(current):return Truestable = all(@@ -1133,8 +1151,9 @@ class InferenceOrchestrator:< max(tolerance_mib, int(max(current[index], previous[index]) * 0.02))for index in current)+ stable_run = stable_run + 1 if stable else 0# Stable low samples can precede delayed driver reclaim; trust stability only after a release.- if stable and observed_reclaim and not expected_mib:+ if stable and observed_reclaim and not expected_mib and stable_run >= stable_needed:return Trueprevious = currentreturn not expected_free_gb@@ -2359,8 +2378,20 @@ class InferenceOrchestrator:# Always spawn fresh: unsloth patches torch internals, breaking getsource on reuse.had_worker_handle = self._proc is not Noneworker_shutdown_at = 0.0+ before_kill_free_mib = Noneif self._ensure_subprocess_alive():self._cancel_generation()+ if post_handoff_expected_free_gb is None and not (+ len(resolved_gpu_ids or ()) == 1+ and isinstance(gpu_selection, dict)+ and gpu_selection.get("required_gb") is not None+ ):+ # No free-VRAM target for the settle wait below: sample the live worker's+ # footprint so a release already finished by the first poll is recognised.+ try:+ before_kill_free_mib = self._free_vram_mib()+ except Exception: # noqa: BLE001 - an unreadable sample keeps the plain settle wait+ before_kill_free_mib = Noneif self._shutdown_subprocess() is False:# Worker survived kill (wedged CUDA syscall); do not spawn a second over its GPU memory.raise RuntimeError(@@ -2386,6 +2417,7 @@ class InferenceOrchestrator:settled = self._wait_for_worker_vram_settle(worker_shutdown_at or time.monotonic(),expected_free_gb = expected_free_gb,+ before_kill_free_mib = before_kill_free_mib,)if expected_free_gb is not None and not settled:raise RuntimeError(self._vram_wait_message(expected_free_gb))diff --git a/studio/backend/tests/test_inference_worker_vram_settle.py b/studio/backend/tests/test_inference_worker_vram_settle.pynew file mode 100644index 00000000000..ec97cc69f70--- /dev/null+++ b/studio/backend/tests/test_inference_worker_vram_settle.py@@ -0,0 +1,108 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import importlib.util+import sys+import types+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)+++def settle(monkeypatch):+ spec = importlib.util.spec_from_file_location(+ "orchestrator_vram_settle_under_test",+ Path(_BACKEND_DIR) / "core/inference/orchestrator.py",+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ clock = [100.0]+ samples: list = []+ probes = [0]+ module.time = types.SimpleNamespace(+ monotonic = lambda: clock[0],+ sleep = lambda s: clock.__setitem__(0, clock[0] + s),+ time = lambda: clock[0],+ )++ def free_vram_mib():+ probes[0] += 1+ return dict(samples.pop(0)) if samples else None++ monkeypatch.setattr(module.InferenceOrchestrator, "_free_vram_mib", staticmethod(free_vram_mib))++ def run(sequence, **kwargs):+ samples[:] = list(sequence)+ clock[0] = 100.0+ probes[0] = 0+ result = module.InferenceOrchestrator._wait_for_worker_vram_settle(1.0, **kwargs)+ return result, round(clock[0] - 100.0, 6), probes[0]++ return run+++def test_release_before_first_poll_is_recognised_with_pre_kill_sample(settle):+ freed = [{0: 40_000}] * 20+ # Without the pre-kill sample the already-freed card looks stable but unreleased: full 2 s.+ assert settle(freed) == (True, 2.0, 8)+ # With it, the gain since the kill counts as the release; two stable samples confirm it.+ assert settle(freed, before_kill_free_mib = {0: 30_000}) == (True, 0.5, 3)+++def test_pre_kill_sample_without_a_release_still_waits(settle):+ held = [{0: 30_000}] * 20+ assert settle(held, before_kill_free_mib = {0: 30_000}) == (True, 2.0, 8)+ # A gain under the tolerance is noise, not a release.+ assert settle([{0: 30_200}] * 20, before_kill_free_mib = {0: 30_000}) == (True, 2.0, 8)+++def test_pre_kill_sample_waits_for_a_late_release_to_settle(settle):+ # A small gain before the first poll may be partial: the bulk landing later is still waited for,+ # exactly as without the pre-kill sample.+ sequence = [{0: 31_000}, {0: 31_000}, {0: 39_000}, {0: 40_000}, {0: 40_000}]+ assert settle(sequence, before_kill_free_mib = {0: 30_000}) == (True, 1.0, 5)+ assert settle(sequence) == (True, 1.0, 5)+ # Still moving after the first release: keeps polling until two samples agree.+ sequence = [{0: 35_000}, {0: 39_000}, {0: 45_000}, {0: 45_100}]+ assert settle(sequence, before_kill_free_mib = {0: 30_000}) == (True, 0.75, 4)+++def test_pre_kill_sample_from_other_devices_is_ignored(settle):+ freed = [{0: 40_000}] * 20+ assert settle(freed, before_kill_free_mib = {1: 30_000}) == (True, 2.0, 8)+ assert settle(freed, before_kill_free_mib = {}) == (True, 2.0, 8)+++def test_expected_free_target_is_unchanged_by_pre_kill_sample(settle):+ short = [{0: 10_000}] * 20+ expected = {0: 20.0}+ without = settle(short, expected_free_gb = expected)+ assert without == (False, 2.0, 8)+ assert settle(short, expected_free_gb = expected, before_kill_free_mib = {0: 1_000}) == without+ assert settle([{0: 30_000}], expected_free_gb = expected) == (True, 0.0, 1)+++def test_unreadable_telemetry_keeps_its_answer(settle):+ assert settle([]) == (True, 0.0, 1)+ assert settle([], expected_free_gb = {0: 1.0}) == (False, 0.0, 1)+ assert settle([], before_kill_free_mib = {0: 1}) == (True, 0.0, 1)+++def test_load_model_samples_before_kill_only_without_a_target():+ import inspect+ from core.inference.orchestrator import InferenceOrchestrator++ source = inspect.getsource(InferenceOrchestrator.load_model)+ cancel = source.index("self._cancel_generation()")+ sample = source.index("before_kill_free_mib = self._free_vram_mib()")+ shutdown = source.index("if self._shutdown_subprocess() is False")+ # Cancel first so an in-flight generation never waits on telemetry; sample while the worker still holds memory.+ assert cancel < sample < shutdown+ guard = source[cancel:sample]+ assert "post_handoff_expected_free_gb is None" in guard+ assert 'gpu_selection.get("required_gb") is not None' in guard