Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: recognise a worker VRAM release that finished before the first settle poll (#16404)

Daniel Handfad9c534b24
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 5c79b13a1f2..3e715786b6f 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -1061,6 +1061,26 @@ class InferenceOrchestrator:
seen = self._replies_read
idle_deadline = now + _SHUTDOWN_READER_IDLE_S
+ @staticmethod
+ def _free_vram_mib() -> Optional[dict[int, int]]:
+ """Free MiB per visible device from fresh live telemetry, or None when unreadable."""
+ from utils.hardware import get_visible_gpu_utilization
+ try:
+ from utils.hardware import gpu_query
+
+ result: dict[int, int] = {}
+ # Consecutive samples are compared: a cached one would read as settled.
+ with gpu_query.fresh_reads():
+ devices = get_visible_gpu_utilization().get("devices", [])
+ for device in devices:
+ index = int(device["index"])
+ total = float(device["vram_total_gb"])
+ used = float(device["vram_used_gb"])
+ result[index] = max(int((total - used) * 1024), 0)
+ return result or None
+ except (KeyError, TypeError, ValueError, OverflowError):
+ return None
+
@staticmethod
def _wait_for_worker_vram_settle(
since_kill: float,
@@ -1069,10 +1089,9 @@ class InferenceOrchestrator:
max_wait: float = 2.0,
interval: float = 0.25,
tolerance_mib: int = 256,
+ before_kill_free_mib: Optional[dict[int, int]] = None,
) -> bool:
"""Poll cross-platform live telemetry until dead-worker VRAM stabilises."""
- from utils.hardware import get_visible_gpu_utilization
-
if since_kill <= 0:
return not expected_free_gb
deadline = time.monotonic() + max(max_wait, 0.0)
@@ -1080,21 +1099,14 @@ class InferenceOrchestrator:
def _probe() -> Optional[dict[int, int]]:
if time.monotonic() >= deadline:
return None
- try:
- from utils.hardware import gpu_query
+ return InferenceOrchestrator._free_vram_mib()
- result: dict[int, int] = {}
- # Consecutive samples are compared: a cached one would read as settled.
- with gpu_query.fresh_reads():
- devices = get_visible_gpu_utilization().get("devices", [])
- for device in devices:
- index = int(device["index"])
- total = float(device["vram_total_gb"])
- used = float(device["vram_used_gb"])
- result[index] = max(int((total - used) * 1024), 0)
- return result or None
- except (KeyError, TypeError, ValueError, OverflowError):
- return None
+ def _released(after: dict[int, int], before: dict[int, int]) -> bool:
+ return any(
+ after[index] - before[index]
+ >= max(tolerance_mib, int(max(after[index], before[index]) * 0.02))
+ for index in after
+ )
previous = _probe()
if not previous:
@@ -1111,7 +1123,16 @@ class InferenceOrchestrator:
if _threshold_reached(previous):
return True
- observed_reclaim = False
+ # A release that landed before the first poll is still a release: without the pre-kill
+ # sample it read as stable-but-unreleased and the wait ran to its deadline.
+ observed_reclaim = bool(
+ before_kill_free_mib
+ and before_kill_free_mib.keys() == previous.keys()
+ and _released(previous, before_kill_free_mib)
+ )
+ # A release seen only against the pre-kill sample may be partial: ask for two stable polls.
+ stable_needed = 2 if observed_reclaim else 1
+ stable_run = 0
while time.monotonic() < deadline:
remaining = deadline - time.monotonic()
if remaining <= 0:
@@ -1120,12 +1141,9 @@ class InferenceOrchestrator:
current = _probe()
if not current or current.keys() != previous.keys():
return not expected_free_gb
- if any(
- current[index] - previous[index]
- >= max(tolerance_mib, int(max(current[index], previous[index]) * 0.02))
- for index in current
- ):
+ if _released(current, previous):
observed_reclaim = True
+ stable_needed = 1
if _threshold_reached(current):
return True
stable = all(
@@ -1133,8 +1151,9 @@ class InferenceOrchestrator:
< max(tolerance_mib, int(max(current[index], previous[index]) * 0.02))
for index in current
)
+ stable_run = stable_run + 1 if stable else 0
# Stable low samples can precede delayed driver reclaim; trust stability only after a release.
- if stable and observed_reclaim and not expected_mib:
+ if stable and observed_reclaim and not expected_mib and stable_run >= stable_needed:
return True
previous = current
return not expected_free_gb
@@ -2359,8 +2378,20 @@ class InferenceOrchestrator:
# Always spawn fresh: unsloth patches torch internals, breaking getsource on reuse.
had_worker_handle = self._proc is not None
worker_shutdown_at = 0.0
+ before_kill_free_mib = None
if self._ensure_subprocess_alive():
self._cancel_generation()
+ if post_handoff_expected_free_gb is None and not (
+ len(resolved_gpu_ids or ()) == 1
+ and isinstance(gpu_selection, dict)
+ and gpu_selection.get("required_gb") is not None
+ ):
+ # No free-VRAM target for the settle wait below: sample the live worker's
+ # footprint so a release already finished by the first poll is recognised.
+ try:
+ before_kill_free_mib = self._free_vram_mib()
+ except Exception: # noqa: BLE001 - an unreadable sample keeps the plain settle wait
+ before_kill_free_mib = None
if self._shutdown_subprocess() is False:
# Worker survived kill (wedged CUDA syscall); do not spawn a second over its GPU memory.
raise RuntimeError(
@@ -2386,6 +2417,7 @@ class InferenceOrchestrator:
settled = self._wait_for_worker_vram_settle(
worker_shutdown_at or time.monotonic(),
expected_free_gb = expected_free_gb,
+ before_kill_free_mib = before_kill_free_mib,
)
if expected_free_gb is not None and not settled:
raise RuntimeError(self._vram_wait_message(expected_free_gb))
diff --git a/studio/backend/tests/test_inference_worker_vram_settle.py b/studio/backend/tests/test_inference_worker_vram_settle.py
new file mode 100644
index 00000000000..ec97cc69f70
--- /dev/null
+++ b/studio/backend/tests/test_inference_worker_vram_settle.py
@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+
+def settle(monkeypatch):
+ spec = importlib.util.spec_from_file_location(
+ "orchestrator_vram_settle_under_test",
+ Path(_BACKEND_DIR) / "core/inference/orchestrator.py",
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ clock = [100.0]
+ samples: list = []
+ probes = [0]
+ module.time = types.SimpleNamespace(
+ monotonic = lambda: clock[0],
+ sleep = lambda s: clock.__setitem__(0, clock[0] + s),
+ time = lambda: clock[0],
+ )
+
+ def free_vram_mib():
+ probes[0] += 1
+ return dict(samples.pop(0)) if samples else None
+
+ monkeypatch.setattr(module.InferenceOrchestrator, "_free_vram_mib", staticmethod(free_vram_mib))
+
+ def run(sequence, **kwargs):
+ samples[:] = list(sequence)
+ clock[0] = 100.0
+ probes[0] = 0
+ result = module.InferenceOrchestrator._wait_for_worker_vram_settle(1.0, **kwargs)
+ return result, round(clock[0] - 100.0, 6), probes[0]
+
+ return run
+
+
+def test_release_before_first_poll_is_recognised_with_pre_kill_sample(settle):
+ freed = [{0: 40_000}] * 20
+ # Without the pre-kill sample the already-freed card looks stable but unreleased: full 2 s.
+ assert settle(freed) == (True, 2.0, 8)
+ # With it, the gain since the kill counts as the release; two stable samples confirm it.
+ assert settle(freed, before_kill_free_mib = {0: 30_000}) == (True, 0.5, 3)
+
+
+def test_pre_kill_sample_without_a_release_still_waits(settle):
+ held = [{0: 30_000}] * 20
+ assert settle(held, before_kill_free_mib = {0: 30_000}) == (True, 2.0, 8)
+ # A gain under the tolerance is noise, not a release.
+ assert settle([{0: 30_200}] * 20, before_kill_free_mib = {0: 30_000}) == (True, 2.0, 8)
+
+
+def test_pre_kill_sample_waits_for_a_late_release_to_settle(settle):
+ # A small gain before the first poll may be partial: the bulk landing later is still waited for,
+ # exactly as without the pre-kill sample.
+ sequence = [{0: 31_000}, {0: 31_000}, {0: 39_000}, {0: 40_000}, {0: 40_000}]
+ assert settle(sequence, before_kill_free_mib = {0: 30_000}) == (True, 1.0, 5)
+ assert settle(sequence) == (True, 1.0, 5)
+ # Still moving after the first release: keeps polling until two samples agree.
+ sequence = [{0: 35_000}, {0: 39_000}, {0: 45_000}, {0: 45_100}]
+ assert settle(sequence, before_kill_free_mib = {0: 30_000}) == (True, 0.75, 4)
+
+
+def test_pre_kill_sample_from_other_devices_is_ignored(settle):
+ freed = [{0: 40_000}] * 20
+ assert settle(freed, before_kill_free_mib = {1: 30_000}) == (True, 2.0, 8)
+ assert settle(freed, before_kill_free_mib = {}) == (True, 2.0, 8)
+
+
+def test_expected_free_target_is_unchanged_by_pre_kill_sample(settle):
+ short = [{0: 10_000}] * 20
+ expected = {0: 20.0}
+ without = settle(short, expected_free_gb = expected)
+ assert without == (False, 2.0, 8)
+ assert settle(short, expected_free_gb = expected, before_kill_free_mib = {0: 1_000}) == without
+ assert settle([{0: 30_000}], expected_free_gb = expected) == (True, 0.0, 1)
+
+
+def test_unreadable_telemetry_keeps_its_answer(settle):
+ assert settle([]) == (True, 0.0, 1)
+ assert settle([], expected_free_gb = {0: 1.0}) == (False, 0.0, 1)
+ assert settle([], before_kill_free_mib = {0: 1}) == (True, 0.0, 1)
+
+
+def test_load_model_samples_before_kill_only_without_a_target():
+ import inspect
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ source = inspect.getsource(InferenceOrchestrator.load_model)
+ cancel = source.index("self._cancel_generation()")
+ sample = source.index("before_kill_free_mib = self._free_vram_mib()")
+ shutdown = source.index("if self._shutdown_subprocess() is False")
+ # Cancel first so an in-flight generation never waits on telemetry; sample while the worker still holds memory.
+ assert cancel < sample < shutdown
+ guard = source[cancel:sample]
+ assert "post_handoff_expected_free_gb is None" in guard
+ assert 'gpu_selection.get("required_gb") is not None' in guard