Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: unload an idle decision model when another model loads (#14555)

Nilay2466ce274490
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index d938cfcfeb6..69208750416 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -278,6 +278,16 @@ def _release_idle_video_servers(reason: str) -> None:
logger.warning("gpu_arbiter: could not release the idle video sd-server: %s", exc)
+def _release_idle_decision_worker() -> None:
+ module = sys.modules.get("core.systemone.laya_runtime")
+ if module is None:
+ return
+ try:
+ module.evict_idle_worker()
+ except Exception as exc: # noqa: BLE001
+ logger.warning("gpu_arbiter: could not release the idle decision worker: %s", exc)
+
+
def _evict_video() -> None:
from core.inference.video import get_video_backend
get_video_backend().unload()
@@ -729,6 +739,8 @@ def acquire_for(
if owner != VIDEO and not (coreside and VIDEO in _residents):
# A resident H3 sd-server sits outside every owner's teardown.
_release_idle_video_servers(f"GPU acquired for {owner}")
+ if owner != DECISIONS and (register is not None or replacing) and policy == "off":
+ _release_idle_decision_worker()
if owner != DIFFUSION or register is not None or replacing:
# A load (or another kind taking the GPU) gets the segmenter's VRAM; an image model's re-assert keeps it.
_release_mask_segmenter(f"GPU acquired for {owner}", handover = True)
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.py
index 3b34b575152..abdd5f544d7 100644
--- a/studio/backend/core/systemone/laya_runtime.py
+++ b/studio/backend/core/systemone/laya_runtime.py
@@ -569,6 +569,38 @@ def _gpu_taken() -> bool:
return any(owner != gpu_arbiter.DECISIONS for owner in gpu_arbiter.residents())
+def _torch_gpu(checkpoint: Checkpoint) -> bool:
+ if _is_mlx(checkpoint) or _is_native(checkpoint):
+ return False
+ if checkpoint.layout == "clef":
+ return True
+ from utils.systemone_settings import get_device as preferred_device
+ from utils.hardware.hardware import DeviceType, get_device
+
+ return preferred_device() == "gpu" and get_device() in (DeviceType.CUDA, DeviceType.XPU)
+
+
+def _gpu_load_in_flight() -> bool:
+ from core.inference import gpu_arbiter
+ from core.inference.gpu_residency import chat_load_in_flight
+
+ if chat_load_in_flight():
+ return True
+ # Held by the acquire that may have just ended the idle worker.
+ if not gpu_arbiter._lock.acquire(blocking = False):
+ return True
+ try:
+ # A CPU-only image or video load releases its GPU claim, so it does not count.
+ return any(
+ owner in gpu_arbiter._residents and gpu_arbiter._media_loading(owner)
+ for owner in (gpu_arbiter.DIFFUSION, gpu_arbiter.VIDEO)
+ )
+ except Exception:
+ return False
+ finally:
+ gpu_arbiter._lock.release()
+
+
def _auto_keeps_pytorch(checkpoint: Checkpoint, native: Checkpoint) -> str | None:
"""Why Auto answers text on PyTorch although llama.cpp could: as before llama.cpp existed here."""
from .catalog import clef_unsupported_reason
@@ -1520,6 +1552,25 @@ def evict_for_gpu() -> None:
_close(agent, release_gpu = False)
+def evict_idle_worker() -> None:
+ global _agent, _loaded, _device_name, _generation, _unloads
+ from .clef_runtime import ClefAgent
+
+ if not _run_lock.acquire(blocking = False):
+ return
+ try:
+ with _state_lock:
+ if _loading is not None or not isinstance(_agent, (LayaWorker, ClefAgent)):
+ return
+ agent, _agent, _loaded, _device_name = _agent, None, None, None
+ _generation += 1
+ _unloads += 1
+ finally:
+ _run_lock.release()
+ logger.info("System One unloaded the idle decision model to free the GPU")
+ _close(agent)
+
+
def _holds_gpu() -> bool:
# A worker that fell back to CPU still holds its CUDA or XPU context until it ends.
return _device_name not in (None, "cpu") or isinstance(_agent, LayaWorker)
@@ -1586,6 +1637,16 @@ def _ensure_loading(checkpoint: Checkpoint, unloads: int | None = None) -> threa
raise Unavailable(503, "model_loading", f"{checkpoint.name} is loading", retry_after = 5)
# Claimed before the registry probe, so a Hub download admitted from now on sees this load.
_loading = checkpoint
+ # Checked after the claim, so an eviction that lands before it is still seen.
+ if _torch_gpu(checkpoint) and _gpu_load_in_flight():
+ with _state_lock:
+ _loading = None
+ raise Unavailable(
+ 409,
+ "gpu_busy",
+ f"Another model is loading on the GPU; {checkpoint.name} loads once it has.",
+ retry_after = 1,
+ )
# Outside _state_lock: the registry calls loading_repo_ids() while holding its own lock.
if _hub_download_active(checkpoint):
with _state_lock:
diff --git a/studio/backend/tests/test_decision_serving.py b/studio/backend/tests/test_decision_serving.py
index a2f9b685265..d2780b4eda6 100644
--- a/studio/backend/tests/test_decision_serving.py
+++ b/studio/backend/tests/test_decision_serving.py
@@ -223,6 +223,23 @@ def test_shutdown_ends_a_gpu_worker(home, client, gpu):
assert client.get("/api/settings/systemone").json()["loaded_device"] is None
+def test_a_chat_load_ends_an_idle_gpu_worker(home, client, gpu, monkeypatch):
+ from core.inference import gpu_arbiter
+
+ monkeypatch.setattr(gpu_arbiter, "_owner", None)
+ assert _put(client, enabled = True, model = _fine_tune(home, "laya_served_1")).status_code == 200
+ assert _post(client).status_code == 200
+ gpu_arbiter.acquire_for(gpu_arbiter.CHAT)
+ with laya_runtime._run_lock:
+ gpu_arbiter.acquire_for(gpu_arbiter.CHAT, lambda: None)
+ assert gpu.closed == 0
+ gpu_arbiter.acquire_for(gpu_arbiter.CHAT, lambda: None)
+ assert gpu.closed == 1
+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None
+ assert _post(client).status_code == 200
+ assert gpu.placed == ["cuda", "cuda"]
+
+
def test_training_takes_the_gpu_from_a_load_in_flight_or_a_model_it_missed(home, client, gpu):
def training_starts():
gpu.training, gpu.on_build = True, None
@@ -639,6 +656,127 @@ def test_a_chat_model_that_takes_the_gpu_stops_the_reload(home, client, clef, mo
assert len(clef.agents) == 2
+def test_an_image_load_ends_an_idle_clef_worker(home, client, clef):
+ from core.inference import gpu_arbiter
+
+ assert (
+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200
+ )
+ assert _post(client).status_code == 200
+ gpu_arbiter.acquire_for(gpu_arbiter.DIFFUSION, lambda: None)
+ assert clef.agents[0].closed
+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None
+
+
+def test_a_load_claimed_as_the_eviction_starts_keeps_the_idle_worker(
+ home, client, clef, monkeypatch
+):
+ from core.inference import gpu_arbiter
+
+ assert (
+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200
+ )
+ assert _post(client).status_code == 200
+ run_lock, claimed = laya_runtime._run_lock, laya_runtime._loaded
+
+ class ClaimedInTheGap:
+ def acquire(self, *args, **kwargs):
+ laya_runtime._loading = claimed
+ return run_lock.acquire(*args, **kwargs)
+
+ def release(self):
+ run_lock.release()
+
+ monkeypatch.setattr(laya_runtime, "_loading", None)
+ monkeypatch.setattr(laya_runtime, "_run_lock", ClaimedInTheGap())
+ unloads = laya_runtime._unloads
+ gpu_arbiter.acquire_for(gpu_arbiter.DIFFUSION, lambda: None)
+ assert not clef.agents[0].closed
+ assert laya_runtime._unloads == unloads
+
+
[email protected]("owner", ["chat", "diffusion"])
[email protected]("policy, ends", [("off", True), ("fit", False), ("always", False)])
+def test_only_off_ends_an_idle_worker_on_a_load(
+ home, client, clef, monkeypatch, owner, policy, ends
+):
+ from core.inference import gpu_arbiter
+
+ for name in ("_residents", "_media_needs", "_accounts"):
+ monkeypatch.setattr(gpu_arbiter, name, {})
+ assert (
+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200
+ )
+ assert _post(client).status_code == 200
+ gpu_arbiter.acquire_for(owner, lambda: None, policy = policy)
+ assert clef.agents[0].closed is ends
+ assert (client.get("/api/settings/systemone").json()["loaded_model"] is None) is ends
+ if not ends:
+ assert _post(client).status_code == 200 and len(clef.agents) == 1
+
+
[email protected]("loading", ["chat", "diffusion", "video"])
+def test_a_decision_waits_for_a_gpu_load_in_flight_after_the_eviction(
+ home, client, clef, monkeypatch, loading
+):
+ from core.inference import gpu_arbiter, gpu_residency
+
+ assert (
+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200
+ )
+ assert _post(client).status_code == 200
+ gpu_arbiter.acquire_for(gpu_arbiter.DIFFUSION, lambda: None)
+ assert clef.agents[0].closed
+ monkeypatch.setattr(gpu_residency, "chat_load_in_flight", lambda: loading == "chat")
+ monkeypatch.setattr(gpu_arbiter, "_media_loading", lambda owner: owner == loading)
+ if loading != "chat":
+ monkeypatch.setitem(gpu_arbiter._residents, loading, None)
+ response = _post(client)
+ assert response.status_code == 409 and response.json()["detail"]["error_type"] == "gpu_busy"
+ assert len(clef.agents) == 1
+ monkeypatch.setattr(gpu_residency, "chat_load_in_flight", lambda: False)
+ monkeypatch.setattr(gpu_arbiter, "_media_loading", lambda owner: False)
+ assert _post(client).status_code == 200 and len(clef.agents) == 2
+
+
+def test_a_cpu_media_load_or_a_settled_gpu_leaves_a_decision_load_alone(
+ home, client, clef, monkeypatch
+):
+ from core.inference import gpu_arbiter
+
+ monkeypatch.setattr(gpu_arbiter, "_media_loading", lambda owner: True)
+ for owner in (gpu_arbiter.DIFFUSION, gpu_arbiter.VIDEO):
+ monkeypatch.delitem(gpu_arbiter._residents, owner, raising = False)
+ assert (
+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200
+ )
+ assert _post(client).status_code == 200 and len(clef.agents) == 1
+
+
+def test_a_decision_load_waits_while_the_arbiter_is_mid_acquire(home, client, clef, monkeypatch):
+ from core.inference import gpu_arbiter
+
+ assert (
+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200
+ )
+ with gpu_arbiter._lock:
+ response = _post(client)
+ assert response.status_code == 409 and response.json()["detail"]["error_type"] == "gpu_busy"
+ assert not clef.agents and laya_runtime._loading is None
+ assert _post(client).status_code == 200
+
+
+def test_a_resident_worker_answers_during_a_gpu_load(home, client, clef, monkeypatch):
+ from core.inference import gpu_residency
+
+ assert (
+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200
+ )
+ assert _post(client).status_code == 200
+ monkeypatch.setattr(gpu_residency, "chat_load_in_flight", lambda: True)
+ assert _post(client).status_code == 200 and len(clef.agents) == 1
+
+
def test_a_clef_worker_that_fails_the_warm_up_is_retired(home, client, clef, monkeypatch):
from core.systemone import clef_runtime