Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: unload an idle decision model when another model loads (#14555)
Nilay2466ce274490
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.pyindex d938cfcfeb6..69208750416 100644--- a/studio/backend/core/inference/gpu_arbiter.py+++ b/studio/backend/core/inference/gpu_arbiter.py@@ -278,6 +278,16 @@ def _release_idle_video_servers(reason: str) -> None:logger.warning("gpu_arbiter: could not release the idle video sd-server: %s", exc)+def _release_idle_decision_worker() -> None:+ module = sys.modules.get("core.systemone.laya_runtime")+ if module is None:+ return+ try:+ module.evict_idle_worker()+ except Exception as exc: # noqa: BLE001+ logger.warning("gpu_arbiter: could not release the idle decision worker: %s", exc)++def _evict_video() -> None:from core.inference.video import get_video_backendget_video_backend().unload()@@ -729,6 +739,8 @@ def acquire_for(if owner != VIDEO and not (coreside and VIDEO in _residents):# A resident H3 sd-server sits outside every owner's teardown._release_idle_video_servers(f"GPU acquired for {owner}")+ if owner != DECISIONS and (register is not None or replacing) and policy == "off":+ _release_idle_decision_worker()if owner != DIFFUSION or register is not None or replacing:# A load (or another kind taking the GPU) gets the segmenter's VRAM; an image model's re-assert keeps it._release_mask_segmenter(f"GPU acquired for {owner}", handover = True)diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.pyindex 3b34b575152..abdd5f544d7 100644--- a/studio/backend/core/systemone/laya_runtime.py+++ b/studio/backend/core/systemone/laya_runtime.py@@ -569,6 +569,38 @@ def _gpu_taken() -> bool:return any(owner != gpu_arbiter.DECISIONS for owner in gpu_arbiter.residents())+def _torch_gpu(checkpoint: Checkpoint) -> bool:+ if _is_mlx(checkpoint) or _is_native(checkpoint):+ return False+ if checkpoint.layout == "clef":+ return True+ from utils.systemone_settings import get_device as preferred_device+ from utils.hardware.hardware import DeviceType, get_device++ return preferred_device() == "gpu" and get_device() in (DeviceType.CUDA, DeviceType.XPU)+++def _gpu_load_in_flight() -> bool:+ from core.inference import gpu_arbiter+ from core.inference.gpu_residency import chat_load_in_flight++ if chat_load_in_flight():+ return True+ # Held by the acquire that may have just ended the idle worker.+ if not gpu_arbiter._lock.acquire(blocking = False):+ return True+ try:+ # A CPU-only image or video load releases its GPU claim, so it does not count.+ return any(+ owner in gpu_arbiter._residents and gpu_arbiter._media_loading(owner)+ for owner in (gpu_arbiter.DIFFUSION, gpu_arbiter.VIDEO)+ )+ except Exception:+ return False+ finally:+ gpu_arbiter._lock.release()++def _auto_keeps_pytorch(checkpoint: Checkpoint, native: Checkpoint) -> str | None:"""Why Auto answers text on PyTorch although llama.cpp could: as before llama.cpp existed here."""from .catalog import clef_unsupported_reason@@ -1520,6 +1552,25 @@ def evict_for_gpu() -> None:_close(agent, release_gpu = False)+def evict_idle_worker() -> None:+ global _agent, _loaded, _device_name, _generation, _unloads+ from .clef_runtime import ClefAgent++ if not _run_lock.acquire(blocking = False):+ return+ try:+ with _state_lock:+ if _loading is not None or not isinstance(_agent, (LayaWorker, ClefAgent)):+ return+ agent, _agent, _loaded, _device_name = _agent, None, None, None+ _generation += 1+ _unloads += 1+ finally:+ _run_lock.release()+ logger.info("System One unloaded the idle decision model to free the GPU")+ _close(agent)++def _holds_gpu() -> bool:# A worker that fell back to CPU still holds its CUDA or XPU context until it ends.return _device_name not in (None, "cpu") or isinstance(_agent, LayaWorker)@@ -1586,6 +1637,16 @@ def _ensure_loading(checkpoint: Checkpoint, unloads: int | None = None) -> threaraise Unavailable(503, "model_loading", f"{checkpoint.name} is loading", retry_after = 5)# Claimed before the registry probe, so a Hub download admitted from now on sees this load._loading = checkpoint+ # Checked after the claim, so an eviction that lands before it is still seen.+ if _torch_gpu(checkpoint) and _gpu_load_in_flight():+ with _state_lock:+ _loading = None+ raise Unavailable(+ 409,+ "gpu_busy",+ f"Another model is loading on the GPU; {checkpoint.name} loads once it has.",+ retry_after = 1,+ )# Outside _state_lock: the registry calls loading_repo_ids() while holding its own lock.if _hub_download_active(checkpoint):with _state_lock:diff --git a/studio/backend/tests/test_decision_serving.py b/studio/backend/tests/test_decision_serving.pyindex a2f9b685265..d2780b4eda6 100644--- a/studio/backend/tests/test_decision_serving.py+++ b/studio/backend/tests/test_decision_serving.py@@ -223,6 +223,23 @@ def test_shutdown_ends_a_gpu_worker(home, client, gpu):assert client.get("/api/settings/systemone").json()["loaded_device"] is None+def test_a_chat_load_ends_an_idle_gpu_worker(home, client, gpu, monkeypatch):+ from core.inference import gpu_arbiter++ monkeypatch.setattr(gpu_arbiter, "_owner", None)+ assert _put(client, enabled = True, model = _fine_tune(home, "laya_served_1")).status_code == 200+ assert _post(client).status_code == 200+ gpu_arbiter.acquire_for(gpu_arbiter.CHAT)+ with laya_runtime._run_lock:+ gpu_arbiter.acquire_for(gpu_arbiter.CHAT, lambda: None)+ assert gpu.closed == 0+ gpu_arbiter.acquire_for(gpu_arbiter.CHAT, lambda: None)+ assert gpu.closed == 1+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None+ assert _post(client).status_code == 200+ assert gpu.placed == ["cuda", "cuda"]++def test_training_takes_the_gpu_from_a_load_in_flight_or_a_model_it_missed(home, client, gpu):def training_starts():gpu.training, gpu.on_build = True, None@@ -639,6 +656,127 @@ def test_a_chat_model_that_takes_the_gpu_stops_the_reload(home, client, clef, moassert len(clef.agents) == 2+def test_an_image_load_ends_an_idle_clef_worker(home, client, clef):+ from core.inference import gpu_arbiter++ assert (+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200+ )+ assert _post(client).status_code == 200+ gpu_arbiter.acquire_for(gpu_arbiter.DIFFUSION, lambda: None)+ assert clef.agents[0].closed+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None+++def test_a_load_claimed_as_the_eviction_starts_keeps_the_idle_worker(+ home, client, clef, monkeypatch+):+ from core.inference import gpu_arbiter++ assert (+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200+ )+ assert _post(client).status_code == 200+ run_lock, claimed = laya_runtime._run_lock, laya_runtime._loaded++ class ClaimedInTheGap:+ def acquire(self, *args, **kwargs):+ laya_runtime._loading = claimed+ return run_lock.acquire(*args, **kwargs)++ def release(self):+ run_lock.release()++ monkeypatch.setattr(laya_runtime, "_loading", None)+ monkeypatch.setattr(laya_runtime, "_run_lock", ClaimedInTheGap())+ unloads = laya_runtime._unloads+ gpu_arbiter.acquire_for(gpu_arbiter.DIFFUSION, lambda: None)+ assert not clef.agents[0].closed+ assert laya_runtime._unloads == unloads++[email protected]("owner", ["chat", "diffusion"])[email protected]("policy, ends", [("off", True), ("fit", False), ("always", False)])+def test_only_off_ends_an_idle_worker_on_a_load(+ home, client, clef, monkeypatch, owner, policy, ends+):+ from core.inference import gpu_arbiter++ for name in ("_residents", "_media_needs", "_accounts"):+ monkeypatch.setattr(gpu_arbiter, name, {})+ assert (+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200+ )+ assert _post(client).status_code == 200+ gpu_arbiter.acquire_for(owner, lambda: None, policy = policy)+ assert clef.agents[0].closed is ends+ assert (client.get("/api/settings/systemone").json()["loaded_model"] is None) is ends+ if not ends:+ assert _post(client).status_code == 200 and len(clef.agents) == 1++[email protected]("loading", ["chat", "diffusion", "video"])+def test_a_decision_waits_for_a_gpu_load_in_flight_after_the_eviction(+ home, client, clef, monkeypatch, loading+):+ from core.inference import gpu_arbiter, gpu_residency++ assert (+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200+ )+ assert _post(client).status_code == 200+ gpu_arbiter.acquire_for(gpu_arbiter.DIFFUSION, lambda: None)+ assert clef.agents[0].closed+ monkeypatch.setattr(gpu_residency, "chat_load_in_flight", lambda: loading == "chat")+ monkeypatch.setattr(gpu_arbiter, "_media_loading", lambda owner: owner == loading)+ if loading != "chat":+ monkeypatch.setitem(gpu_arbiter._residents, loading, None)+ response = _post(client)+ assert response.status_code == 409 and response.json()["detail"]["error_type"] == "gpu_busy"+ assert len(clef.agents) == 1+ monkeypatch.setattr(gpu_residency, "chat_load_in_flight", lambda: False)+ monkeypatch.setattr(gpu_arbiter, "_media_loading", lambda owner: False)+ assert _post(client).status_code == 200 and len(clef.agents) == 2+++def test_a_cpu_media_load_or_a_settled_gpu_leaves_a_decision_load_alone(+ home, client, clef, monkeypatch+):+ from core.inference import gpu_arbiter++ monkeypatch.setattr(gpu_arbiter, "_media_loading", lambda owner: True)+ for owner in (gpu_arbiter.DIFFUSION, gpu_arbiter.VIDEO):+ monkeypatch.delitem(gpu_arbiter._residents, owner, raising = False)+ assert (+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200+ )+ assert _post(client).status_code == 200 and len(clef.agents) == 1+++def test_a_decision_load_waits_while_the_arbiter_is_mid_acquire(home, client, clef, monkeypatch):+ from core.inference import gpu_arbiter++ assert (+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200+ )+ with gpu_arbiter._lock:+ response = _post(client)+ assert response.status_code == 409 and response.json()["detail"]["error_type"] == "gpu_busy"+ assert not clef.agents and laya_runtime._loading is None+ assert _post(client).status_code == 200+++def test_a_resident_worker_answers_during_a_gpu_load(home, client, clef, monkeypatch):+ from core.inference import gpu_residency++ assert (+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200+ )+ assert _post(client).status_code == 200+ monkeypatch.setattr(gpu_residency, "chat_load_in_flight", lambda: True)+ assert _post(client).status_code == 200 and len(clef.agents) == 1++def test_a_clef_worker_that_fails_the_warm_up_is_retired(home, client, clef, monkeypatch):from core.systemone import clef_runtime