Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let the owner turn off the Decision API while a model loads (#14557)

Nilayc5a9fe021a8f
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.py
index abdd5f544d7..c47938c5690 100644
--- a/studio/backend/core/systemone/laya_runtime.py
+++ b/studio/backend/core/systemone/laya_runtime.py
@@ -945,9 +945,17 @@ def _close(agent, release_gpu: bool = True) -> None:
gpu_arbiter.release_if(gpu_arbiter.DECISIONS, unclaimed)
+def _unloaded() -> bool:
+ unloads = getattr(_load_generation, "unloads", None)
+ return unloads is not None and unloads != _unloads
+
+
def _evict(external: bool = False) -> None:
global _agent, _loaded, _device_name, _generation, _unloads
with _run_lock:
+ # Every load evicts once its files are on disk: one unloaded meanwhile stops here.
+ if _unloaded():
+ raise RuntimeError("It was unloaded while it downloaded.")
agent, _agent, _loaded, _device_name = _agent, None, None, None
_generation += 1
_unloads += external
@@ -1114,11 +1122,7 @@ def _load_checkpoint(checkpoint: Checkpoint):
from .clef_runtime import ClefAgent, ClefWorkerError
_evict()
- unloads = getattr(_load_generation, "unloads", _unloads)
- agent = ClefAgent(
- root,
- cancelled = lambda: _clef_preempted() or _unloads != unloads,
- )
+ agent = ClefAgent(root, cancelled = lambda: _clef_preempted() or _unloaded())
# Training may have started while this loaded, and Clef has no CPU fallback.
if _clef_preempted():
agent.close()
@@ -1135,10 +1139,12 @@ def _load_checkpoint(checkpoint: Checkpoint):
device = "cpu" if for_training else _device()
if device in ("cuda", "xpu"):
try:
- agent = LayaWorker(root, checkpoint, device, cancelled = _training_active)
+ agent = LayaWorker(
+ root, checkpoint, device, cancelled = lambda: _training_active() or _unloaded()
+ )
return agent, agent.device
except LayaWorkerError:
- if not _training_active():
+ if _unloaded() or not _training_active():
raise
device, for_training = "cpu", True
agent, placed = _build_laya(root, checkpoint, device, for_training)
@@ -1199,7 +1205,7 @@ def _load_native(checkpoint: Checkpoint):
gpu = gpu,
clef_answers = checkpoint.layout == "clef",
ctx = get_native_ctx(),
- cancelled = (lambda: _training_active()) if gpu else None,
+ cancelled = lambda: (gpu and _training_active()) or _unloaded(),
)
if gpu and _training_active():
agent.close()
@@ -1472,18 +1478,28 @@ def _load(checkpoint: Checkpoint, unloads: int | None = None) -> None:
else:
logger.warning("System One load failed: %s", message, exc_info = True)
return
+ finally:
+ del _load_generation.unloads
with _state_lock:
stale = _unloads != unloads
if not stale:
_agent, _loaded, _device_name = agent, checkpoint, str(getattr(agent, "device", device))
- _failure = _loading = None
+ _loading = None
+ _failure = None
_gpu_load = False
_generation += 1
_last_used = time.monotonic()
generation = _generation
if stale:
logger.info("System One dropped %s: it was unloaded while it loaded", checkpoint.name)
- _close(agent)
+ # _loading stays set until its memory is back, so no other load claims the GPU first.
+ try:
+ _close(agent)
+ del agent
+ _release_memory()
+ finally:
+ with _state_lock:
+ _loading = None
return
if _arbitrated(checkpoint):
_schedule_idle_unload(generation)
@@ -1637,6 +1653,8 @@ def _ensure_loading(checkpoint: Checkpoint, unloads: int | None = None) -> threa
raise Unavailable(503, "model_loading", f"{checkpoint.name} is loading", retry_after = 5)
# Claimed before the registry probe, so a Hub download admitted from now on sees this load.
_loading = checkpoint
+ if unloads is None:
+ unloads = _unloads
# Checked after the claim, so an eviction that lands before it is still seen.
if _torch_gpu(checkpoint) and _gpu_load_in_flight():
with _state_lock:
@@ -2455,19 +2473,10 @@ def status() -> dict[str, Any]:
}
-def ensure_can_unload() -> None:
- with _state_lock:
- if _reloading is not None and _reloading == _loading:
- return
- # _loading: a load claimed but not yet started would otherwise land after this unload.
- if _loading is not None or (_loader is not None and _loader.is_alive()):
- raise Unavailable(409, "model_loading", "Wait for the load to finish before unloading")
-
-
def unload(reload_after_training: bool = False) -> bool:
global _agent, _loaded, _device_name, _failure, _generation, _unloads
- ensure_can_unload()
- with _run_lock:
+ # _state_lock too: a load publishes under it, so it can never land just after this unload.
+ with _run_lock, _state_lock:
agent, was_loaded, checkpoint = _agent, _agent is not None, _loaded
_agent = _loaded = _device_name = None
_failure = None
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 9da8ebb23ce..370586a1660 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -1875,15 +1875,9 @@ async def validate_systemone_settings(
def _validate_systemone_settings(payload: SystemOneSettingsPayload) -> None:
- from core.systemone import laya_runtime
with _SYSTEMONE_SETTINGS_LOCK:
_check_systemone_expectations(payload)
- values = _systemone_values(payload)
- if values:
- try:
- laya_runtime.ensure_can_unload()
- except laya_runtime.Unavailable as exc:
- raise HTTPException(status_code = 409, detail = exc.message) from None
+ _systemone_values(payload)
@_owner_settings_router.get(
diff --git a/studio/backend/tests/test_decision_serving.py b/studio/backend/tests/test_decision_serving.py
index d2780b4eda6..72636c8777c 100644
--- a/studio/backend/tests/test_decision_serving.py
+++ b/studio/backend/tests/test_decision_serving.py
@@ -865,7 +865,7 @@ def test_a_chat_model_that_takes_the_gpu_during_the_warm_up_retires_the_reload(
assert client.get("/api/settings/systemone").json()["loaded_model"] is None
-def test_a_settings_change_during_the_reloads_lookup_stops_its_worker(
+def test_a_settings_change_during_the_reloads_lookup_never_starts_its_worker(
home, client, clef, monkeypatch
):
from core.systemone import clef_runtime
@@ -900,7 +900,7 @@ def test_a_settings_change_during_the_reloads_lookup_stops_its_worker(
laya_runtime.unload(reload_after_training = True)
clef.training = False
_reload_finished()
- assert stopped == [True]
+ assert stopped == [] and len(clef.agents) == 1
settings = client.get("/api/settings/systemone").json()
assert settings["loaded_model"] is settings["loading_model"] is settings["error"] is None
diff --git a/studio/backend/tests/test_systemone.py b/studio/backend/tests/test_systemone.py
index 632e156650c..e9247427c45 100644
--- a/studio/backend/tests/test_systemone.py
+++ b/studio/backend/tests/test_systemone.py
@@ -490,12 +490,13 @@ def test_stale_check_uses_the_display_name_for_a_local_env_model(client, monkeyp
assert response.status_code == 204
-def test_settings_change_waits_for_a_running_load(client, monkeypatch):
+def test_settings_change_cancels_a_running_load(client, monkeypatch, runtime):
release = threading.Event()
def slow(checkpoint):
+ runtime.append(checkpoint.name)
release.wait(5)
- return FakeAgent(), "cpu"
+ return FakeAgent(checkpoint.name), "cpu"
monkeypatch.setattr(laya_runtime, "_load_checkpoint", slow)
monkeypatch.setattr(laya_runtime, "LOAD_WAIT_S", 0.05)
@@ -509,15 +510,18 @@ def test_settings_change_waits_for_a_running_load(client, monkeypatch):
"expected_model": "laya-multilingual",
},
).status_code
- == 409
+ == 204
)
- assert client.put("/api/settings/systemone", json = {"model": "laya-english"}).status_code == 409
- assert client.get("/api/settings/systemone").json()["model"] == "laya-multilingual"
+ assert client.put("/api/settings/systemone", json = {"model": "laya-english"}).status_code == 200
release.set()
laya_runtime._loader.join(5)
+ assert laya_runtime.status()["loaded_model"] is None
+ monkeypatch.setattr(laya_runtime, "LOAD_WAIT_S", 5.0)
+ assert _post(client).json()["model"] == "laya-english"
+ assert runtime == ["laya-multilingual", "laya-english"]
-def test_turning_it_off_waits_for_a_claimed_load(client, monkeypatch):
+def test_turning_it_off_cancels_a_claimed_load(client, monkeypatch, runtime):
probing, release = threading.Event(), threading.Event()
def probe(checkpoint):
@@ -531,11 +535,192 @@ def test_turning_it_off_waits_for_a_claimed_load(client, monkeypatch):
)
claim.start()
assert probing.wait(5)
- assert client.put("/api/settings/systemone", json = {"enabled": False}).status_code == 409
+ response = client.put("/api/settings/systemone", json = {"enabled": False})
+ assert response.status_code == 200 and response.json()["enabled"] is False
release.set()
claim.join(5)
laya_runtime._loader.join(5)
- assert client.get("/api/settings/systemone").json()["enabled"] is True
+ assert laya_runtime.status()["loaded_model"] is None
+ assert runtime == []
+
+
+def test_a_load_turned_off_while_it_downloads_never_builds(client, monkeypatch, tmp_path):
+ downloading, release, built = threading.Event(), threading.Event(), []
+
+ def download(checkpoint, **kwargs):
+ downloading.set()
+ release.wait(5)
+ return tmp_path
+
+ def build(*args, **kwargs):
+ built.append(args)
+ return FakeAgent(), "cpu"
+
+ monkeypatch.setattr(laya_runtime, "_load_checkpoint", _REAL_LOAD)
+ monkeypatch.setattr(laya_runtime, "_checkpoint_dir", download)
+ monkeypatch.setattr(laya_runtime, "_laya", lambda: None)
+ monkeypatch.setattr(laya_runtime, "_device", lambda: "cpu")
+ monkeypatch.setattr(laya_runtime, "_build_laya", build)
+ monkeypatch.setattr(laya_runtime, "LOAD_WAIT_S", 0.05)
+ assert _post(client).status_code == 503
+ assert downloading.wait(5)
+ assert client.put("/api/settings/systemone", json = {"enabled": False}).status_code == 200
+ release.set()
+ laya_runtime._loader.join(5)
+ assert built == []
+ assert laya_runtime.status()["loaded_model"] is None
+
+
+def test_a_load_waiting_behind_a_decision_sees_the_unload_that_ran_first(
+ client, monkeypatch, tmp_path
+):
+ downloaded, built = threading.Event(), []
+
+ def download(checkpoint, **kwargs):
+ downloaded.set()
+ return tmp_path
+
+ def build(*args, **kwargs):
+ built.append(args)
+ return FakeAgent(), "cpu"
+
+ monkeypatch.setattr(laya_runtime, "_load_checkpoint", _REAL_LOAD)
+ monkeypatch.setattr(laya_runtime, "_checkpoint_dir", download)
+ monkeypatch.setattr(laya_runtime, "_laya", lambda: None)
+ monkeypatch.setattr(laya_runtime, "_device", lambda: "cpu")
+ monkeypatch.setattr(laya_runtime, "_build_laya", build)
+ monkeypatch.setattr(laya_runtime, "LOAD_WAIT_S", 0.05)
+ with laya_runtime._run_lock:
+ assert _post(client).status_code == 503
+ assert downloaded.wait(5)
+ time.sleep(0.5)
+ laya_runtime._unloads += 1
+ laya_runtime._loader.join(5)
+ assert built == []
+ assert laya_runtime.status()["loaded_model"] is None
+
+
+def test_a_load_turned_off_while_it_builds_stops_its_worker(client, monkeypatch, tmp_path):
+ building, stopped = threading.Event(), []
+
+ class Worker:
+ def __init__(self, root, checkpoint, device, cancelled):
+ building.set()
+ deadline = time.monotonic() + 5
+ while not cancelled() and time.monotonic() < deadline:
+ time.sleep(0.01)
+ stopped.append(cancelled())
+ if stopped[-1]:
+ raise laya_runtime.LayaWorkerError("stopped")
+ self.device = device
+
+ def close(self):
+ pass
+
+ monkeypatch.setattr(laya_runtime, "_load_checkpoint", _REAL_LOAD)
+ monkeypatch.setattr(laya_runtime, "_checkpoint_dir", lambda checkpoint, **kwargs: tmp_path)
+ monkeypatch.setattr(laya_runtime, "_laya", lambda: None)
+ monkeypatch.setattr(laya_runtime, "_device", lambda: "cuda")
+ monkeypatch.setattr(laya_runtime, "LayaWorker", Worker)
+ monkeypatch.setattr(laya_runtime, "LOAD_WAIT_S", 0.05)
+ assert _post(client).status_code == 503
+ assert building.wait(5)
+ assert client.put("/api/settings/systemone", json = {"enabled": False}).status_code == 200
+ laya_runtime._loader.join(5)
+ assert stopped == [True]
+ assert laya_runtime.status()["loaded_model"] is None
+
+
+def test_a_load_turned_off_during_training_never_falls_back_to_cpu(client, monkeypatch, tmp_path):
+ building, go, training, built = threading.Event(), threading.Event(), [False], []
+
+ class Worker:
+ def __init__(self, root, checkpoint, device, cancelled):
+ building.set()
+ go.wait(5)
+ if cancelled():
+ raise laya_runtime.LayaWorkerError("stopped")
+ self.device = device
+
+ def build(*args, **kwargs):
+ built.append(args)
+ return FakeAgent(), "cpu"
+
+ monkeypatch.setattr(laya_runtime, "_load_checkpoint", _REAL_LOAD)
+ monkeypatch.setattr(laya_runtime, "_checkpoint_dir", lambda checkpoint, **kwargs: tmp_path)
+ monkeypatch.setattr(laya_runtime, "_laya", lambda: None)
+ monkeypatch.setattr(laya_runtime, "_device", lambda: "cuda")
+ monkeypatch.setattr(laya_runtime, "_training_active", lambda: training[0])
+ monkeypatch.setattr(laya_runtime, "LayaWorker", Worker)
+ monkeypatch.setattr(laya_runtime, "_build_laya", build)
+ monkeypatch.setattr(laya_runtime, "LOAD_WAIT_S", 0.05)
+ assert _post(client).status_code == 503
+ assert building.wait(5)
+ assert client.put("/api/settings/systemone", json = {"enabled": False}).status_code == 200
+ training[0] = True
+ go.set()
+ laya_runtime._loader.join(5)
+ assert built == []
+ assert laya_runtime.status()["loaded_model"] is None
+
+
+def test_a_load_unloaded_while_it_builds_in_process_frees_its_memory(client, monkeypatch, tmp_path):
+ agents, freed = [], []
+
+ def build(*args, **kwargs):
+ agent = FakeAgent()
+ agents.append(weakref.ref(agent))
+ laya_runtime.unload()
+ return agent, "mps"
+
+ monkeypatch.setattr(laya_runtime, "_load_checkpoint", _REAL_LOAD)
+ monkeypatch.setattr(laya_runtime, "_checkpoint_dir", lambda checkpoint, **kwargs: tmp_path)
+ monkeypatch.setattr(laya_runtime, "_laya", lambda: None)
+ monkeypatch.setattr(laya_runtime, "_device", lambda: "mps")
+ monkeypatch.setattr(laya_runtime, "_build_laya", build)
+ monkeypatch.setattr(
+ laya_runtime, "_release_memory", lambda: freed.append(bool(agents) and agents[0]() is None)
+ )
+ monkeypatch.setattr(laya_runtime, "LOAD_WAIT_S", 0.05)
+ _post(client)
+ laya_runtime._loader.join(5)
+ assert laya_runtime.status()["loaded_model"] is None
+ assert freed[-1] is True
+
+
+def test_a_load_unloaded_as_it_publishes_stays_fenced_until_it_has_closed(
+ client, monkeypatch, tmp_path
+):
+ fenced = []
+
+ class Closing(FakeAgent):
+ def close(self):
+ fenced.append(laya_runtime._loading is not None)
+
+ def build(*args, **kwargs):
+ laya_runtime.unload()
+ return Closing(), "mps"
+
+ monkeypatch.setattr(laya_runtime, "_load_checkpoint", _REAL_LOAD)
+ monkeypatch.setattr(laya_runtime, "_checkpoint_dir", lambda checkpoint, **kwargs: tmp_path)
+ monkeypatch.setattr(laya_runtime, "_laya", lambda: None)
+ monkeypatch.setattr(laya_runtime, "_device", lambda: "mps")
+ monkeypatch.setattr(laya_runtime, "_build_laya", build)
+ monkeypatch.setattr(laya_runtime, "LOAD_WAIT_S", 0.05)
+ _post(client)
+ laya_runtime._loader.join(5)
+ assert fenced == [True]
+ assert laya_runtime._loading is None
+ assert laya_runtime.status()["loaded_model"] is None
+
+
+def test_an_unload_waits_for_a_load_that_is_publishing(client):
+ done = threading.Event()
+ unloading = threading.Thread(target = lambda: (laya_runtime.unload(), done.set()), daemon = True)
+ with laya_runtime._state_lock:
+ unloading.start()
+ assert not done.wait(0.2)
+ assert done.wait(5)
def test_keyless_callers_only_reach_the_configured_model(client, monkeypatch, runtime):