Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: reload a Clef as soon as the training run that unloaded it ends (#13482)

Nilay9b30c4f75cb6
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.py
index a8c56b5c0a9..a421f7b0343 100644
--- a/studio/backend/core/systemone/laya_runtime.py
+++ b/studio/backend/core/systemone/laya_runtime.py
@@ -48,6 +48,13 @@ _unloads = 0
_last_used = 0.0
_idle_timer: threading.Timer | None = None
_fallback_reason: str | None = None
+# A Clef a training run unloaded, reloading once the run ends; an unload cancels it instead of waiting.
+_reloading: Checkpoint | None = None
+# Why the reload's load was stopped ("training" or "gpu"), recorded when it happens: a short run can be over by the join.
+_reload_stopped_by: str | None = None
+# The unload generation the running load started for: an unload during its Hub lookup still stops the worker.
+_load_generation = threading.local()
+TRAINING_POLL_S = 2.0
# A llama.cpp decision server unloads after this long without a request.
IDLE_UNLOAD_S = 300.0
# Binaries (path, mtime) whose server answered without a "decisions" output.
@@ -1024,9 +1031,13 @@ def _load_checkpoint(checkpoint: Checkpoint):
from .clef_runtime import ClefAgent, ClefWorkerError
_evict()
- agent = ClefAgent(root, cancelled = _training_active)
+ unloads = getattr(_load_generation, "unloads", _unloads)
+ agent = ClefAgent(
+ root,
+ cancelled = lambda: _clef_preempted() or _unloads != unloads,
+ )
# Training may have started while this loaded, and Clef has no CPU fallback.
- if _training_active():
+ if _clef_preempted():
agent.close()
raise ClefWorkerError(
f"{checkpoint.name} needs the GPU, which a training run took while it loaded."
@@ -1352,19 +1363,31 @@ def loading_repo_ids() -> tuple[str, ...]:
return ()
-def _load(checkpoint: Checkpoint) -> None:
+def _load(checkpoint: Checkpoint, unloads: int | None = None) -> None:
global _agent, _loaded, _device_name, _loading, _failure, _generation, _last_used, _gpu_load
started = time.monotonic()
- unloads = _unloads
+ if unloads is None:
+ unloads = _unloads
+ if _unloads != unloads:
+ with _state_lock:
+ _loading = None
+ logger.info("System One dropped %s: it was unloaded before it loaded", checkpoint.name)
+ return
+ _load_generation.unloads = unloads
try:
agent, device = _load_checkpoint(checkpoint)
except Exception as exc:
message = f"Could not load {checkpoint.name}: {type(exc).__name__}: {exc}"
- logger.warning("System One load failed: %s", message, exc_info = True)
with _state_lock:
- _failure = (checkpoint, message, time.monotonic() + FAILURE_BACKOFF_S)
+ stale = _unloads != unloads
+ if not stale:
+ _failure = (checkpoint, message, time.monotonic() + FAILURE_BACKOFF_S)
_loading = None
_gpu_load = False
+ if stale:
+ logger.info("System One dropped %s: it was unloaded while it loaded", checkpoint.name)
+ else:
+ logger.warning("System One load failed: %s", message, exc_info = True)
return
with _state_lock:
stale = _unloads != unloads
@@ -1481,7 +1504,7 @@ def _clef_blocked_by_training(checkpoint: Checkpoint) -> None:
)
-def _ensure_loading(checkpoint: Checkpoint) -> threading.Thread | None:
+def _ensure_loading(checkpoint: Checkpoint, unloads: int | None = None) -> threading.Thread | None:
global _loader, _loading
# Asked before _state_lock: the training backend takes its own lock.
_clef_blocked_by_training(checkpoint)
@@ -1520,17 +1543,19 @@ def _ensure_loading(checkpoint: Checkpoint) -> threading.Thread | None:
raise Unavailable(503, "model_loading", f"{checkpoint.name} is downloading", retry_after = 5)
with _state_lock:
_loader = threading.Thread(
- target = _load, args = (checkpoint,), name = "systemone-load", daemon = True
+ target = _load, args = (checkpoint, unloads), name = "systemone-load", daemon = True
)
_loader.start()
return _loader
def _agent_for(checkpoint: Checkpoint, wait: float):
+ unloads = _unloads
loader = _ensure_loading(checkpoint)
if loader is not None:
loader.join(wait)
- if loader.is_alive() or _ensure_loading(checkpoint) is not None:
+ # An unload while this waited (a settings change) wins over a load of the old checkpoint.
+ if loader.is_alive() or _unloads != unloads or _ensure_loading(checkpoint) is not None:
raise Unavailable(
503, "model_loading", f"{checkpoint.name} is still loading", retry_after = 5
)
@@ -2306,27 +2331,112 @@ def status() -> dict[str, Any]:
def ensure_can_unload() -> None:
with _state_lock:
+ if _reloading is not None and _reloading == _loading:
+ return
# _loading: a load claimed but not yet started would otherwise land after this unload.
if _loading is not None or (_loader is not None and _loader.is_alive()):
raise Unavailable(409, "model_loading", "Wait for the load to finish before unloading")
-def unload() -> bool:
+def unload(reload_after_training: bool = False) -> bool:
global _agent, _loaded, _device_name, _failure, _generation, _unloads
ensure_can_unload()
with _run_lock:
- agent, was_loaded = _agent, _agent is not None
+ agent, was_loaded, checkpoint = _agent, _agent is not None, _loaded
_agent = _loaded = _device_name = None
_failure = None
_generation += 1
_unloads += 1
+ unloads = _unloads
_close(agent)
# Dropped first, so an in-process MLX or MPS Laya is collected before the allocator caches are cleared.
del agent
_release_memory()
+ if (
+ reload_after_training
+ and getattr(checkpoint, "layout", None) == "clef"
+ and not _arbitrated(checkpoint)
+ ):
+ threading.Thread(
+ target = _reload_after_training,
+ args = (checkpoint, unloads),
+ name = "systemone-reload",
+ daemon = True,
+ ).start()
return was_loaded
+def _clef_preempted() -> bool:
+ global _reload_stopped_by
+ from core.inference.gpu_arbiter import current_owner
+
+ if _training_active():
+ reason = "training"
+ elif _reloading is not None and current_owner() is not None:
+ # A reload nobody asked for yields to a chat, image or video model that claims the GPU meanwhile.
+ reason = "gpu"
+ else:
+ return False
+ if _reloading is not None:
+ _reload_stopped_by = reason
+ return True
+
+
+def _reload_after_training(checkpoint: Checkpoint, unloads: int) -> None:
+ global _reloading, _reload_stopped_by, _failure
+ from core.inference.gpu_arbiter import current_owner
+
+ while True:
+ while _training_active():
+ time.sleep(TRAINING_POLL_S)
+ with _state_lock:
+ if (
+ _unloads != unloads
+ or _agent is not None
+ or _loading is not None
+ or current_owner() is not None
+ ):
+ return
+ _reloading = checkpoint
+ _reload_stopped_by = None
+ try:
+ loader = _ensure_loading(checkpoint, unloads)
+ if loader is not None:
+ loader.join()
+ except Unavailable:
+ pass
+ finally:
+ with _state_lock:
+ _reloading = None
+ training = _reload_stopped_by == "training" or _training_active()
+ if training or _reload_stopped_by == "gpu" or current_owner() is not None:
+ # Another run, or a chat, image or video model, took the GPU while this loaded: not a failure to report.
+ with _state_lock:
+ if _failure is not None and _failure[0] == checkpoint:
+ _failure = None
+ if not training:
+ break
+ agent = _agent
+ if agent is not None and _loaded == checkpoint:
+ from .clef_runtime import ClefWorkerError
+
+ # The first decision builds the GPU kernels; done now, a request does not wait for it.
+ try:
+ if current_owner() is None:
+ agent.decide("ready", {"ready": {"type": "noul"}})
+ except ClefWorkerError:
+ logger.info(
+ "Retiring %s: its worker failed the warm-up", checkpoint.name, exc_info = True
+ )
+ _evict_agent(agent)
+ return
+ except Exception:
+ logger.debug("Could not warm up %s", checkpoint.name, exc_info = True)
+ if current_owner() is not None:
+ logger.info("Retiring %s: another model took the GPU as it reloaded", checkpoint.name)
+ _evict_agent(agent)
+
+
def shutdown() -> None:
"""Studio shutdown: end a llama.cpp decision server or Laya worker (in-process models go with the process)."""
global _agent, _loaded, _device_name, _generation, _unloads
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index 518072a644c..e62f34083a1 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -2291,7 +2291,7 @@ async def start_training(
from core.systemone import laya_runtime
if laya_runtime.status()["holds_gpu"]:
logger.info("Unloading the Decision API model to free GPU memory for training")
- laya_runtime.unload()
+ laya_runtime.unload(reload_after_training = True)
except Exception as e:
logger.warning("Could not unload the Decision API model for training: %s", e)
diff --git a/studio/backend/tests/test_decision_serving.py b/studio/backend/tests/test_decision_serving.py
index 22deb8c28f6..ce081f79660 100644
--- a/studio/backend/tests/test_decision_serving.py
+++ b/studio/backend/tests/test_decision_serving.py
@@ -367,9 +367,11 @@ def clef(home, monkeypatch):
cancelled = None,
):
self.folder, self.closed, self.cancelled = folder, False, cancelled
+ self.states = []
state.agents.append(self)
def decide(self, state_, questions):
+ self.states.append(state_)
answers = {name: {"type": "noul", "noul": 0.25} for name in questions}
return {"answers": answers, "input_tokens": 7, "truncated": False}
@@ -380,6 +382,8 @@ def clef(home, monkeypatch):
_spoof_device(monkeypatch, "cuda")
monkeypatch.setattr(laya_runtime, "_load_checkpoint", _REAL_LOAD)
monkeypatch.setattr(laya_runtime, "_training_active", lambda: state.training)
+ # An owner an earlier test left would stop the reload after training.
+ monkeypatch.setattr("core.inference.gpu_arbiter._owner", None)
return state
@@ -445,7 +449,344 @@ def test_a_clef_load_that_training_overtakes_frees_the_gpu(home, clef, monkeypat
assert clef.agents[0].closed
assert laya_runtime._agent is None and laya_runtime._device_name is None
# The loader also stops the worker mid-load, not only once it reports ready.
- assert clef.agents[0].cancelled is laya_runtime._training_active
+ assert clef.agents[0].cancelled()
+
+
+def _reload_finished():
+ for thread in threading.enumerate():
+ if thread.name == "systemone-reload":
+ thread.join(5)
+
+
+def test_a_clef_that_training_unloaded_is_back_and_warm_when_the_run_ends(
+ home, client, clef, monkeypatch
+):
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ assert clef.agents[0].closed
+ time.sleep(0.2)
+ assert len(clef.agents) == 1
+ clef.training = False
+ _reload_finished()
+ assert client.get("/api/settings/systemone").json()["loaded_model"] == served
+ assert len(clef.agents[1].states) == 1
+ assert _post(client).status_code == 200
+ assert len(clef.agents) == 2
+
+
+def test_a_settings_change_cancels_the_reload_after_training(home, client, clef, monkeypatch):
+ from core.systemone import clef_runtime
+
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ started = threading.Event()
+ agent = clef_runtime.ClefAgent
+
+ def loading(folder, cancelled = None):
+ agent(folder, cancelled)
+ started.set()
+ while not cancelled():
+ time.sleep(0.01)
+ raise clef_runtime.ClefWorkerError("stopped")
+
+ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ clef.training = False
+ assert started.wait(5)
+ assert _put(client, enabled = False).status_code == 200
+ _reload_finished()
+ settings = client.get("/api/settings/systemone").json()
+ assert settings["loaded_model"] is settings["loading_model"] is settings["error"] is None
+
+
+def test_a_request_waiting_on_the_reload_does_not_bring_back_a_disabled_clef(
+ home, client, clef, monkeypatch
+):
+ from core.systemone import clef_runtime
+
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ started = threading.Event()
+ agent = clef_runtime.ClefAgent
+
+ def loading(folder, cancelled = None):
+ built = agent(folder, cancelled)
+ if len(clef.agents) > 2:
+ return built
+ started.set()
+ while not cancelled():
+ time.sleep(0.01)
+ raise clef_runtime.ClefWorkerError("stopped")
+
+ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ clef.training = False
+ assert started.wait(5)
+ answers, waiting = [], threading.Event()
+ ensure = laya_runtime._ensure_loading
+
+ def ensuring(checkpoint, unloads = None):
+ loader = ensure(checkpoint, unloads)
+ # Only the reload passes unloads; a request is handed the reload's loader.
+ if unloads is None and loader is not None:
+ waiting.set()
+ return loader
+
+ monkeypatch.setattr(laya_runtime, "_ensure_loading", ensuring)
+ request = threading.Thread(target = lambda: answers.append(_post(client)))
+ request.start()
+ assert waiting.wait(10)
+ assert _put(client, enabled = False).status_code == 200
+ request.join(10)
+ _reload_finished()
+ assert answers[0].status_code == 503
+ assert len(clef.agents) == 2
+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None
+
+
+def test_an_unload_as_the_reload_starts_wins(home, client, clef, monkeypatch):
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ probe = laya_runtime._hub_download_active
+
+ def unloaded_meanwhile(checkpoint):
+ if threading.current_thread().name == "systemone-reload":
+ assert client.post("/api/settings/systemone/unload").status_code == 200
+ return probe(checkpoint)
+
+ monkeypatch.setattr(laya_runtime, "_hub_download_active", unloaded_meanwhile)
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ clef.training = False
+ _reload_finished()
+ assert len(clef.agents) == 1
+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None
+
+
+def test_a_run_that_starts_during_the_reload_is_waited_out(home, client, clef, monkeypatch):
+ from core.systemone import clef_runtime
+
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ started = threading.Event()
+ agent = clef_runtime.ClefAgent
+
+ def loading(folder, cancelled = None):
+ built = agent(folder, cancelled)
+ if len(clef.agents) > 2:
+ return built
+ started.set()
+ while not cancelled():
+ time.sleep(0.01)
+ raise clef_runtime.ClefWorkerError("A training run took the GPU while Clef loaded.")
+
+ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ clef.training = False
+ assert started.wait(5)
+ clef.training = True
+ time.sleep(0.3)
+ assert client.get("/api/settings/systemone").json()["error"] is None
+ clef.training = False
+ _reload_finished()
+ settings = client.get("/api/settings/systemone").json()
+ assert settings["loaded_model"] == served and settings["error"] is None
+ assert len(clef.agents) == 3
+
+
+def test_a_chat_model_that_takes_the_gpu_stops_the_reload(home, client, clef, monkeypatch):
+ from core.inference import gpu_arbiter
+ from core.systemone import clef_runtime
+
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ started = threading.Event()
+ agent = clef_runtime.ClefAgent
+
+ def loading(folder, cancelled = None):
+ agent(folder, cancelled)
+ started.set()
+ while not cancelled():
+ time.sleep(0.01)
+ raise clef_runtime.ClefWorkerError("stopped")
+
+ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ clef.training = False
+ assert started.wait(5)
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
+ _reload_finished()
+ settings = client.get("/api/settings/systemone").json()
+ assert settings["loaded_model"] is settings["loading_model"] is settings["error"] is None
+ assert len(clef.agents) == 2
+
+
+def test_a_clef_worker_that_fails_the_warm_up_is_retired(home, client, clef, monkeypatch):
+ from core.systemone import clef_runtime
+
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ agent = clef_runtime.ClefAgent
+
+ class Dying(agent):
+ def decide(self, state_, questions):
+ if state_ == "ready":
+ raise clef_runtime.ClefWorkerError("The Clef worker exited (code -9)")
+ return super().decide(state_, questions)
+
+ monkeypatch.setattr(clef_runtime, "ClefAgent", Dying)
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ clef.training = False
+ _reload_finished()
+ assert clef.agents[1].closed
+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None
+ assert _post(client).status_code == 200
+ assert len(clef.agents) == 3
+
+
+def test_a_run_over_before_the_worker_is_reaped_still_retries_the_reload(
+ home, client, clef, monkeypatch
+):
+ from core.systemone import clef_runtime
+
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ started = threading.Event()
+ agent = clef_runtime.ClefAgent
+
+ def loading(folder, cancelled = None):
+ built = agent(folder, cancelled)
+ if len(clef.agents) > 2:
+ return built
+ started.set()
+ while not cancelled():
+ time.sleep(0.01)
+ # A run that failed at once is over before the worker is reaped.
+ clef.training = False
+ raise clef_runtime.ClefWorkerError("A training run took the GPU while Clef loaded.")
+
+ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ clef.training = False
+ assert started.wait(5)
+ clef.training = True
+ _reload_finished()
+ settings = client.get("/api/settings/systemone").json()
+ assert settings["loaded_model"] == served and settings["error"] is None
+ assert len(clef.agents) == 3
+
+
+def test_a_chat_model_that_takes_the_gpu_during_the_warm_up_retires_the_reload(
+ home, client, clef, monkeypatch
+):
+ from core.inference import gpu_arbiter
+ from core.systemone import clef_runtime
+
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ agent = clef_runtime.ClefAgent
+
+ class Warming(agent):
+ def decide(self, state_, questions):
+ if state_ == "ready":
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
+ return super().decide(state_, questions)
+
+ monkeypatch.setattr(clef_runtime, "ClefAgent", Warming)
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ clef.training = False
+ _reload_finished()
+ assert clef.agents[1].states == ["ready"] and clef.agents[1].closed
+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None
+
+
+def test_a_settings_change_during_the_reloads_lookup_stops_its_worker(
+ home, client, clef, monkeypatch
+):
+ from core.systemone import clef_runtime
+
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ lookup = laya_runtime._checkpoint_dir
+ stopped = []
+
+ def looked_up(checkpoint, *args, **kwargs):
+ if threading.current_thread().name == "systemone-load" and len(clef.agents) == 1:
+ assert _put(client, enabled = False).status_code == 200
+ return lookup(checkpoint, *args, **kwargs)
+
+ agent = clef_runtime.ClefAgent
+
+ def loading(folder, cancelled = None):
+ built = agent(folder, cancelled)
+ deadline = time.monotonic() + 3
+ while time.monotonic() < deadline:
+ if cancelled():
+ stopped.append(True)
+ raise clef_runtime.ClefWorkerError("stopped")
+ time.sleep(0.01)
+ return built
+
+ monkeypatch.setattr(laya_runtime, "_checkpoint_dir", looked_up)
+ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ clef.training = False
+ _reload_finished()
+ assert stopped == [True]
+ settings = client.get("/api/settings/systemone").json()
+ assert settings["loaded_model"] is settings["loading_model"] is settings["error"] is None
+
+
[email protected]("meanwhile", ["unloaded", "chat_on_gpu"])
+def test_no_reload_after_training_once_the_user_moved_on(
+ home, client, clef, monkeypatch, meanwhile
+):
+ from core.inference import gpu_arbiter
+
+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)
+ served = _clef_fine_tune(home, "clef_served_1")
+ assert _put(client, enabled = True, model = served).status_code == 200
+ assert _post(client).status_code == 200
+ clef.training = True
+ laya_runtime.unload(reload_after_training = True)
+ if meanwhile == "unloaded":
+ assert client.post("/api/settings/systemone/unload").status_code == 200
+ else:
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
+ clef.training = False
+ _reload_finished()
+ assert len(clef.agents) == 1
+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None
def test_a_clef_worker_that_never_reports_ready_is_stopped(monkeypatch):