Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: reload a Clef as soon as the training run that unloaded it ends (#13482)
Nilay9b30c4f75cb6
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.pyindex a8c56b5c0a9..a421f7b0343 100644--- a/studio/backend/core/systemone/laya_runtime.py+++ b/studio/backend/core/systemone/laya_runtime.py@@ -48,6 +48,13 @@ _unloads = 0_last_used = 0.0_idle_timer: threading.Timer | None = None_fallback_reason: str | None = None+# A Clef a training run unloaded, reloading once the run ends; an unload cancels it instead of waiting.+_reloading: Checkpoint | None = None+# Why the reload's load was stopped ("training" or "gpu"), recorded when it happens: a short run can be over by the join.+_reload_stopped_by: str | None = None+# The unload generation the running load started for: an unload during its Hub lookup still stops the worker.+_load_generation = threading.local()+TRAINING_POLL_S = 2.0# A llama.cpp decision server unloads after this long without a request.IDLE_UNLOAD_S = 300.0# Binaries (path, mtime) whose server answered without a "decisions" output.@@ -1024,9 +1031,13 @@ def _load_checkpoint(checkpoint: Checkpoint):from .clef_runtime import ClefAgent, ClefWorkerError_evict()- agent = ClefAgent(root, cancelled = _training_active)+ unloads = getattr(_load_generation, "unloads", _unloads)+ agent = ClefAgent(+ root,+ cancelled = lambda: _clef_preempted() or _unloads != unloads,+ )# Training may have started while this loaded, and Clef has no CPU fallback.- if _training_active():+ if _clef_preempted():agent.close()raise ClefWorkerError(f"{checkpoint.name} needs the GPU, which a training run took while it loaded."@@ -1352,19 +1363,31 @@ def loading_repo_ids() -> tuple[str, ...]:return ()-def _load(checkpoint: Checkpoint) -> None:+def _load(checkpoint: Checkpoint, unloads: int | None = None) -> None:global _agent, _loaded, _device_name, _loading, _failure, _generation, _last_used, _gpu_loadstarted = time.monotonic()- unloads = _unloads+ if unloads is None:+ unloads = _unloads+ if _unloads != unloads:+ with _state_lock:+ _loading = None+ logger.info("System One dropped %s: it was unloaded before it loaded", checkpoint.name)+ return+ _load_generation.unloads = unloadstry:agent, device = _load_checkpoint(checkpoint)except Exception as exc:message = f"Could not load {checkpoint.name}: {type(exc).__name__}: {exc}"- logger.warning("System One load failed: %s", message, exc_info = True)with _state_lock:- _failure = (checkpoint, message, time.monotonic() + FAILURE_BACKOFF_S)+ stale = _unloads != unloads+ if not stale:+ _failure = (checkpoint, message, time.monotonic() + FAILURE_BACKOFF_S)_loading = None_gpu_load = False+ if stale:+ logger.info("System One dropped %s: it was unloaded while it loaded", checkpoint.name)+ else:+ logger.warning("System One load failed: %s", message, exc_info = True)returnwith _state_lock:stale = _unloads != unloads@@ -1481,7 +1504,7 @@ def _clef_blocked_by_training(checkpoint: Checkpoint) -> None:)-def _ensure_loading(checkpoint: Checkpoint) -> threading.Thread | None:+def _ensure_loading(checkpoint: Checkpoint, unloads: int | None = None) -> threading.Thread | None:global _loader, _loading# Asked before _state_lock: the training backend takes its own lock._clef_blocked_by_training(checkpoint)@@ -1520,17 +1543,19 @@ def _ensure_loading(checkpoint: Checkpoint) -> threading.Thread | None:raise Unavailable(503, "model_loading", f"{checkpoint.name} is downloading", retry_after = 5)with _state_lock:_loader = threading.Thread(- target = _load, args = (checkpoint,), name = "systemone-load", daemon = True+ target = _load, args = (checkpoint, unloads), name = "systemone-load", daemon = True)_loader.start()return _loaderdef _agent_for(checkpoint: Checkpoint, wait: float):+ unloads = _unloadsloader = _ensure_loading(checkpoint)if loader is not None:loader.join(wait)- if loader.is_alive() or _ensure_loading(checkpoint) is not None:+ # An unload while this waited (a settings change) wins over a load of the old checkpoint.+ if loader.is_alive() or _unloads != unloads or _ensure_loading(checkpoint) is not None:raise Unavailable(503, "model_loading", f"{checkpoint.name} is still loading", retry_after = 5)@@ -2306,27 +2331,112 @@ def status() -> dict[str, Any]:def ensure_can_unload() -> None:with _state_lock:+ if _reloading is not None and _reloading == _loading:+ return# _loading: a load claimed but not yet started would otherwise land after this unload.if _loading is not None or (_loader is not None and _loader.is_alive()):raise Unavailable(409, "model_loading", "Wait for the load to finish before unloading")-def unload() -> bool:+def unload(reload_after_training: bool = False) -> bool:global _agent, _loaded, _device_name, _failure, _generation, _unloadsensure_can_unload()with _run_lock:- agent, was_loaded = _agent, _agent is not None+ agent, was_loaded, checkpoint = _agent, _agent is not None, _loaded_agent = _loaded = _device_name = None_failure = None_generation += 1_unloads += 1+ unloads = _unloads_close(agent)# Dropped first, so an in-process MLX or MPS Laya is collected before the allocator caches are cleared.del agent_release_memory()+ if (+ reload_after_training+ and getattr(checkpoint, "layout", None) == "clef"+ and not _arbitrated(checkpoint)+ ):+ threading.Thread(+ target = _reload_after_training,+ args = (checkpoint, unloads),+ name = "systemone-reload",+ daemon = True,+ ).start()return was_loaded+def _clef_preempted() -> bool:+ global _reload_stopped_by+ from core.inference.gpu_arbiter import current_owner++ if _training_active():+ reason = "training"+ elif _reloading is not None and current_owner() is not None:+ # A reload nobody asked for yields to a chat, image or video model that claims the GPU meanwhile.+ reason = "gpu"+ else:+ return False+ if _reloading is not None:+ _reload_stopped_by = reason+ return True+++def _reload_after_training(checkpoint: Checkpoint, unloads: int) -> None:+ global _reloading, _reload_stopped_by, _failure+ from core.inference.gpu_arbiter import current_owner++ while True:+ while _training_active():+ time.sleep(TRAINING_POLL_S)+ with _state_lock:+ if (+ _unloads != unloads+ or _agent is not None+ or _loading is not None+ or current_owner() is not None+ ):+ return+ _reloading = checkpoint+ _reload_stopped_by = None+ try:+ loader = _ensure_loading(checkpoint, unloads)+ if loader is not None:+ loader.join()+ except Unavailable:+ pass+ finally:+ with _state_lock:+ _reloading = None+ training = _reload_stopped_by == "training" or _training_active()+ if training or _reload_stopped_by == "gpu" or current_owner() is not None:+ # Another run, or a chat, image or video model, took the GPU while this loaded: not a failure to report.+ with _state_lock:+ if _failure is not None and _failure[0] == checkpoint:+ _failure = None+ if not training:+ break+ agent = _agent+ if agent is not None and _loaded == checkpoint:+ from .clef_runtime import ClefWorkerError++ # The first decision builds the GPU kernels; done now, a request does not wait for it.+ try:+ if current_owner() is None:+ agent.decide("ready", {"ready": {"type": "noul"}})+ except ClefWorkerError:+ logger.info(+ "Retiring %s: its worker failed the warm-up", checkpoint.name, exc_info = True+ )+ _evict_agent(agent)+ return+ except Exception:+ logger.debug("Could not warm up %s", checkpoint.name, exc_info = True)+ if current_owner() is not None:+ logger.info("Retiring %s: another model took the GPU as it reloaded", checkpoint.name)+ _evict_agent(agent)++def shutdown() -> None:"""Studio shutdown: end a llama.cpp decision server or Laya worker (in-process models go with the process)."""global _agent, _loaded, _device_name, _generation, _unloadsdiff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.pyindex 518072a644c..e62f34083a1 100644--- a/studio/backend/routes/training.py+++ b/studio/backend/routes/training.py@@ -2291,7 +2291,7 @@ async def start_training(from core.systemone import laya_runtimeif laya_runtime.status()["holds_gpu"]:logger.info("Unloading the Decision API model to free GPU memory for training")- laya_runtime.unload()+ laya_runtime.unload(reload_after_training = True)except Exception as e:logger.warning("Could not unload the Decision API model for training: %s", e)diff --git a/studio/backend/tests/test_decision_serving.py b/studio/backend/tests/test_decision_serving.pyindex 22deb8c28f6..ce081f79660 100644--- a/studio/backend/tests/test_decision_serving.py+++ b/studio/backend/tests/test_decision_serving.py@@ -367,9 +367,11 @@ def clef(home, monkeypatch):cancelled = None,):self.folder, self.closed, self.cancelled = folder, False, cancelled+ self.states = []state.agents.append(self)def decide(self, state_, questions):+ self.states.append(state_)answers = {name: {"type": "noul", "noul": 0.25} for name in questions}return {"answers": answers, "input_tokens": 7, "truncated": False}@@ -380,6 +382,8 @@ def clef(home, monkeypatch):_spoof_device(monkeypatch, "cuda")monkeypatch.setattr(laya_runtime, "_load_checkpoint", _REAL_LOAD)monkeypatch.setattr(laya_runtime, "_training_active", lambda: state.training)+ # An owner an earlier test left would stop the reload after training.+ monkeypatch.setattr("core.inference.gpu_arbiter._owner", None)return state@@ -445,7 +449,344 @@ def test_a_clef_load_that_training_overtakes_frees_the_gpu(home, clef, monkeypatassert clef.agents[0].closedassert laya_runtime._agent is None and laya_runtime._device_name is None# The loader also stops the worker mid-load, not only once it reports ready.- assert clef.agents[0].cancelled is laya_runtime._training_active+ assert clef.agents[0].cancelled()+++def _reload_finished():+ for thread in threading.enumerate():+ if thread.name == "systemone-reload":+ thread.join(5)+++def test_a_clef_that_training_unloaded_is_back_and_warm_when_the_run_ends(+ home, client, clef, monkeypatch+):+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ assert clef.agents[0].closed+ time.sleep(0.2)+ assert len(clef.agents) == 1+ clef.training = False+ _reload_finished()+ assert client.get("/api/settings/systemone").json()["loaded_model"] == served+ assert len(clef.agents[1].states) == 1+ assert _post(client).status_code == 200+ assert len(clef.agents) == 2+++def test_a_settings_change_cancels_the_reload_after_training(home, client, clef, monkeypatch):+ from core.systemone import clef_runtime++ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ started = threading.Event()+ agent = clef_runtime.ClefAgent++ def loading(folder, cancelled = None):+ agent(folder, cancelled)+ started.set()+ while not cancelled():+ time.sleep(0.01)+ raise clef_runtime.ClefWorkerError("stopped")++ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ clef.training = False+ assert started.wait(5)+ assert _put(client, enabled = False).status_code == 200+ _reload_finished()+ settings = client.get("/api/settings/systemone").json()+ assert settings["loaded_model"] is settings["loading_model"] is settings["error"] is None+++def test_a_request_waiting_on_the_reload_does_not_bring_back_a_disabled_clef(+ home, client, clef, monkeypatch+):+ from core.systemone import clef_runtime++ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ started = threading.Event()+ agent = clef_runtime.ClefAgent++ def loading(folder, cancelled = None):+ built = agent(folder, cancelled)+ if len(clef.agents) > 2:+ return built+ started.set()+ while not cancelled():+ time.sleep(0.01)+ raise clef_runtime.ClefWorkerError("stopped")++ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ clef.training = False+ assert started.wait(5)+ answers, waiting = [], threading.Event()+ ensure = laya_runtime._ensure_loading++ def ensuring(checkpoint, unloads = None):+ loader = ensure(checkpoint, unloads)+ # Only the reload passes unloads; a request is handed the reload's loader.+ if unloads is None and loader is not None:+ waiting.set()+ return loader++ monkeypatch.setattr(laya_runtime, "_ensure_loading", ensuring)+ request = threading.Thread(target = lambda: answers.append(_post(client)))+ request.start()+ assert waiting.wait(10)+ assert _put(client, enabled = False).status_code == 200+ request.join(10)+ _reload_finished()+ assert answers[0].status_code == 503+ assert len(clef.agents) == 2+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None+++def test_an_unload_as_the_reload_starts_wins(home, client, clef, monkeypatch):+ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ probe = laya_runtime._hub_download_active++ def unloaded_meanwhile(checkpoint):+ if threading.current_thread().name == "systemone-reload":+ assert client.post("/api/settings/systemone/unload").status_code == 200+ return probe(checkpoint)++ monkeypatch.setattr(laya_runtime, "_hub_download_active", unloaded_meanwhile)+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ clef.training = False+ _reload_finished()+ assert len(clef.agents) == 1+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None+++def test_a_run_that_starts_during_the_reload_is_waited_out(home, client, clef, monkeypatch):+ from core.systemone import clef_runtime++ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ started = threading.Event()+ agent = clef_runtime.ClefAgent++ def loading(folder, cancelled = None):+ built = agent(folder, cancelled)+ if len(clef.agents) > 2:+ return built+ started.set()+ while not cancelled():+ time.sleep(0.01)+ raise clef_runtime.ClefWorkerError("A training run took the GPU while Clef loaded.")++ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ clef.training = False+ assert started.wait(5)+ clef.training = True+ time.sleep(0.3)+ assert client.get("/api/settings/systemone").json()["error"] is None+ clef.training = False+ _reload_finished()+ settings = client.get("/api/settings/systemone").json()+ assert settings["loaded_model"] == served and settings["error"] is None+ assert len(clef.agents) == 3+++def test_a_chat_model_that_takes_the_gpu_stops_the_reload(home, client, clef, monkeypatch):+ from core.inference import gpu_arbiter+ from core.systemone import clef_runtime++ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ started = threading.Event()+ agent = clef_runtime.ClefAgent++ def loading(folder, cancelled = None):+ agent(folder, cancelled)+ started.set()+ while not cancelled():+ time.sleep(0.01)+ raise clef_runtime.ClefWorkerError("stopped")++ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ clef.training = False+ assert started.wait(5)+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)+ _reload_finished()+ settings = client.get("/api/settings/systemone").json()+ assert settings["loaded_model"] is settings["loading_model"] is settings["error"] is None+ assert len(clef.agents) == 2+++def test_a_clef_worker_that_fails_the_warm_up_is_retired(home, client, clef, monkeypatch):+ from core.systemone import clef_runtime++ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ agent = clef_runtime.ClefAgent++ class Dying(agent):+ def decide(self, state_, questions):+ if state_ == "ready":+ raise clef_runtime.ClefWorkerError("The Clef worker exited (code -9)")+ return super().decide(state_, questions)++ monkeypatch.setattr(clef_runtime, "ClefAgent", Dying)+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ clef.training = False+ _reload_finished()+ assert clef.agents[1].closed+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None+ assert _post(client).status_code == 200+ assert len(clef.agents) == 3+++def test_a_run_over_before_the_worker_is_reaped_still_retries_the_reload(+ home, client, clef, monkeypatch+):+ from core.systemone import clef_runtime++ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ started = threading.Event()+ agent = clef_runtime.ClefAgent++ def loading(folder, cancelled = None):+ built = agent(folder, cancelled)+ if len(clef.agents) > 2:+ return built+ started.set()+ while not cancelled():+ time.sleep(0.01)+ # A run that failed at once is over before the worker is reaped.+ clef.training = False+ raise clef_runtime.ClefWorkerError("A training run took the GPU while Clef loaded.")++ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ clef.training = False+ assert started.wait(5)+ clef.training = True+ _reload_finished()+ settings = client.get("/api/settings/systemone").json()+ assert settings["loaded_model"] == served and settings["error"] is None+ assert len(clef.agents) == 3+++def test_a_chat_model_that_takes_the_gpu_during_the_warm_up_retires_the_reload(+ home, client, clef, monkeypatch+):+ from core.inference import gpu_arbiter+ from core.systemone import clef_runtime++ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ agent = clef_runtime.ClefAgent++ class Warming(agent):+ def decide(self, state_, questions):+ if state_ == "ready":+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)+ return super().decide(state_, questions)++ monkeypatch.setattr(clef_runtime, "ClefAgent", Warming)+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ clef.training = False+ _reload_finished()+ assert clef.agents[1].states == ["ready"] and clef.agents[1].closed+ assert client.get("/api/settings/systemone").json()["loaded_model"] is None+++def test_a_settings_change_during_the_reloads_lookup_stops_its_worker(+ home, client, clef, monkeypatch+):+ from core.systemone import clef_runtime++ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ lookup = laya_runtime._checkpoint_dir+ stopped = []++ def looked_up(checkpoint, *args, **kwargs):+ if threading.current_thread().name == "systemone-load" and len(clef.agents) == 1:+ assert _put(client, enabled = False).status_code == 200+ return lookup(checkpoint, *args, **kwargs)++ agent = clef_runtime.ClefAgent++ def loading(folder, cancelled = None):+ built = agent(folder, cancelled)+ deadline = time.monotonic() + 3+ while time.monotonic() < deadline:+ if cancelled():+ stopped.append(True)+ raise clef_runtime.ClefWorkerError("stopped")+ time.sleep(0.01)+ return built++ monkeypatch.setattr(laya_runtime, "_checkpoint_dir", looked_up)+ monkeypatch.setattr(clef_runtime, "ClefAgent", loading)+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ clef.training = False+ _reload_finished()+ assert stopped == [True]+ settings = client.get("/api/settings/systemone").json()+ assert settings["loaded_model"] is settings["loading_model"] is settings["error"] is None++[email protected]("meanwhile", ["unloaded", "chat_on_gpu"])+def test_no_reload_after_training_once_the_user_moved_on(+ home, client, clef, monkeypatch, meanwhile+):+ from core.inference import gpu_arbiter++ monkeypatch.setattr(laya_runtime, "TRAINING_POLL_S", 0.01, raising = False)+ served = _clef_fine_tune(home, "clef_served_1")+ assert _put(client, enabled = True, model = served).status_code == 200+ assert _post(client).status_code == 200+ clef.training = True+ laya_runtime.unload(reload_after_training = True)+ if meanwhile == "unloaded":+ assert client.post("/api/settings/systemone/unload").status_code == 200+ else:+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)+ clef.training = False+ _reload_finished()+ assert len(clef.agents) == 1+ assert client.get("/api/settings/systemone").json()["loaded_model"] is Nonedef test_a_clef_worker_that_never_reports_ready_is_stopped(monkeypatch):