Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop a Clef decision model from outliving a crashed Unsloth (#14556)
Nilay458190c9b68c
diff --git a/studio/backend/core/systemone/clef_runtime.py b/studio/backend/core/systemone/clef_runtime.pyindex 1a575dc1d20..36a75a5fcd5 100644--- a/studio/backend/core/systemone/clef_runtime.py+++ b/studio/backend/core/systemone/clef_runtime.py@@ -138,27 +138,44 @@ class ClefAgent:folder: Path,cancelled: Callable[[], bool] | None = None,):+ from utils.native_path_leases import (+ native_path_secret_removed_for_child_start,+ run_without_native_path_secret,+ )+ from utils.process_lifetime import (+ adopt_pid,+ is_process_shutting_down,+ spawn_on_lifetime_thread,+ )++ if is_process_shutting_down():+ raise ClefWorkerError("Unsloth is shutting down; the Clef worker was not started.")self._lock = threading.Lock()self._broken: str | None = Noneself._conn, child = _CTX.Pipe()env = os.environ.get("UNSLOTH_IS_PRESENT")os.environ["UNSLOTH_IS_PRESENT"] = "1"try:- self._process = _CTX.Process(- target = run_clef_worker,- args = (child, str(folder)),- name = "systemone-clef",- daemon = True,- )- self._process.start()+ with native_path_secret_removed_for_child_start():+ self._process = _CTX.Process(+ target = run_without_native_path_secret,+ args = (run_clef_worker, child, str(folder)),+ name = "systemone-clef",+ daemon = True,+ )+ # Linux PDEATHSIG follows the spawning thread, and this runs on a loader thread that ends after the load.+ spawn_on_lifetime_thread(self._process.start)finally:if env is None:os.environ.pop("UNSLOTH_IS_PRESENT", None)+ adopt_pid(self._process.pid)child.close()try:# Polled in slices so a training run starting mid-load stops the worker before both need the GPU.deadline = time.monotonic() + LOAD_TIMEOUT_Swhile time.monotonic() < deadline and not self._conn.poll(1.0):+ if is_process_shutting_down():+ raise ClefWorkerError("The Clef worker was stopped while it loaded.")if cancelled is not None and cancelled():raise ClefWorkerError("A training run took the GPU while Clef loaded.")if not self._conn.poll(0):@@ -215,3 +232,6 @@ class ClefAgent:self._process.kill()self._process.join(CLOSE_TIMEOUT_S)self._conn.close()+ if not self._process.is_alive():+ from utils.process_lifetime import forget_pid+ forget_pid(self._process.pid)diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.pyindex 268e10c17b8..3b34b575152 100644--- a/studio/backend/core/systemone/laya_runtime.py+++ b/studio/backend/core/systemone/laya_runtime.py@@ -2494,15 +2494,17 @@ def _reload_after_training(checkpoint: Checkpoint, unloads: int) -> None:def shutdown() -> None:- """Studio shutdown: end a llama.cpp decision server or Laya worker (in-process models go with the process)."""+ """Studio shutdown: end a llama.cpp decision server, Laya or Clef worker (in-process models go with the process)."""global _agent, _loaded, _device_name, _generation, _unloads+ from .clef_runtime import ClefAgent+with _state_lock:timer = _idle_timerif timer is not None:timer.cancel()with _run_lock:_unloads += 1- if not _is_native(_loaded) and not isinstance(_agent, LayaWorker):+ if not _is_native(_loaded) and not isinstance(_agent, (LayaWorker, ClefAgent)):returnagent, _agent, _loaded, _device_name = _agent, None, None, None_generation += 1diff --git a/studio/backend/tests/test_decision_serving.py b/studio/backend/tests/test_decision_serving.pyindex 9e8eae9995f..a2f9b685265 100644--- a/studio/backend/tests/test_decision_serving.py+++ b/studio/backend/tests/test_decision_serving.py@@ -795,10 +795,10 @@ def test_a_clef_worker_that_never_reports_ready_is_stopped(monkeypatch):calls = []class Process:- exitcode = None+ exitcode = pid = Nonedef __init__(self, **kwargs):- self.child = kwargs["args"][0]+ self.child = kwargs["args"][1]def start(self):# A live child holds its own end of the pipe, so the parent sees silence, not EOF.@@ -835,10 +835,10 @@ def test_a_training_run_stops_a_clef_worker_still_loading(monkeypatch):calls = []class Process:- exitcode = None+ exitcode = pid = Nonedef __init__(self, **kwargs):- self.child = kwargs["args"][0]+ self.child = kwargs["args"][1]def start(self):# A live child holds its own end of the pipe, so the parent sees silence, not EOF.@@ -871,6 +871,114 @@ def test_a_training_run_stops_a_clef_worker_still_loading(monkeypatch):assert "kill" in calls+def test_unsloth_shutting_down_stops_a_clef_worker_still_loading(monkeypatch):+ from core.systemone import clef_runtime+ from utils import process_lifetime++ calls = []++ class Process:+ exitcode = pid = None++ def __init__(self, **kwargs):+ self.child = kwargs["args"][1]++ def start(self):+ held.append(self.child)+ self.child.close = lambda: None++ def join(self, timeout = None):+ calls.append("join")++ def is_alive(self):+ return "kill" not in calls++ def kill(self):+ calls.append("kill")++ held = []+ reads = iter([False])+ monkeypatch.setattr(process_lifetime, "is_process_shutting_down", lambda: next(reads, True))+ monkeypatch.setattr(clef_runtime._CTX, "Process", Process)+ monkeypatch.setattr(clef_runtime, "LOAD_TIMEOUT_S", 600)+ try:+ started = time.monotonic()+ with pytest.raises(clef_runtime.ClefWorkerError, match = "stopped while it loaded"):+ clef_runtime.ClefAgent("unused")+ assert time.monotonic() - started < 30+ finally:+ for child in held:+ del child.close+ child.close()+ assert "kill" in calls+++def _running(pid):+ try:+ with open(f"/proc/{pid}/stat", encoding = "utf-8") as stat:+ return stat.read().rsplit(")", 1)[1].split()[0] != "Z"+ except OSError:+ return False++[email protected](not sys.platform.startswith("linux"), reason = "PDEATHSIG is Linux only")+def test_a_loading_clef_worker_dies_with_unsloth(tmp_path):+ import signal+ import subprocess++ # The worker's load never returns, as a long GPU load would: its first import blocks.+ (tmp_path / "torch").mkdir()+ (tmp_path / "torch" / "__init__.py").write_text(+ "import os, time\n"+ "with open(os.environ['CLEF_WORKER_PID'], 'w') as f:\n"+ " f.write(str(os.getpid()))\n"+ "time.sleep(600)\n",+ encoding = "utf-8",+ )+ pid_file = tmp_path / "worker.pid"+ unsloth = subprocess.Popen(+ [+ sys.executable,+ "-c",+ "import threading\n"+ "from core.systemone.clef_runtime import ClefAgent\n"+ "threading.Thread(target = ClefAgent, args = ('unused',)).start()\n"+ "threading.Event().wait()\n",+ ],+ cwd = Path(laya_runtime.__file__).resolve().parents[2],+ env = dict(os.environ, PYTHONPATH = str(tmp_path), CLEF_WORKER_PID = str(pid_file)),+ )+ worker = None+ try:+ deadline = time.monotonic() + 60+ while worker is None and time.monotonic() < deadline:+ text = pid_file.read_text(encoding = "utf-8") if pid_file.exists() else ""+ worker = int(text) if text else None+ time.sleep(0.1)+ assert worker is not None, "the Clef worker never started loading"+ unsloth.kill()+ unsloth.wait()+ deadline = time.monotonic() + 15+ while _running(worker) and time.monotonic() < deadline:+ time.sleep(0.1)+ assert not _running(worker)+ finally:+ unsloth.kill()+ unsloth.wait()+ if worker is not None and _running(worker):+ os.kill(worker, signal.SIGKILL)+++def test_shutdown_ends_a_clef_worker(home, client, clef):+ assert (+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200+ )+ assert _post(client).status_code == 200+ laya_runtime.shutdown()+ assert clef.agents[0].closed+ assert client.get("/api/settings/systemone").json()["loaded_device"] is None++def test_the_catalog_offers_the_stock_clef_models():for name, repo in (("clef", "Cloudflare/clef"), ("clef-flash", "Cloudflare/clef-flash")):checkpoint = catalog.CHECKPOINTS[name]diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.pyindex b28be02b76c..bde3423adc3 100644--- a/studio/backend/tests/test_orchestrator_unload_cancel.py+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py@@ -3884,6 +3884,7 @@ def test_every_long_lived_spawner_consults_the_shutdown_latch():"core/inference/stt_transformers_worker.py","core/inference/video_ltx25.py","core/rag/embed_llama_server.py",+ "core/systemone/clef_runtime.py","core/systemone/native_worker.py","core/systemone/laya_worker.py","core/training/token_preview.py",