Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop a Clef decision model from outliving a crashed Unsloth (#14556)

Nilay458190c9b68c
diff --git a/studio/backend/core/systemone/clef_runtime.py b/studio/backend/core/systemone/clef_runtime.py
index 1a575dc1d20..36a75a5fcd5 100644
--- a/studio/backend/core/systemone/clef_runtime.py
+++ b/studio/backend/core/systemone/clef_runtime.py
@@ -138,27 +138,44 @@ class ClefAgent:
folder: Path,
cancelled: Callable[[], bool] | None = None,
):
+ from utils.native_path_leases import (
+ native_path_secret_removed_for_child_start,
+ run_without_native_path_secret,
+ )
+ from utils.process_lifetime import (
+ adopt_pid,
+ is_process_shutting_down,
+ spawn_on_lifetime_thread,
+ )
+
+ if is_process_shutting_down():
+ raise ClefWorkerError("Unsloth is shutting down; the Clef worker was not started.")
self._lock = threading.Lock()
self._broken: str | None = None
self._conn, child = _CTX.Pipe()
env = os.environ.get("UNSLOTH_IS_PRESENT")
os.environ["UNSLOTH_IS_PRESENT"] = "1"
try:
- self._process = _CTX.Process(
- target = run_clef_worker,
- args = (child, str(folder)),
- name = "systemone-clef",
- daemon = True,
- )
- self._process.start()
+ with native_path_secret_removed_for_child_start():
+ self._process = _CTX.Process(
+ target = run_without_native_path_secret,
+ args = (run_clef_worker, child, str(folder)),
+ name = "systemone-clef",
+ daemon = True,
+ )
+ # Linux PDEATHSIG follows the spawning thread, and this runs on a loader thread that ends after the load.
+ spawn_on_lifetime_thread(self._process.start)
finally:
if env is None:
os.environ.pop("UNSLOTH_IS_PRESENT", None)
+ adopt_pid(self._process.pid)
child.close()
try:
# Polled in slices so a training run starting mid-load stops the worker before both need the GPU.
deadline = time.monotonic() + LOAD_TIMEOUT_S
while time.monotonic() < deadline and not self._conn.poll(1.0):
+ if is_process_shutting_down():
+ raise ClefWorkerError("The Clef worker was stopped while it loaded.")
if cancelled is not None and cancelled():
raise ClefWorkerError("A training run took the GPU while Clef loaded.")
if not self._conn.poll(0):
@@ -215,3 +232,6 @@ class ClefAgent:
self._process.kill()
self._process.join(CLOSE_TIMEOUT_S)
self._conn.close()
+ if not self._process.is_alive():
+ from utils.process_lifetime import forget_pid
+ forget_pid(self._process.pid)
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.py
index 268e10c17b8..3b34b575152 100644
--- a/studio/backend/core/systemone/laya_runtime.py
+++ b/studio/backend/core/systemone/laya_runtime.py
@@ -2494,15 +2494,17 @@ def _reload_after_training(checkpoint: Checkpoint, unloads: int) -> None:
def shutdown() -> None:
- """Studio shutdown: end a llama.cpp decision server or Laya worker (in-process models go with the process)."""
+ """Studio shutdown: end a llama.cpp decision server, Laya or Clef worker (in-process models go with the process)."""
global _agent, _loaded, _device_name, _generation, _unloads
+ from .clef_runtime import ClefAgent
+
with _state_lock:
timer = _idle_timer
if timer is not None:
timer.cancel()
with _run_lock:
_unloads += 1
- if not _is_native(_loaded) and not isinstance(_agent, LayaWorker):
+ if not _is_native(_loaded) and not isinstance(_agent, (LayaWorker, ClefAgent)):
return
agent, _agent, _loaded, _device_name = _agent, None, None, None
_generation += 1
diff --git a/studio/backend/tests/test_decision_serving.py b/studio/backend/tests/test_decision_serving.py
index 9e8eae9995f..a2f9b685265 100644
--- a/studio/backend/tests/test_decision_serving.py
+++ b/studio/backend/tests/test_decision_serving.py
@@ -795,10 +795,10 @@ def test_a_clef_worker_that_never_reports_ready_is_stopped(monkeypatch):
calls = []
class Process:
- exitcode = None
+ exitcode = pid = None
def __init__(self, **kwargs):
- self.child = kwargs["args"][0]
+ self.child = kwargs["args"][1]
def start(self):
# A live child holds its own end of the pipe, so the parent sees silence, not EOF.
@@ -835,10 +835,10 @@ def test_a_training_run_stops_a_clef_worker_still_loading(monkeypatch):
calls = []
class Process:
- exitcode = None
+ exitcode = pid = None
def __init__(self, **kwargs):
- self.child = kwargs["args"][0]
+ self.child = kwargs["args"][1]
def start(self):
# A live child holds its own end of the pipe, so the parent sees silence, not EOF.
@@ -871,6 +871,114 @@ def test_a_training_run_stops_a_clef_worker_still_loading(monkeypatch):
assert "kill" in calls
+def test_unsloth_shutting_down_stops_a_clef_worker_still_loading(monkeypatch):
+ from core.systemone import clef_runtime
+ from utils import process_lifetime
+
+ calls = []
+
+ class Process:
+ exitcode = pid = None
+
+ def __init__(self, **kwargs):
+ self.child = kwargs["args"][1]
+
+ def start(self):
+ held.append(self.child)
+ self.child.close = lambda: None
+
+ def join(self, timeout = None):
+ calls.append("join")
+
+ def is_alive(self):
+ return "kill" not in calls
+
+ def kill(self):
+ calls.append("kill")
+
+ held = []
+ reads = iter([False])
+ monkeypatch.setattr(process_lifetime, "is_process_shutting_down", lambda: next(reads, True))
+ monkeypatch.setattr(clef_runtime._CTX, "Process", Process)
+ monkeypatch.setattr(clef_runtime, "LOAD_TIMEOUT_S", 600)
+ try:
+ started = time.monotonic()
+ with pytest.raises(clef_runtime.ClefWorkerError, match = "stopped while it loaded"):
+ clef_runtime.ClefAgent("unused")
+ assert time.monotonic() - started < 30
+ finally:
+ for child in held:
+ del child.close
+ child.close()
+ assert "kill" in calls
+
+
+def _running(pid):
+ try:
+ with open(f"/proc/{pid}/stat", encoding = "utf-8") as stat:
+ return stat.read().rsplit(")", 1)[1].split()[0] != "Z"
+ except OSError:
+ return False
+
+
[email protected](not sys.platform.startswith("linux"), reason = "PDEATHSIG is Linux only")
+def test_a_loading_clef_worker_dies_with_unsloth(tmp_path):
+ import signal
+ import subprocess
+
+ # The worker's load never returns, as a long GPU load would: its first import blocks.
+ (tmp_path / "torch").mkdir()
+ (tmp_path / "torch" / "__init__.py").write_text(
+ "import os, time\n"
+ "with open(os.environ['CLEF_WORKER_PID'], 'w') as f:\n"
+ " f.write(str(os.getpid()))\n"
+ "time.sleep(600)\n",
+ encoding = "utf-8",
+ )
+ pid_file = tmp_path / "worker.pid"
+ unsloth = subprocess.Popen(
+ [
+ sys.executable,
+ "-c",
+ "import threading\n"
+ "from core.systemone.clef_runtime import ClefAgent\n"
+ "threading.Thread(target = ClefAgent, args = ('unused',)).start()\n"
+ "threading.Event().wait()\n",
+ ],
+ cwd = Path(laya_runtime.__file__).resolve().parents[2],
+ env = dict(os.environ, PYTHONPATH = str(tmp_path), CLEF_WORKER_PID = str(pid_file)),
+ )
+ worker = None
+ try:
+ deadline = time.monotonic() + 60
+ while worker is None and time.monotonic() < deadline:
+ text = pid_file.read_text(encoding = "utf-8") if pid_file.exists() else ""
+ worker = int(text) if text else None
+ time.sleep(0.1)
+ assert worker is not None, "the Clef worker never started loading"
+ unsloth.kill()
+ unsloth.wait()
+ deadline = time.monotonic() + 15
+ while _running(worker) and time.monotonic() < deadline:
+ time.sleep(0.1)
+ assert not _running(worker)
+ finally:
+ unsloth.kill()
+ unsloth.wait()
+ if worker is not None and _running(worker):
+ os.kill(worker, signal.SIGKILL)
+
+
+def test_shutdown_ends_a_clef_worker(home, client, clef):
+ assert (
+ _put(client, enabled = True, model = _clef_fine_tune(home, "clef_served_1")).status_code == 200
+ )
+ assert _post(client).status_code == 200
+ laya_runtime.shutdown()
+ assert clef.agents[0].closed
+ assert client.get("/api/settings/systemone").json()["loaded_device"] is None
+
+
def test_the_catalog_offers_the_stock_clef_models():
for name, repo in (("clef", "Cloudflare/clef"), ("clef-flash", "Cloudflare/clef-flash")):
checkpoint = catalog.CHECKPOINTS[name]
diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py
index b28be02b76c..bde3423adc3 100644
--- a/studio/backend/tests/test_orchestrator_unload_cancel.py
+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py
@@ -3884,6 +3884,7 @@ def test_every_long_lived_spawner_consults_the_shutdown_latch():
"core/inference/stt_transformers_worker.py",
"core/inference/video_ltx25.py",
"core/rag/embed_llama_server.py",
+ "core/systemone/clef_runtime.py",
"core/systemone/native_worker.py",
"core/systemone/laya_worker.py",
"core/training/token_preview.py",