Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): keep recipe workers alive when request threads retire (#11021)

alkinun73eb6fb29d3a
diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.py
index 15a827587cb..ab2850a8109 100644
--- a/studio/backend/core/data_recipe/jobs/manager.py
+++ b/studio/backend/core/data_recipe/jobs/manager.py
@@ -215,8 +215,11 @@ class JobManager:
kwargs = process_kwargs,
daemon = True,
)
- proc.start()
- from utils.process_lifetime import adopt_pid
+ from utils.process_lifetime import adopt_pid, spawn_on_lifetime_thread
+
+ # Linux PDEATHSIG follows the spawning thread. A sync request's pool
+ # thread can retire while this recipe is still generating (#11002).
+ spawn_on_lifetime_thread(proc.start)
adopt_pid(proc.pid)
diff --git a/studio/backend/tests/test_data_recipe_worker_lifetime.py b/studio/backend/tests/test_data_recipe_worker_lifetime.py
new file mode 100644
index 00000000000..9e4f50657b1
--- /dev/null
+++ b/studio/backend/tests/test_data_recipe_worker_lifetime.py
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A recipe outlives the request thread that starts it, but remains cancellable."""
+
+import signal
+import sys
+import threading
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parents[1]
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+from core.data_recipe.jobs import manager as manager_mod
+from utils import process_lifetime
+
+
[email protected](not sys.platform.startswith("linux"), reason = "Linux parent-death signal")
[email protected]("cancel", [False, True], ids = ["complete", "cancel"])
+def test_recipe_survives_its_starting_thread(tmp_path, monkeypatch, cancel):
+ if not process_lifetime._pdeathsig_available():
+ pytest.skip("PR_SET_PDEATHSIG unavailable")
+
+ # Keep the real manager, spawn context and secret-scrubbing/lifetime wrapper.
+ # Only the expensive Data Designer target is replaced with a cooperative worker.
+ module_name = "recipe_lifetime_probe"
+ (tmp_path / f"{module_name}.py").write_text(
+ "def run(event_queue, recipe, run):\n"
+ " event_queue.put('ready')\n"
+ " import time\n"
+ " from pathlib import Path\n"
+ " deadline = time.monotonic() + 30\n"
+ " while not Path(run['release']).exists() and time.monotonic() < deadline:\n"
+ " time.sleep(0.01)\n",
+ encoding = "utf-8",
+ )
+ monkeypatch.syspath_prepend(str(tmp_path))
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ monkeypatch.setattr(
+ manager_mod,
+ "account_process_spec",
+ lambda module, target, env, kwargs: ((module_name, "run", env), kwargs),
+ )
+ # The test consumes the worker's ready event itself. No progress pump is needed.
+ monkeypatch.setattr(manager_mod.JobManager, "_pump_loop", lambda self: None)
+ monkeypatch.setattr(process_lifetime, "adopt_pid", lambda pid: None)
+ manager = manager_mod.JobManager()
+ release = tmp_path / "release"
+ errors = []
+
+ def request_thread():
+ try:
+ manager.start(recipe = {}, run = {"release": str(release)})
+ # Do not end the spawning thread until the child has armed PDEATHSIG.
+ assert manager._mp_q.get(timeout = 10) == "ready"
+ except BaseException as exc:
+ errors.append(exc)
+
+ caller = threading.Thread(target = request_thread)
+ try:
+ caller.start()
+ caller.join(timeout = 15)
+ assert not caller.is_alive(), "job start did not return"
+ assert not errors, errors
+ worker = manager._proc
+ assert worker is not None
+ worker.join(timeout = 0.5)
+ assert worker.is_alive(), f"request thread exit killed the worker: {worker.exitcode}"
+ if cancel:
+ assert manager.cancel(manager.get_current_job_id())
+ else:
+ release.touch()
+ worker.join(timeout = 5)
+ assert not worker.is_alive()
+ assert worker.exitcode == (-signal.SIGTERM if cancel else 0)
+ finally:
+ release.touch()
+ caller.join(timeout = 15)
+ if manager._proc is not None:
+ if manager._proc.is_alive():
+ manager._proc.kill()
+ manager._proc.join(timeout = 5)
+ if manager._mp_q is not None:
+ manager._mp_q.close()
+ manager._mp_q.join_thread()
diff --git a/studio/backend/tests/test_process_lifetime.py b/studio/backend/tests/test_process_lifetime.py
index aff7edb95e2..56e6196e23c 100644
--- a/studio/backend/tests/test_process_lifetime.py
+++ b/studio/backend/tests/test_process_lifetime.py
@@ -337,7 +337,8 @@ def test_terminate_all_skips_recycled_pid(monkeypatch):
@pytest.mark.skipif(not IS_LINUX, reason = "PR_SET_PDEATHSIG is Linux-only")
-def test_bind_kills_multiprocessing_child_on_parent_death(tmp_path):
[email protected]("lifetime_thread", [False, True])
+def test_bind_kills_multiprocessing_child_on_parent_death(tmp_path, lifetime_thread):
# multiprocessing workers can't take a preexec_fn, so the child binds itself
# via bind_current_process_to_parent_lifetime(). Killing the parent must reap
# it (the gap reviewers found in adopt_pid alone).
@@ -345,13 +346,17 @@ def test_bind_kills_multiprocessing_child_on_parent_death(tmp_path):
mid.write_text(
"import sys, time, multiprocessing as mp\n"
f"sys.path.insert(0, {str(_BACKEND)!r})\n"
- "from utils.process_lifetime import bind_current_process_to_parent_lifetime\n"
- "def _child():\n"
+ "from utils.process_lifetime import bind_current_process_to_parent_lifetime, spawn_on_lifetime_thread\n"
+ "def _child(ready):\n"
" bind_current_process_to_parent_lifetime()\n"
+ " ready.set()\n"
" time.sleep(300)\n"
"if __name__ == '__main__':\n"
- " p = mp.get_context('spawn').Process(target = _child, daemon = True)\n"
- " p.start()\n"
+ " ctx = mp.get_context('spawn')\n"
+ " ready = ctx.Event()\n"
+ " p = ctx.Process(target = _child, args = (ready,), daemon = True)\n"
+ f" {'spawn_on_lifetime_thread(p.start)' if lifetime_thread else 'p.start()'}\n"
+ " assert ready.wait(10), 'child did not arm its parent-death signal'\n"
" print(p.pid, flush = True)\n"
" time.sleep(300)\n"
)