Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): keep recipe workers alive when request threads retire (#11021)
alkinun73eb6fb29d3a
diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.pyindex 15a827587cb..ab2850a8109 100644--- a/studio/backend/core/data_recipe/jobs/manager.py+++ b/studio/backend/core/data_recipe/jobs/manager.py@@ -215,8 +215,11 @@ class JobManager:kwargs = process_kwargs,daemon = True,)- proc.start()- from utils.process_lifetime import adopt_pid+ from utils.process_lifetime import adopt_pid, spawn_on_lifetime_thread++ # Linux PDEATHSIG follows the spawning thread. A sync request's pool+ # thread can retire while this recipe is still generating (#11002).+ spawn_on_lifetime_thread(proc.start)adopt_pid(proc.pid)diff --git a/studio/backend/tests/test_data_recipe_worker_lifetime.py b/studio/backend/tests/test_data_recipe_worker_lifetime.pynew file mode 100644index 00000000000..9e4f50657b1--- /dev/null+++ b/studio/backend/tests/test_data_recipe_worker_lifetime.py@@ -0,0 +1,88 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A recipe outlives the request thread that starts it, but remains cancellable."""++import signal+import sys+import threading+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parents[1]+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++from core.data_recipe.jobs import manager as manager_mod+from utils import process_lifetime++[email protected](not sys.platform.startswith("linux"), reason = "Linux parent-death signal")[email protected]("cancel", [False, True], ids = ["complete", "cancel"])+def test_recipe_survives_its_starting_thread(tmp_path, monkeypatch, cancel):+ if not process_lifetime._pdeathsig_available():+ pytest.skip("PR_SET_PDEATHSIG unavailable")++ # Keep the real manager, spawn context and secret-scrubbing/lifetime wrapper.+ # Only the expensive Data Designer target is replaced with a cooperative worker.+ module_name = "recipe_lifetime_probe"+ (tmp_path / f"{module_name}.py").write_text(+ "def run(event_queue, recipe, run):\n"+ " event_queue.put('ready')\n"+ " import time\n"+ " from pathlib import Path\n"+ " deadline = time.monotonic() + 30\n"+ " while not Path(run['release']).exists() and time.monotonic() < deadline:\n"+ " time.sleep(0.01)\n",+ encoding = "utf-8",+ )+ monkeypatch.syspath_prepend(str(tmp_path))+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ monkeypatch.setattr(+ manager_mod,+ "account_process_spec",+ lambda module, target, env, kwargs: ((module_name, "run", env), kwargs),+ )+ # The test consumes the worker's ready event itself. No progress pump is needed.+ monkeypatch.setattr(manager_mod.JobManager, "_pump_loop", lambda self: None)+ monkeypatch.setattr(process_lifetime, "adopt_pid", lambda pid: None)+ manager = manager_mod.JobManager()+ release = tmp_path / "release"+ errors = []++ def request_thread():+ try:+ manager.start(recipe = {}, run = {"release": str(release)})+ # Do not end the spawning thread until the child has armed PDEATHSIG.+ assert manager._mp_q.get(timeout = 10) == "ready"+ except BaseException as exc:+ errors.append(exc)++ caller = threading.Thread(target = request_thread)+ try:+ caller.start()+ caller.join(timeout = 15)+ assert not caller.is_alive(), "job start did not return"+ assert not errors, errors+ worker = manager._proc+ assert worker is not None+ worker.join(timeout = 0.5)+ assert worker.is_alive(), f"request thread exit killed the worker: {worker.exitcode}"+ if cancel:+ assert manager.cancel(manager.get_current_job_id())+ else:+ release.touch()+ worker.join(timeout = 5)+ assert not worker.is_alive()+ assert worker.exitcode == (-signal.SIGTERM if cancel else 0)+ finally:+ release.touch()+ caller.join(timeout = 15)+ if manager._proc is not None:+ if manager._proc.is_alive():+ manager._proc.kill()+ manager._proc.join(timeout = 5)+ if manager._mp_q is not None:+ manager._mp_q.close()+ manager._mp_q.join_thread()diff --git a/studio/backend/tests/test_process_lifetime.py b/studio/backend/tests/test_process_lifetime.pyindex aff7edb95e2..56e6196e23c 100644--- a/studio/backend/tests/test_process_lifetime.py+++ b/studio/backend/tests/test_process_lifetime.py@@ -337,7 +337,8 @@ def test_terminate_all_skips_recycled_pid(monkeypatch):@pytest.mark.skipif(not IS_LINUX, reason = "PR_SET_PDEATHSIG is Linux-only")-def test_bind_kills_multiprocessing_child_on_parent_death(tmp_path):[email protected]("lifetime_thread", [False, True])+def test_bind_kills_multiprocessing_child_on_parent_death(tmp_path, lifetime_thread):# multiprocessing workers can't take a preexec_fn, so the child binds itself# via bind_current_process_to_parent_lifetime(). Killing the parent must reap# it (the gap reviewers found in adopt_pid alone).@@ -345,13 +346,17 @@ def test_bind_kills_multiprocessing_child_on_parent_death(tmp_path):mid.write_text("import sys, time, multiprocessing as mp\n"f"sys.path.insert(0, {str(_BACKEND)!r})\n"- "from utils.process_lifetime import bind_current_process_to_parent_lifetime\n"- "def _child():\n"+ "from utils.process_lifetime import bind_current_process_to_parent_lifetime, spawn_on_lifetime_thread\n"+ "def _child(ready):\n"" bind_current_process_to_parent_lifetime()\n"+ " ready.set()\n"" time.sleep(300)\n""if __name__ == '__main__':\n"- " p = mp.get_context('spawn').Process(target = _child, daemon = True)\n"- " p.start()\n"+ " ctx = mp.get_context('spawn')\n"+ " ready = ctx.Event()\n"+ " p = ctx.Process(target = _child, args = (ready,), daemon = True)\n"+ f" {'spawn_on_lifetime_thread(p.start)' if lifetime_thread else 'p.start()'}\n"+ " assert ready.wait(10), 'child did not arm its parent-death signal'\n"" print(p.pid, flush = True)\n"" time.sleep(300)\n")