Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: wait on worker exit instead of fixed sleeps when switching models (#16344)
Daniel Han82e9740ae733
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 5b1e2806090..51a25f27814 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -214,6 +214,10 @@ _DISPATCH_POLL_INTERVAL = 0.5_DISPATCH_STOP_TIMEOUT = 5.0_DISPATCH_IDLE_TIMEOUT = 30.0_CANCELLED_ROWS_GRACE = 5.0+_SHUTDOWN_CANCEL_GRACE_S = 0.5+_SHUTDOWN_JOIN_SLICE_S = 0.05+_SHUTDOWN_READER_IDLE_S = 0.8+_SHUTDOWN_READER_MAX_S = 5.0# Transformers TTS only; generous since _ensure_subprocess_alive already catches dead workers._AUDIO_GENERATION_TIMEOUT = 900.0@@ -446,6 +450,8 @@ class InferenceOrchestrator:routes/inference.py needs minimal changes); all heavy ML work happens in a persistentsubprocess."""+ _replies_read = 0+_load_download_keys: Sequence[str] = ()def __init__(self):@@ -855,24 +861,22 @@ class InferenceOrchestrator:# Already gone: a nonzero status is an unwaited crash and keeps its replay.exitcode = getattr(self._proc, "exitcode", 0) if self._proc is not None else 0self._worker_stopped_deliberately = exitcode == 0+ self._stop_shutdown_drainer()self._proc = Nonereturn Trueself._teardown_going_out()+ if self._drain_event is not None:+ self._drain_event.set()self._cancel_generation()- time.sleep(0.5)-- self._drain_queue()+ self._drain_unread_replies()try:self._cmd_queue.put({"type": "shutdown"})except (OSError, ValueError):self._teardown_not_sent()- try:- self._proc.join(timeout = timeout)- except Exception:- pass+ self._join_worker_draining(timeout + _SHUTDOWN_CANCEL_GRACE_S)if self._proc is not None and self._proc.is_alive():logger.warning("Inference subprocess did not exit gracefully, terminating")@@ -905,6 +909,10 @@ class InferenceOrchestrator:)return False+ # Termination can leave a partial queue frame that blocks even get_nowait().+ if self._proc is not None and self._proc.exitcode == 0:+ self._let_readers_finish()+ self._stop_shutdown_drainer()# Without this flag every model switch replayed a healthy worker's stderr at ERROR.self._worker_stopped_deliberately = Trueself._proc = None@@ -918,6 +926,135 @@ class InferenceOrchestrator:logger.info("Inference subprocess shut down")return True+ def _join_worker_draining(self, timeout: float) -> None:+ """Join while draining replies: a child exits only once its queue feeder flushes."""+ proc = self._proc+ try:+ if getattr(proc, "sentinel", None) is None:+ proc.join(timeout = timeout)+ return+ now = time.monotonic()+ deadline = now + max(timeout, 0.0)+ reader_hard_deadline = now + _SHUTDOWN_READER_MAX_S+ reader_idle_deadline = now + _SHUTDOWN_READER_IDLE_S+ seen = self._replies_read+ while True:+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ break+ proc.join(timeout = min(_SHUTDOWN_JOIN_SLICE_S, remaining))+ now = time.monotonic()+ if self._replies_read != seen:+ seen = self._replies_read+ reader_idle_deadline = now + _SHUTDOWN_READER_IDLE_S+ abandoned = False+ if now >= min(reader_hard_deadline, reader_idle_deadline):+ with self._mailbox_lock:+ for rid, mailbox in tuple(self._direct_mailboxes.items()):+ owner = self._request_cancel_events.get(rid)+ if owner is None:+ continue+ owner.set()+ self._request_cancel_events.pop(rid, None)+ mailbox.put({"type": "gen_done", "request_id": rid})+ mailbox.put(+ {+ "type": "audio_error",+ "request_id": rid,+ "cancelled": True,+ "error": "Audio generation cancelled",+ }+ )+ self._direct_mailboxes.pop(rid, None)+ abandoned = True+ self._drain_unread_replies()+ if abandoned:+ proc.join(timeout = _SHUTDOWN_JOIN_SLICE_S)+ if not proc.is_alive():+ break+ except Exception:+ pass++ def _drain_unread_replies(self) -> None:+ """Route pending replies unless a direct reader owns the queue."""+ with self._mailbox_lock:+ if self._direct_mailboxes:+ return+ resp_queue, worker = self._resp_queue, self._proc+ if resp_queue is None:+ return+ state = getattr(self, "_shutdown_drainer", None)+ if state is not None and state[0] is resp_queue and state[1].is_alive():+ return+ stop = threading.Event()++ def drain():+ try:+ while not stop.is_set():+ with self._mailbox_lock:+ if (+ self._direct_mailboxes+ or self._resp_queue is not resp_queue+ or self._proc is not worker+ ):+ return+ try:+ resp = resp_queue.get(timeout = _SHUTDOWN_JOIN_SLICE_S)+ except queue.Empty:+ continue+ except (EOFError, OSError, ValueError):+ return+ rid = resp.get("request_id") if isinstance(resp, dict) else None+ with self._mailbox_lock:+ if self._resp_queue is not resp_queue or self._proc is not worker:+ return+ mbox = self._mailboxes.get(rid) if rid else None+ if mbox is not None:+ mbox.put(resp)+ finally:+ if stop.is_set():+ reader = getattr(resp_queue, "_reader", None)+ if reader is not None:+ reader.close()++ thread = threading.Thread(target = drain, daemon = True, name = "inference-shutdown-drain")+ self._shutdown_drainer = (resp_queue, thread, stop)+ thread.start()++ def _stop_shutdown_drainer(self) -> None:+ state = getattr(self, "_shutdown_drainer", None)+ if state is None:+ return+ resp_queue, thread, stop = state+ stop.set()+ # A dead producer can leave a partial frame; its parent's writer otherwise prevents EOF.+ writer = getattr(resp_queue, "_writer", None)+ if writer is not None:+ writer.close()+ thread.join(timeout = _SHUTDOWN_JOIN_SLICE_S * 2)+ if not thread.is_alive():+ reader = getattr(resp_queue, "_reader", None)+ if reader is not None:+ reader.close()+ self._shutdown_drainer = None++ def _let_readers_finish(self) -> None:+ """A stream reports a worker crash once the handle swaps, so let it read its tail first."""+ now = time.monotonic()+ hard_deadline = now + _SHUTDOWN_READER_MAX_S+ idle_deadline = now + _SHUTDOWN_READER_IDLE_S+ seen = self._replies_read+ while now < min(hard_deadline, idle_deadline):+ with self._mailbox_lock:+ if not self._mailboxes and not self._direct_mailboxes:+ return+ self._drain_unread_replies()+ time.sleep(0.02)+ now = time.monotonic()+ if self._replies_read != seen:+ seen = self._replies_read+ idle_deadline = now + _SHUTDOWN_READER_IDLE_S+@staticmethoddef _wait_for_worker_vram_settle(since_kill: float,@@ -1130,6 +1267,7 @@ class InferenceOrchestrator:def _observe_response(self, resp, worker):"""Retire ``worker`` if its Metal queue is dead; nothing else reaps it."""+ self._replies_read += 1detail = resp.get("error") or ""if worker is None or not is_metal_queue_dead(detail):return resp@@ -2188,7 +2326,6 @@ class InferenceOrchestrator:worker_shutdown_at = 0.0if self._ensure_subprocess_alive():self._cancel_generation()- time.sleep(0.3)if self._shutdown_subprocess() is False:# Worker survived kill (wedged CUDA syscall); do not spawn a second over its GPU memory.raise RuntimeError(diff --git a/studio/backend/tests/test_orchestrator_shutdown_drain.py b/studio/backend/tests/test_orchestrator_shutdown_drain.pynew file mode 100644index 00000000000..2a8e2ffa36d--- /dev/null+++ b/studio/backend/tests/test_orchestrator_shutdown_drain.py@@ -0,0 +1,101 @@+# SPDX-License-Identifier: AGPL-3.0-only++import ast+import logging+import multiprocessing as mp+from pathlib import Path++from core.inference import orchestrator as orch_mod+from core.inference.orchestrator import InferenceOrchestrator+++def _queued_generate_worker(cmd_queue, resp_queue, cancel_event, drain_event, ready, ran):+ worker_path = Path(orch_mod.__file__).with_name("worker.py")+ tree = ast.parse(worker_path.read_text(encoding = "utf-8"))+ entrypoint = next(+ node+ for node in tree.body+ if isinstance(node, ast.FunctionDef) and node.name == "run_inference_process"+ )+ command_loop = next(node for node in reversed(entrypoint.body) if isinstance(node, ast.While))+ branch = next(+ node+ for node in ast.walk(command_loop)+ if isinstance(node, ast.If) and ast.unparse(node.test) == "cmd_type == 'generate'"+ )+ skip = next(+ node+ for node in tree.body+ if isinstance(node, ast.FunctionDef) and node.name == "_drain_skip_generate"+ )+ generate = ast.parse(+ "def generate(cmd, resp_queue, drain_event, cancel_event, backend):\n pass"+ ).body[0]+ generate.body = [+ ast.For(+ target = ast.Name(id = "_once", ctx = ast.Store()),+ iter = ast.Tuple(elts = [ast.Constant(value = None)], ctx = ast.Load()),+ body = branch.body,+ orelse = [],+ )+ ]++ def dispatch(backend, cmd, replies, cancel):+ ran.set()+ cancel.wait(30)++ namespace = {+ "Any": object,+ "logger": logging.getLogger(__name__),+ "_send_response": lambda replies, response: replies.put(response),+ "_dispatch_generate": dispatch,+ }+ module = ast.fix_missing_locations(ast.Module(body = [skip, generate], type_ignores = []))+ exec(compile(module, str(worker_path), "exec"), namespace)+ ready.set()+ if not cancel_event.wait(30):+ raise RuntimeError("Shutdown never signalled cancellation")+ while True:+ cmd = cmd_queue.get(timeout = 30)+ if cmd["type"] == "shutdown":+ return+ namespace["generate"](cmd, resp_queue, drain_event, cancel_event, None)+++def test_shutdown_skips_queued_generate_before_it_can_clear_cancel(monkeypatch):+ import utils.process_lifetime as lifetime++ ctx = mp.get_context("spawn")+ o = InferenceOrchestrator()+ o._cmd_queue, o._resp_queue = ctx.Queue(), ctx.Queue()+ o._cancel_event, o._drain_event = ctx.Event(), ctx.Event()+ ready, ran = ctx.Event(), ctx.Event()+ proc = ctx.Process(+ target = _queued_generate_worker,+ args = (o._cmd_queue, o._resp_queue, o._cancel_event, o._drain_event, ready, ran),+ daemon = True,+ )+ o._proc = proc+ forced = []++ def terminate(pid, timeout = 5):+ assert pid == proc.pid+ forced.append(pid)+ proc.terminate()+ proc.join(timeout)++ monkeypatch.setattr(lifetime, "terminate_pid", terminate)+ proc.start()+ try:+ o._cmd_queue.put({"type": "generate", "request_id": "queued"})+ assert ready.wait(30)+ assert o._shutdown_subprocess(timeout = 0.3) is True+ assert not forced, "a queued generate cleared cancellation and forced worker termination"+ assert not ran.is_set()+ assert proc.exitcode == 0+ finally:+ if proc.is_alive():+ proc.terminate()+ proc.join(5)+ o._proc = None+ proc.close()diff --git a/studio/backend/tests/test_orchestrator_switch_waits.py b/studio/backend/tests/test_orchestrator_switch_waits.pynew file mode 100644index 00000000000..1dc9295ab40--- /dev/null+++ b/studio/backend/tests/test_orchestrator_switch_waits.py@@ -0,0 +1,419 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Worker teardown on a model switch waits on worker exit, not on fixed sleeps."""++import inspect+import multiprocessing as mp+import os+import queue+import threading+import time++import pytest++import core.inference.orchestrator as orch_mod+from core.inference.orchestrator import InferenceOrchestrator+++def _worker(cmd_queue, resp_queue, cancel_event, unload_s, ignore_cancel_s, tok_bytes):+ while True:+ try:+ cmd = cmd_queue.get(timeout = 0.2)+ except queue.Empty:+ continue+ kind = cmd.get("type")+ if kind == "generate":+ cancel_event.clear()+ rid = cmd.get("request_id")+ started = time.monotonic()+ sent = 0+ while not cancel_event.is_set() or time.monotonic() - started < ignore_cancel_s:+ resp_queue.put({"type": "token", "request_id": rid, "text": "x" * tok_bytes})+ sent += 1+ time.sleep(0.005)+ resp_queue.put({"type": "gen_done", "request_id": rid, "stats": {"tokens": sent}})+ elif kind == "shutdown":+ time.sleep(unload_s)+ resp_queue.put({"type": "shutdown_ack"})+ return+++def _live(+ unload_s = 0.0,+ ignore_cancel_s = 0.0,+ tok_bytes = 8,+ generating = False,+):+ ctx = mp.get_context("spawn")+ o = InferenceOrchestrator()+ o._cmd_queue, o._resp_queue = ctx.Queue(), ctx.Queue()+ o._cancel_event, o._drain_event = ctx.Event(), ctx.Event()+ o._proc = ctx.Process(+ target = _worker,+ args = (o._cmd_queue, o._resp_queue, o._cancel_event, unload_s, ignore_cancel_s, tok_bytes),+ daemon = True,+ )+ o._proc.start()+ if generating:+ o._cmd_queue.put({"type": "generate"})+ o._resp_queue.get(timeout = 30) # streaming has begun+ return o+++class _Recorder:+ def __init__(self, monkeypatch):+ self.terminated = []+ self.sleeps = []+ import utils.process_lifetime as lifetime++ monkeypatch.setattr(+ lifetime, "terminate_pid", lambda pid, timeout = 5: self.terminated.append(pid)+ )+ real_sleep = time.sleep+ main = threading.get_ident()++ def sleep(seconds):+ if threading.get_ident() == main:+ self.sleeps.append(seconds)+ real_sleep(seconds)++ monkeypatch.setattr(orch_mod.time, "sleep", sleep)+++def test_idle_worker_shutdown_has_no_fixed_sleep(monkeypatch):+ rec = _Recorder(monkeypatch)+ o = _live()+ proc = o._proc+ assert o._shutdown_subprocess() is True+ assert not proc.is_alive() and o._proc is None+ assert rec.sleeps == [] and rec.terminated == []+++def test_slow_teardown_is_waited_out_not_terminated(monkeypatch):+ rec = _Recorder(monkeypatch)+ o = _live(unload_s = 1.0)+ proc = o._proc+ started = time.monotonic()+ assert o._shutdown_subprocess() is True+ assert time.monotonic() - started >= 1.0+ assert not proc.is_alive()+ assert rec.terminated == []+++def test_cancelled_generation_finishes_before_exit(monkeypatch):+ rec = _Recorder(monkeypatch)+ o = _live(generating = True)+ proc = o._proc+ assert o._shutdown_subprocess() is True+ assert not proc.is_alive() and rec.terminated == []+++def test_late_cancel_with_full_pipe_still_exits_cleanly(monkeypatch):+ # 0.8 s of ~4 KB tokens overfills a 64 KB pipe: the worker exits only if the parent keeps draining.+ rec = _Recorder(monkeypatch)+ o = _live(ignore_cancel_s = 0.8, tok_bytes = 4096, generating = True)+ proc = o._proc+ started = time.monotonic()+ assert o._shutdown_subprocess(timeout = 5) is True+ assert time.monotonic() - started < 5.0+ assert not proc.is_alive()+ assert rec.terminated == [], "a full reply pipe forced the terminate fallback"+++class _NeverExits:+ pid = 999999+ exitcode = None++ def __init__(self, with_sentinel):+ if with_sentinel:+ self.sentinel = object()+ self.joins = []+ self.alive = True++ def is_alive(self):+ return self.alive++ def join(self, timeout = None):+ self.joins.append(timeout)+ if timeout and self.alive:+ time.sleep(timeout) # module time, unpatched here++ def terminate(self):+ self.alive = False++ def kill(self):+ self.alive = False+++def _fake(proc):+ o = InferenceOrchestrator()+ o._proc = proc+ o._cmd_queue = queue.Queue()+ o._resp_queue = queue.Queue()+ o._cancel_event = threading.Event()+ return o+++def test_condition_never_met_keeps_old_budget_then_escalates(monkeypatch):+ import utils.process_lifetime as lifetime++ terminated = []+ proc = _NeverExits(with_sentinel = True)++ def terminate_pid(pid, timeout = 5):+ terminated.append(pid)+ proc.alive = False++ monkeypatch.setattr(lifetime, "terminate_pid", terminate_pid)+ o = _fake(proc)+ cmds = o._cmd_queue+ started = time.monotonic()+ assert o._shutdown_subprocess(timeout = 0.3) is True+ waited = time.monotonic() - started+ assert 0.3 + orch_mod._SHUTDOWN_CANCEL_GRACE_S - 0.05 <= waited < 2.0, waited+ assert terminated == [proc.pid]+ assert proc.joins and all(t <= orch_mod._SHUTDOWN_JOIN_SLICE_S + 1e-9 for t in proc.joins[:-1])+ assert cmds.get_nowait() == {"type": "shutdown"}+++def test_handle_without_sentinel_gets_one_join_with_the_full_budget(monkeypatch):+ import utils.process_lifetime as lifetime++ monkeypatch.setattr(lifetime, "terminate_pid", lambda pid, timeout = 5: None)+ proc = _NeverExits(with_sentinel = False)+ proc.join = lambda timeout = None: proc.joins.append(timeout)+ o = _fake(proc)+ o._shutdown_subprocess(timeout = 2)+ assert proc.joins[0] == 2 + orch_mod._SHUTDOWN_CANCEL_GRACE_S+++def test_source_has_no_fixed_switch_sleeps():+ shutdown_src = inspect.getsource(InferenceOrchestrator._shutdown_subprocess_locked)+ assert "time.sleep" not in shutdown_src+ load_src = inspect.getsource(InferenceOrchestrator.load_model)+ assert "time.sleep(0.3)" not in load_src++[email protected]("reader", ["direct", "dispatched"])+def test_streaming_reader_keeps_its_tail_across_a_switch(monkeypatch, reader):+ _Recorder(monkeypatch)+ o = _live()+ proc = o._proc+ cancel_event = threading.Event()+ if reader == "direct":+ read_one, drain, release = o._direct_reader("r1", cancel_event)+ else:+ o._start_dispatcher()+ mailbox = orch_mod._WorkerMailbox(proc)+ with o._mailbox_lock:+ o._mailboxes["r1"] = mailbox++ def read_one(timeout):+ try:+ return o._read_mailbox(mailbox, timeout)+ except queue.Empty:+ return None++ def drain():+ o._drain_mailbox(mailbox, timeout = 5.0)++ def release():+ with o._mailbox_lock:+ o._mailboxes.pop("r1", None)++ o._cmd_queue.put({"type": "generate", "request_id": "r1"})+ seen = {"tokens": 0, "error": None, "finished": False}+ stats = {}++ def consume():+ try:+ for item in o._consume_token_stream(+ read_one,+ drain,+ crash_context = "generation",+ request_id = "r1",+ cancel_event = cancel_event,+ stats_holder = stats,+ read_timeout = 2.0,+ ):+ if isinstance(item, orch_mod.GenStreamError):+ seen["error"] = str(item)+ return+ seen["tokens"] += 1+ time.sleep(0.02) # real thread sleep: slower than the worker's 5 ms tokens+ seen["finished"] = True+ finally:+ release()++ reader = threading.Thread(target = consume, daemon = True)+ reader.start()+ deadline = time.monotonic() + 30+ while seen["tokens"] == 0 and time.monotonic() < deadline:+ time.sleep(0.01) # worker spawned and streaming: cancel must not land before it clears+ time.sleep(0.2)+ assert o._shutdown_subprocess() is True+ reader.join(5)+ assert not proc.is_alive()+ assert seen["error"] is None, seen+ assert seen["finished"] and seen["tokens"] > 0, seen+ assert stats["stats"]["tokens"] == seen["tokens"], (stats, seen)+++def test_idle_switch_does_not_wait_for_readers(monkeypatch):+ rec = _Recorder(monkeypatch)+ o = _live()+ assert o._shutdown_subprocess() is True+ assert rec.sleeps == [] and rec.terminated == []+++def test_stalled_reader_is_abandoned_after_the_idle_grace(monkeypatch):+ rec = _Recorder(monkeypatch)+ o = _live()+ with o._mailbox_lock:+ o._mailboxes["gone"] = orch_mod._WorkerMailbox(o._proc)+ waits = []+ real = o._let_readers_finish++ def timed():+ started = time.monotonic()+ real()+ waits.append(time.monotonic() - started)++ o._let_readers_finish = timed+ assert o._shutdown_subprocess() is True+ assert orch_mod._SHUTDOWN_READER_IDLE_S <= waits[0] < orch_mod._SHUTDOWN_READER_MAX_S, waits+ assert rec.terminated == []+++def _send_interrupted_frame(replies, send):+ send.wait()+ replies.put({"type": "diffusion_frame", "request_id": "r1", "image": "x" * 1048576})+ time.sleep(60)+++def _shutdown_interrupted_writer():+ ctx = mp.get_context("spawn")+ o = InferenceOrchestrator()+ o._cmd_queue, o._resp_queue = ctx.Queue(), ctx.Queue()+ o._cancel_event = ctx.Event()+ send = ctx.Event()+ o._proc = ctx.Process(target = _send_interrupted_frame, args = (o._resp_queue, send))+ o._proc.start()+ proc = o._proc+ o._mailboxes["r1"] = orch_mod._WorkerMailbox(proc)+ drain = o._drain_unread_replies+ calls = 0++ def enqueue_after_drain():+ nonlocal calls+ replies = drain()+ calls += 1+ if calls == 2:+ send.set()+ assert o._resp_queue._reader.poll(30)+ time.sleep(orch_mod._SHUTDOWN_CANCEL_GRACE_S + 0.1)+ return replies++ o._drain_unread_replies = enqueue_after_drain+ try:+ assert o._shutdown_subprocess(timeout = 0.01) is True+ assert o._proc is None and not proc.is_alive()+ finally:+ if proc.is_alive():+ proc.kill()+ proc.join(5)+++def test_forced_shutdown_does_not_read_an_interrupted_queue_frame():+ runner = mp.get_context("spawn").Process(target = _shutdown_interrupted_writer)+ runner.start()+ try:+ runner.join(30)+ assert not runner.is_alive(), "shutdown blocked reading the killed writer's partial frame"+ assert runner.exitcode == 0+ finally:+ if runner.is_alive():+ runner.kill()+ runner.join(5)+++def _crash_during_frame(replies, send):+ send.wait()+ replies.put({"type": "diffusion_frame", "request_id": "r1", "image": "x" * 1048576})+ assert replies._reader.poll(30)+ os._exit(1)+++def _shutdown_crashed_writer():+ ctx = mp.get_context("spawn")+ o = InferenceOrchestrator()+ o._cmd_queue, o._resp_queue = ctx.Queue(), ctx.Queue()+ o._cancel_event = ctx.Event()+ send = ctx.Event()+ proc = ctx.Process(target = _crash_during_frame, args = (o._resp_queue, send))+ o._proc = proc+ proc.start()+ o._mailboxes["r1"] = orch_mod._WorkerMailbox(proc)+ drain = o._drain_unread_replies+ calls = 0++ def crash_after_drain():+ nonlocal calls+ replies = drain()+ calls += 1+ if calls == 1:+ send.set()+ proc.join(30)+ assert proc.exitcode == 1+ return replies++ o._drain_unread_replies = crash_after_drain+ try:+ assert o._shutdown_subprocess(timeout = 0.01) is True+ assert o._proc is None and not proc.is_alive()+ finally:+ if proc.is_alive():+ proc.kill()+ proc.join(5)+++def test_worker_crash_mid_frame_cannot_block_the_shutdown_deadline():+ runner = mp.get_context("spawn").Process(target = _shutdown_crashed_writer)+ runner.start()+ try:+ runner.join(40)+ assert not runner.is_alive(), "join loop blocked on the crashed writer's partial frame"+ assert runner.exitcode == 0+ finally:+ if runner.is_alive():+ runner.kill()+ runner.join(5)+++def _speech_then_shutdown(commands, replies):+ replies.put({"type": "speech", "request_id": "r1", "speech": "x" * 1048576})+ replies.put({"type": "gen_done", "request_id": "r1"})+ assert commands.get(timeout = 30) == {"type": "shutdown"}+++def test_paused_direct_reader_cannot_force_termination_of_a_finished_worker(monkeypatch):+ rec = _Recorder(monkeypatch)+ ctx = mp.get_context("spawn")+ o = InferenceOrchestrator()+ o._cmd_queue, o._resp_queue = ctx.Queue(), ctx.Queue()+ o._cancel_event = ctx.Event()+ proc = ctx.Process(target = _speech_then_shutdown, args = (o._cmd_queue, o._resp_queue))+ o._proc = proc+ owner = threading.Event()+ _, _, release = o._direct_reader("r1", owner)+ proc.start()+ try:+ assert o._resp_queue._reader.poll(30)+ assert o._shutdown_subprocess(timeout = 2) is True+ assert proc.exitcode == 0, "the paused reader prevented a clean worker exit"+ assert rec.terminated == []+ assert owner.is_set()+ finally:+ release()+ if proc.is_alive():+ proc.kill()+ proc.join(5)diff --git a/studio/backend/tests/test_shutdown_preserves_live_worker.py b/studio/backend/tests/test_shutdown_preserves_live_worker.pyindex d8c2a62d062..49974d10eef 100644--- a/studio/backend/tests/test_shutdown_preserves_live_worker.py+++ b/studio/backend/tests/test_shutdown_preserves_live_worker.py@@ -2,6 +2,7 @@"""A worker that survives kill must keep its handle, so is_worker_alive() still guards the swap."""import threading+import queueimport pytest@@ -20,6 +21,12 @@ class _FakeProc:def is_alive(self):return self._alive+ @property+ def exitcode(self):+ if self._alive:+ return None+ return 0 if self._dies_on == "join" else -15+def join(self, timeout = None):if self._dies_on == "join":self._alive = False@@ -47,7 +54,7 @@ def _bare_inference():passo._cmd_queue = _Q()- o._resp_queue = _Q()+ o._resp_queue = queue.Queue()o._cancel_event = Noneo._drain_event = Noneo._active_cancel_lock = threading.Lock()