Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: wait on worker exit instead of fixed sleeps when switching models (#16344)

Daniel Han82e9740ae733
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 5b1e2806090..51a25f27814 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -214,6 +214,10 @@ _DISPATCH_POLL_INTERVAL = 0.5
_DISPATCH_STOP_TIMEOUT = 5.0
_DISPATCH_IDLE_TIMEOUT = 30.0
_CANCELLED_ROWS_GRACE = 5.0
+_SHUTDOWN_CANCEL_GRACE_S = 0.5
+_SHUTDOWN_JOIN_SLICE_S = 0.05
+_SHUTDOWN_READER_IDLE_S = 0.8
+_SHUTDOWN_READER_MAX_S = 5.0
# Transformers TTS only; generous since _ensure_subprocess_alive already catches dead workers.
_AUDIO_GENERATION_TIMEOUT = 900.0
@@ -446,6 +450,8 @@ class InferenceOrchestrator:
routes/inference.py needs minimal changes); all heavy ML work happens in a persistent
subprocess."""
+ _replies_read = 0
+
_load_download_keys: Sequence[str] = ()
def __init__(self):
@@ -855,24 +861,22 @@ class InferenceOrchestrator:
# Already gone: a nonzero status is an unwaited crash and keeps its replay.
exitcode = getattr(self._proc, "exitcode", 0) if self._proc is not None else 0
self._worker_stopped_deliberately = exitcode == 0
+ self._stop_shutdown_drainer()
self._proc = None
return True
self._teardown_going_out()
+ if self._drain_event is not None:
+ self._drain_event.set()
self._cancel_generation()
- time.sleep(0.5)
-
- self._drain_queue()
+ self._drain_unread_replies()
try:
self._cmd_queue.put({"type": "shutdown"})
except (OSError, ValueError):
self._teardown_not_sent()
- try:
- self._proc.join(timeout = timeout)
- except Exception:
- pass
+ self._join_worker_draining(timeout + _SHUTDOWN_CANCEL_GRACE_S)
if self._proc is not None and self._proc.is_alive():
logger.warning("Inference subprocess did not exit gracefully, terminating")
@@ -905,6 +909,10 @@ class InferenceOrchestrator:
)
return False
+ # Termination can leave a partial queue frame that blocks even get_nowait().
+ if self._proc is not None and self._proc.exitcode == 0:
+ self._let_readers_finish()
+ self._stop_shutdown_drainer()
# Without this flag every model switch replayed a healthy worker's stderr at ERROR.
self._worker_stopped_deliberately = True
self._proc = None
@@ -918,6 +926,135 @@ class InferenceOrchestrator:
logger.info("Inference subprocess shut down")
return True
+ def _join_worker_draining(self, timeout: float) -> None:
+ """Join while draining replies: a child exits only once its queue feeder flushes."""
+ proc = self._proc
+ try:
+ if getattr(proc, "sentinel", None) is None:
+ proc.join(timeout = timeout)
+ return
+ now = time.monotonic()
+ deadline = now + max(timeout, 0.0)
+ reader_hard_deadline = now + _SHUTDOWN_READER_MAX_S
+ reader_idle_deadline = now + _SHUTDOWN_READER_IDLE_S
+ seen = self._replies_read
+ while True:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ break
+ proc.join(timeout = min(_SHUTDOWN_JOIN_SLICE_S, remaining))
+ now = time.monotonic()
+ if self._replies_read != seen:
+ seen = self._replies_read
+ reader_idle_deadline = now + _SHUTDOWN_READER_IDLE_S
+ abandoned = False
+ if now >= min(reader_hard_deadline, reader_idle_deadline):
+ with self._mailbox_lock:
+ for rid, mailbox in tuple(self._direct_mailboxes.items()):
+ owner = self._request_cancel_events.get(rid)
+ if owner is None:
+ continue
+ owner.set()
+ self._request_cancel_events.pop(rid, None)
+ mailbox.put({"type": "gen_done", "request_id": rid})
+ mailbox.put(
+ {
+ "type": "audio_error",
+ "request_id": rid,
+ "cancelled": True,
+ "error": "Audio generation cancelled",
+ }
+ )
+ self._direct_mailboxes.pop(rid, None)
+ abandoned = True
+ self._drain_unread_replies()
+ if abandoned:
+ proc.join(timeout = _SHUTDOWN_JOIN_SLICE_S)
+ if not proc.is_alive():
+ break
+ except Exception:
+ pass
+
+ def _drain_unread_replies(self) -> None:
+ """Route pending replies unless a direct reader owns the queue."""
+ with self._mailbox_lock:
+ if self._direct_mailboxes:
+ return
+ resp_queue, worker = self._resp_queue, self._proc
+ if resp_queue is None:
+ return
+ state = getattr(self, "_shutdown_drainer", None)
+ if state is not None and state[0] is resp_queue and state[1].is_alive():
+ return
+ stop = threading.Event()
+
+ def drain():
+ try:
+ while not stop.is_set():
+ with self._mailbox_lock:
+ if (
+ self._direct_mailboxes
+ or self._resp_queue is not resp_queue
+ or self._proc is not worker
+ ):
+ return
+ try:
+ resp = resp_queue.get(timeout = _SHUTDOWN_JOIN_SLICE_S)
+ except queue.Empty:
+ continue
+ except (EOFError, OSError, ValueError):
+ return
+ rid = resp.get("request_id") if isinstance(resp, dict) else None
+ with self._mailbox_lock:
+ if self._resp_queue is not resp_queue or self._proc is not worker:
+ return
+ mbox = self._mailboxes.get(rid) if rid else None
+ if mbox is not None:
+ mbox.put(resp)
+ finally:
+ if stop.is_set():
+ reader = getattr(resp_queue, "_reader", None)
+ if reader is not None:
+ reader.close()
+
+ thread = threading.Thread(target = drain, daemon = True, name = "inference-shutdown-drain")
+ self._shutdown_drainer = (resp_queue, thread, stop)
+ thread.start()
+
+ def _stop_shutdown_drainer(self) -> None:
+ state = getattr(self, "_shutdown_drainer", None)
+ if state is None:
+ return
+ resp_queue, thread, stop = state
+ stop.set()
+ # A dead producer can leave a partial frame; its parent's writer otherwise prevents EOF.
+ writer = getattr(resp_queue, "_writer", None)
+ if writer is not None:
+ writer.close()
+ thread.join(timeout = _SHUTDOWN_JOIN_SLICE_S * 2)
+ if not thread.is_alive():
+ reader = getattr(resp_queue, "_reader", None)
+ if reader is not None:
+ reader.close()
+ self._shutdown_drainer = None
+
+ def _let_readers_finish(self) -> None:
+ """A stream reports a worker crash once the handle swaps, so let it read its tail first."""
+ now = time.monotonic()
+ hard_deadline = now + _SHUTDOWN_READER_MAX_S
+ idle_deadline = now + _SHUTDOWN_READER_IDLE_S
+ seen = self._replies_read
+ while now < min(hard_deadline, idle_deadline):
+ with self._mailbox_lock:
+ if not self._mailboxes and not self._direct_mailboxes:
+ return
+ self._drain_unread_replies()
+ time.sleep(0.02)
+ now = time.monotonic()
+ if self._replies_read != seen:
+ seen = self._replies_read
+ idle_deadline = now + _SHUTDOWN_READER_IDLE_S
+
@staticmethod
def _wait_for_worker_vram_settle(
since_kill: float,
@@ -1130,6 +1267,7 @@ class InferenceOrchestrator:
def _observe_response(self, resp, worker):
"""Retire ``worker`` if its Metal queue is dead; nothing else reaps it."""
+ self._replies_read += 1
detail = resp.get("error") or ""
if worker is None or not is_metal_queue_dead(detail):
return resp
@@ -2188,7 +2326,6 @@ class InferenceOrchestrator:
worker_shutdown_at = 0.0
if self._ensure_subprocess_alive():
self._cancel_generation()
- time.sleep(0.3)
if self._shutdown_subprocess() is False:
# Worker survived kill (wedged CUDA syscall); do not spawn a second over its GPU memory.
raise RuntimeError(
diff --git a/studio/backend/tests/test_orchestrator_shutdown_drain.py b/studio/backend/tests/test_orchestrator_shutdown_drain.py
new file mode 100644
index 00000000000..2a8e2ffa36d
--- /dev/null
+++ b/studio/backend/tests/test_orchestrator_shutdown_drain.py
@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+
+import ast
+import logging
+import multiprocessing as mp
+from pathlib import Path
+
+from core.inference import orchestrator as orch_mod
+from core.inference.orchestrator import InferenceOrchestrator
+
+
+def _queued_generate_worker(cmd_queue, resp_queue, cancel_event, drain_event, ready, ran):
+ worker_path = Path(orch_mod.__file__).with_name("worker.py")
+ tree = ast.parse(worker_path.read_text(encoding = "utf-8"))
+ entrypoint = next(
+ node
+ for node in tree.body
+ if isinstance(node, ast.FunctionDef) and node.name == "run_inference_process"
+ )
+ command_loop = next(node for node in reversed(entrypoint.body) if isinstance(node, ast.While))
+ branch = next(
+ node
+ for node in ast.walk(command_loop)
+ if isinstance(node, ast.If) and ast.unparse(node.test) == "cmd_type == 'generate'"
+ )
+ skip = next(
+ node
+ for node in tree.body
+ if isinstance(node, ast.FunctionDef) and node.name == "_drain_skip_generate"
+ )
+ generate = ast.parse(
+ "def generate(cmd, resp_queue, drain_event, cancel_event, backend):\n pass"
+ ).body[0]
+ generate.body = [
+ ast.For(
+ target = ast.Name(id = "_once", ctx = ast.Store()),
+ iter = ast.Tuple(elts = [ast.Constant(value = None)], ctx = ast.Load()),
+ body = branch.body,
+ orelse = [],
+ )
+ ]
+
+ def dispatch(backend, cmd, replies, cancel):
+ ran.set()
+ cancel.wait(30)
+
+ namespace = {
+ "Any": object,
+ "logger": logging.getLogger(__name__),
+ "_send_response": lambda replies, response: replies.put(response),
+ "_dispatch_generate": dispatch,
+ }
+ module = ast.fix_missing_locations(ast.Module(body = [skip, generate], type_ignores = []))
+ exec(compile(module, str(worker_path), "exec"), namespace)
+ ready.set()
+ if not cancel_event.wait(30):
+ raise RuntimeError("Shutdown never signalled cancellation")
+ while True:
+ cmd = cmd_queue.get(timeout = 30)
+ if cmd["type"] == "shutdown":
+ return
+ namespace["generate"](cmd, resp_queue, drain_event, cancel_event, None)
+
+
+def test_shutdown_skips_queued_generate_before_it_can_clear_cancel(monkeypatch):
+ import utils.process_lifetime as lifetime
+
+ ctx = mp.get_context("spawn")
+ o = InferenceOrchestrator()
+ o._cmd_queue, o._resp_queue = ctx.Queue(), ctx.Queue()
+ o._cancel_event, o._drain_event = ctx.Event(), ctx.Event()
+ ready, ran = ctx.Event(), ctx.Event()
+ proc = ctx.Process(
+ target = _queued_generate_worker,
+ args = (o._cmd_queue, o._resp_queue, o._cancel_event, o._drain_event, ready, ran),
+ daemon = True,
+ )
+ o._proc = proc
+ forced = []
+
+ def terminate(pid, timeout = 5):
+ assert pid == proc.pid
+ forced.append(pid)
+ proc.terminate()
+ proc.join(timeout)
+
+ monkeypatch.setattr(lifetime, "terminate_pid", terminate)
+ proc.start()
+ try:
+ o._cmd_queue.put({"type": "generate", "request_id": "queued"})
+ assert ready.wait(30)
+ assert o._shutdown_subprocess(timeout = 0.3) is True
+ assert not forced, "a queued generate cleared cancellation and forced worker termination"
+ assert not ran.is_set()
+ assert proc.exitcode == 0
+ finally:
+ if proc.is_alive():
+ proc.terminate()
+ proc.join(5)
+ o._proc = None
+ proc.close()
diff --git a/studio/backend/tests/test_orchestrator_switch_waits.py b/studio/backend/tests/test_orchestrator_switch_waits.py
new file mode 100644
index 00000000000..1dc9295ab40
--- /dev/null
+++ b/studio/backend/tests/test_orchestrator_switch_waits.py
@@ -0,0 +1,419 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Worker teardown on a model switch waits on worker exit, not on fixed sleeps."""
+
+import inspect
+import multiprocessing as mp
+import os
+import queue
+import threading
+import time
+
+import pytest
+
+import core.inference.orchestrator as orch_mod
+from core.inference.orchestrator import InferenceOrchestrator
+
+
+def _worker(cmd_queue, resp_queue, cancel_event, unload_s, ignore_cancel_s, tok_bytes):
+ while True:
+ try:
+ cmd = cmd_queue.get(timeout = 0.2)
+ except queue.Empty:
+ continue
+ kind = cmd.get("type")
+ if kind == "generate":
+ cancel_event.clear()
+ rid = cmd.get("request_id")
+ started = time.monotonic()
+ sent = 0
+ while not cancel_event.is_set() or time.monotonic() - started < ignore_cancel_s:
+ resp_queue.put({"type": "token", "request_id": rid, "text": "x" * tok_bytes})
+ sent += 1
+ time.sleep(0.005)
+ resp_queue.put({"type": "gen_done", "request_id": rid, "stats": {"tokens": sent}})
+ elif kind == "shutdown":
+ time.sleep(unload_s)
+ resp_queue.put({"type": "shutdown_ack"})
+ return
+
+
+def _live(
+ unload_s = 0.0,
+ ignore_cancel_s = 0.0,
+ tok_bytes = 8,
+ generating = False,
+):
+ ctx = mp.get_context("spawn")
+ o = InferenceOrchestrator()
+ o._cmd_queue, o._resp_queue = ctx.Queue(), ctx.Queue()
+ o._cancel_event, o._drain_event = ctx.Event(), ctx.Event()
+ o._proc = ctx.Process(
+ target = _worker,
+ args = (o._cmd_queue, o._resp_queue, o._cancel_event, unload_s, ignore_cancel_s, tok_bytes),
+ daemon = True,
+ )
+ o._proc.start()
+ if generating:
+ o._cmd_queue.put({"type": "generate"})
+ o._resp_queue.get(timeout = 30) # streaming has begun
+ return o
+
+
+class _Recorder:
+ def __init__(self, monkeypatch):
+ self.terminated = []
+ self.sleeps = []
+ import utils.process_lifetime as lifetime
+
+ monkeypatch.setattr(
+ lifetime, "terminate_pid", lambda pid, timeout = 5: self.terminated.append(pid)
+ )
+ real_sleep = time.sleep
+ main = threading.get_ident()
+
+ def sleep(seconds):
+ if threading.get_ident() == main:
+ self.sleeps.append(seconds)
+ real_sleep(seconds)
+
+ monkeypatch.setattr(orch_mod.time, "sleep", sleep)
+
+
+def test_idle_worker_shutdown_has_no_fixed_sleep(monkeypatch):
+ rec = _Recorder(monkeypatch)
+ o = _live()
+ proc = o._proc
+ assert o._shutdown_subprocess() is True
+ assert not proc.is_alive() and o._proc is None
+ assert rec.sleeps == [] and rec.terminated == []
+
+
+def test_slow_teardown_is_waited_out_not_terminated(monkeypatch):
+ rec = _Recorder(monkeypatch)
+ o = _live(unload_s = 1.0)
+ proc = o._proc
+ started = time.monotonic()
+ assert o._shutdown_subprocess() is True
+ assert time.monotonic() - started >= 1.0
+ assert not proc.is_alive()
+ assert rec.terminated == []
+
+
+def test_cancelled_generation_finishes_before_exit(monkeypatch):
+ rec = _Recorder(monkeypatch)
+ o = _live(generating = True)
+ proc = o._proc
+ assert o._shutdown_subprocess() is True
+ assert not proc.is_alive() and rec.terminated == []
+
+
+def test_late_cancel_with_full_pipe_still_exits_cleanly(monkeypatch):
+ # 0.8 s of ~4 KB tokens overfills a 64 KB pipe: the worker exits only if the parent keeps draining.
+ rec = _Recorder(monkeypatch)
+ o = _live(ignore_cancel_s = 0.8, tok_bytes = 4096, generating = True)
+ proc = o._proc
+ started = time.monotonic()
+ assert o._shutdown_subprocess(timeout = 5) is True
+ assert time.monotonic() - started < 5.0
+ assert not proc.is_alive()
+ assert rec.terminated == [], "a full reply pipe forced the terminate fallback"
+
+
+class _NeverExits:
+ pid = 999999
+ exitcode = None
+
+ def __init__(self, with_sentinel):
+ if with_sentinel:
+ self.sentinel = object()
+ self.joins = []
+ self.alive = True
+
+ def is_alive(self):
+ return self.alive
+
+ def join(self, timeout = None):
+ self.joins.append(timeout)
+ if timeout and self.alive:
+ time.sleep(timeout) # module time, unpatched here
+
+ def terminate(self):
+ self.alive = False
+
+ def kill(self):
+ self.alive = False
+
+
+def _fake(proc):
+ o = InferenceOrchestrator()
+ o._proc = proc
+ o._cmd_queue = queue.Queue()
+ o._resp_queue = queue.Queue()
+ o._cancel_event = threading.Event()
+ return o
+
+
+def test_condition_never_met_keeps_old_budget_then_escalates(monkeypatch):
+ import utils.process_lifetime as lifetime
+
+ terminated = []
+ proc = _NeverExits(with_sentinel = True)
+
+ def terminate_pid(pid, timeout = 5):
+ terminated.append(pid)
+ proc.alive = False
+
+ monkeypatch.setattr(lifetime, "terminate_pid", terminate_pid)
+ o = _fake(proc)
+ cmds = o._cmd_queue
+ started = time.monotonic()
+ assert o._shutdown_subprocess(timeout = 0.3) is True
+ waited = time.monotonic() - started
+ assert 0.3 + orch_mod._SHUTDOWN_CANCEL_GRACE_S - 0.05 <= waited < 2.0, waited
+ assert terminated == [proc.pid]
+ assert proc.joins and all(t <= orch_mod._SHUTDOWN_JOIN_SLICE_S + 1e-9 for t in proc.joins[:-1])
+ assert cmds.get_nowait() == {"type": "shutdown"}
+
+
+def test_handle_without_sentinel_gets_one_join_with_the_full_budget(monkeypatch):
+ import utils.process_lifetime as lifetime
+
+ monkeypatch.setattr(lifetime, "terminate_pid", lambda pid, timeout = 5: None)
+ proc = _NeverExits(with_sentinel = False)
+ proc.join = lambda timeout = None: proc.joins.append(timeout)
+ o = _fake(proc)
+ o._shutdown_subprocess(timeout = 2)
+ assert proc.joins[0] == 2 + orch_mod._SHUTDOWN_CANCEL_GRACE_S
+
+
+def test_source_has_no_fixed_switch_sleeps():
+ shutdown_src = inspect.getsource(InferenceOrchestrator._shutdown_subprocess_locked)
+ assert "time.sleep" not in shutdown_src
+ load_src = inspect.getsource(InferenceOrchestrator.load_model)
+ assert "time.sleep(0.3)" not in load_src
+
+
[email protected]("reader", ["direct", "dispatched"])
+def test_streaming_reader_keeps_its_tail_across_a_switch(monkeypatch, reader):
+ _Recorder(monkeypatch)
+ o = _live()
+ proc = o._proc
+ cancel_event = threading.Event()
+ if reader == "direct":
+ read_one, drain, release = o._direct_reader("r1", cancel_event)
+ else:
+ o._start_dispatcher()
+ mailbox = orch_mod._WorkerMailbox(proc)
+ with o._mailbox_lock:
+ o._mailboxes["r1"] = mailbox
+
+ def read_one(timeout):
+ try:
+ return o._read_mailbox(mailbox, timeout)
+ except queue.Empty:
+ return None
+
+ def drain():
+ o._drain_mailbox(mailbox, timeout = 5.0)
+
+ def release():
+ with o._mailbox_lock:
+ o._mailboxes.pop("r1", None)
+
+ o._cmd_queue.put({"type": "generate", "request_id": "r1"})
+ seen = {"tokens": 0, "error": None, "finished": False}
+ stats = {}
+
+ def consume():
+ try:
+ for item in o._consume_token_stream(
+ read_one,
+ drain,
+ crash_context = "generation",
+ request_id = "r1",
+ cancel_event = cancel_event,
+ stats_holder = stats,
+ read_timeout = 2.0,
+ ):
+ if isinstance(item, orch_mod.GenStreamError):
+ seen["error"] = str(item)
+ return
+ seen["tokens"] += 1
+ time.sleep(0.02) # real thread sleep: slower than the worker's 5 ms tokens
+ seen["finished"] = True
+ finally:
+ release()
+
+ reader = threading.Thread(target = consume, daemon = True)
+ reader.start()
+ deadline = time.monotonic() + 30
+ while seen["tokens"] == 0 and time.monotonic() < deadline:
+ time.sleep(0.01) # worker spawned and streaming: cancel must not land before it clears
+ time.sleep(0.2)
+ assert o._shutdown_subprocess() is True
+ reader.join(5)
+ assert not proc.is_alive()
+ assert seen["error"] is None, seen
+ assert seen["finished"] and seen["tokens"] > 0, seen
+ assert stats["stats"]["tokens"] == seen["tokens"], (stats, seen)
+
+
+def test_idle_switch_does_not_wait_for_readers(monkeypatch):
+ rec = _Recorder(monkeypatch)
+ o = _live()
+ assert o._shutdown_subprocess() is True
+ assert rec.sleeps == [] and rec.terminated == []
+
+
+def test_stalled_reader_is_abandoned_after_the_idle_grace(monkeypatch):
+ rec = _Recorder(monkeypatch)
+ o = _live()
+ with o._mailbox_lock:
+ o._mailboxes["gone"] = orch_mod._WorkerMailbox(o._proc)
+ waits = []
+ real = o._let_readers_finish
+
+ def timed():
+ started = time.monotonic()
+ real()
+ waits.append(time.monotonic() - started)
+
+ o._let_readers_finish = timed
+ assert o._shutdown_subprocess() is True
+ assert orch_mod._SHUTDOWN_READER_IDLE_S <= waits[0] < orch_mod._SHUTDOWN_READER_MAX_S, waits
+ assert rec.terminated == []
+
+
+def _send_interrupted_frame(replies, send):
+ send.wait()
+ replies.put({"type": "diffusion_frame", "request_id": "r1", "image": "x" * 1048576})
+ time.sleep(60)
+
+
+def _shutdown_interrupted_writer():
+ ctx = mp.get_context("spawn")
+ o = InferenceOrchestrator()
+ o._cmd_queue, o._resp_queue = ctx.Queue(), ctx.Queue()
+ o._cancel_event = ctx.Event()
+ send = ctx.Event()
+ o._proc = ctx.Process(target = _send_interrupted_frame, args = (o._resp_queue, send))
+ o._proc.start()
+ proc = o._proc
+ o._mailboxes["r1"] = orch_mod._WorkerMailbox(proc)
+ drain = o._drain_unread_replies
+ calls = 0
+
+ def enqueue_after_drain():
+ nonlocal calls
+ replies = drain()
+ calls += 1
+ if calls == 2:
+ send.set()
+ assert o._resp_queue._reader.poll(30)
+ time.sleep(orch_mod._SHUTDOWN_CANCEL_GRACE_S + 0.1)
+ return replies
+
+ o._drain_unread_replies = enqueue_after_drain
+ try:
+ assert o._shutdown_subprocess(timeout = 0.01) is True
+ assert o._proc is None and not proc.is_alive()
+ finally:
+ if proc.is_alive():
+ proc.kill()
+ proc.join(5)
+
+
+def test_forced_shutdown_does_not_read_an_interrupted_queue_frame():
+ runner = mp.get_context("spawn").Process(target = _shutdown_interrupted_writer)
+ runner.start()
+ try:
+ runner.join(30)
+ assert not runner.is_alive(), "shutdown blocked reading the killed writer's partial frame"
+ assert runner.exitcode == 0
+ finally:
+ if runner.is_alive():
+ runner.kill()
+ runner.join(5)
+
+
+def _crash_during_frame(replies, send):
+ send.wait()
+ replies.put({"type": "diffusion_frame", "request_id": "r1", "image": "x" * 1048576})
+ assert replies._reader.poll(30)
+ os._exit(1)
+
+
+def _shutdown_crashed_writer():
+ ctx = mp.get_context("spawn")
+ o = InferenceOrchestrator()
+ o._cmd_queue, o._resp_queue = ctx.Queue(), ctx.Queue()
+ o._cancel_event = ctx.Event()
+ send = ctx.Event()
+ proc = ctx.Process(target = _crash_during_frame, args = (o._resp_queue, send))
+ o._proc = proc
+ proc.start()
+ o._mailboxes["r1"] = orch_mod._WorkerMailbox(proc)
+ drain = o._drain_unread_replies
+ calls = 0
+
+ def crash_after_drain():
+ nonlocal calls
+ replies = drain()
+ calls += 1
+ if calls == 1:
+ send.set()
+ proc.join(30)
+ assert proc.exitcode == 1
+ return replies
+
+ o._drain_unread_replies = crash_after_drain
+ try:
+ assert o._shutdown_subprocess(timeout = 0.01) is True
+ assert o._proc is None and not proc.is_alive()
+ finally:
+ if proc.is_alive():
+ proc.kill()
+ proc.join(5)
+
+
+def test_worker_crash_mid_frame_cannot_block_the_shutdown_deadline():
+ runner = mp.get_context("spawn").Process(target = _shutdown_crashed_writer)
+ runner.start()
+ try:
+ runner.join(40)
+ assert not runner.is_alive(), "join loop blocked on the crashed writer's partial frame"
+ assert runner.exitcode == 0
+ finally:
+ if runner.is_alive():
+ runner.kill()
+ runner.join(5)
+
+
+def _speech_then_shutdown(commands, replies):
+ replies.put({"type": "speech", "request_id": "r1", "speech": "x" * 1048576})
+ replies.put({"type": "gen_done", "request_id": "r1"})
+ assert commands.get(timeout = 30) == {"type": "shutdown"}
+
+
+def test_paused_direct_reader_cannot_force_termination_of_a_finished_worker(monkeypatch):
+ rec = _Recorder(monkeypatch)
+ ctx = mp.get_context("spawn")
+ o = InferenceOrchestrator()
+ o._cmd_queue, o._resp_queue = ctx.Queue(), ctx.Queue()
+ o._cancel_event = ctx.Event()
+ proc = ctx.Process(target = _speech_then_shutdown, args = (o._cmd_queue, o._resp_queue))
+ o._proc = proc
+ owner = threading.Event()
+ _, _, release = o._direct_reader("r1", owner)
+ proc.start()
+ try:
+ assert o._resp_queue._reader.poll(30)
+ assert o._shutdown_subprocess(timeout = 2) is True
+ assert proc.exitcode == 0, "the paused reader prevented a clean worker exit"
+ assert rec.terminated == []
+ assert owner.is_set()
+ finally:
+ release()
+ if proc.is_alive():
+ proc.kill()
+ proc.join(5)
diff --git a/studio/backend/tests/test_shutdown_preserves_live_worker.py b/studio/backend/tests/test_shutdown_preserves_live_worker.py
index d8c2a62d062..49974d10eef 100644
--- a/studio/backend/tests/test_shutdown_preserves_live_worker.py
+++ b/studio/backend/tests/test_shutdown_preserves_live_worker.py
@@ -2,6 +2,7 @@
"""A worker that survives kill must keep its handle, so is_worker_alive() still guards the swap."""
import threading
+import queue
import pytest
@@ -20,6 +21,12 @@ class _FakeProc:
def is_alive(self):
return self._alive
+ @property
+ def exitcode(self):
+ if self._alive:
+ return None
+ return 0 if self._dies_on == "join" else -15
+
def join(self, timeout = None):
if self._dies_on == "join":
self._alive = False
@@ -47,7 +54,7 @@ def _bare_inference():
pass
o._cmd_queue = _Q()
- o._resp_queue = _Q()
+ o._resp_queue = queue.Queue()
o._cancel_event = None
o._drain_event = None
o._active_cancel_lock = threading.Lock()