Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop a quit during a model load from orphaning llama-server (#10592)

Daniel Hanfcaad20ef248
diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.py
index da8b6314c2d..7ce9ef409e0 100644
--- a/studio/backend/core/export/orchestrator.py
+++ b/studio/backend/core/export/orchestrator.py
@@ -230,9 +230,16 @@ class ExportOrchestrator:
run_without_native_path_secret,
)
from utils.hf_cache_settings import child_environment_for_spawn, get_hf_cache_paths
+ from utils.process_lifetime import is_process_shutting_down
cache_env = get_hf_cache_paths().child_env({})
+ # An export admitted before the quit can still reach this line after the shutdown
+ # sweep has taken its snapshot, and the worker adopted below would then outlive
+ # Studio holding the model in memory.
+ if is_process_shutting_down():
+ raise RuntimeError("Studio is shutting down; not starting an export subprocess")
+
with (
child_environment_for_spawn(cache_env),
native_path_secret_removed_for_child_start(),
@@ -240,7 +247,11 @@ class ExportOrchestrator:
self._cmd_queue = _CTX.Queue()
self._resp_queue = _CTX.Queue()
- self._proc = _CTX.Process(
+ # Kept in a local as well as on self: a concurrent _shutdown_subprocess can
+ # see a process that has not finished starting, decide it is not alive and
+ # clear self._proc, and every read below would then be off a None while the
+ # worker is alive and unadopted.
+ _spawned_proc = _CTX.Process(
target = run_without_native_path_secret,
args = ("core.export.worker", "run_export_process", cache_env),
kwargs = {
@@ -250,11 +261,39 @@ class ExportOrchestrator:
},
daemon = True,
)
- self._proc.start()
- from utils.process_lifetime import adopt_pid
-
- adopt_pid(self._proc.pid)
- logger.info("Export subprocess started (pid=%s)", self._proc.pid)
+ self._proc = _spawned_proc
+ _spawned_proc.start()
+ from utils.process_lifetime import adopt_pid, forget_pid
+
+ adopt_pid(_spawned_proc.pid)
+ # Recheck once the pid is recorded, for the window between the gate above and
+ # this record. Adoption runs first, so a worker torn down here was in the sweep
+ # record for as long as it existed. The handle check catches the other half of
+ # the race: a teardown that already cleared or replaced self._proc leaves this
+ # worker with no owner, so reap it here rather than let it run on.
+ if is_process_shutting_down() or self._proc is not _spawned_proc:
+ logger.info("shutdown began during the spawn; stopping the new export subprocess")
+ if self._proc is _spawned_proc:
+ self._shutdown_subprocess(timeout = 5)
+ else:
+ try:
+ if _spawned_proc.is_alive():
+ _spawned_proc.terminate()
+ _spawned_proc.join(timeout = 5)
+ if _spawned_proc.is_alive():
+ _spawned_proc.kill()
+ _spawned_proc.join(timeout = 3)
+ except Exception: # noqa: BLE001 - the reap is best-effort
+ logger.warning("could not reap the orphaned export worker", exc_info = True)
+ if _spawned_proc.exitcode is not None:
+ forget_pid(_spawned_proc.pid)
+ else:
+ logger.warning(
+ "export worker (pid %s) survived the reap; leaving it adopted",
+ _spawned_proc.pid,
+ )
+ raise RuntimeError("Studio is shutting down; not starting an export subprocess")
+ logger.info("Export subprocess started (pid=%s)", _spawned_proc.pid)
def _shutdown_subprocess(self, timeout: float = 10.0) -> bool:
"""Gracefully shut down the export subprocess.
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 0c96e7271aa..8b65d4a94d2 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -6229,6 +6229,15 @@ class LlamaCppBackend:
3. unload_model(): terminate the subprocess
"""
+ # Held across "is a teardown running?" and publishing the child, never across a
+ # health wait. On the class: doubles built with __new__ never run __init__.
+ _spawn_lock = threading.Lock()
+
+ # Held across a whole teardown, so a lifecycle cannot reopen mid-kill. Separate
+ # from _spawn_lock so that long hold does not also block a spawn, which only
+ # needs to read the flag. Order is always _teardown_lock then _spawn_lock.
+ _teardown_lock = threading.Lock()
+
def __init__(self, *, manages_processes: bool = True):
"""``manages_processes = False`` builds an INERT probe.
@@ -14494,29 +14503,49 @@ class LlamaCppBackend:
# The shim (and its visual server) die with this backend process, so a
# Unsloth crash/restart never orphans a GPU process.
- self._process = subprocess.Popen(
- cmd,
- stdout = subprocess.PIPE,
- stderr = subprocess.STDOUT,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- env = utf8_child_env(env),
- # Deliberately NOT start_new_session, as with the component
- # installer: the desktop stops this backend by signalling its
- # process group and force-kills it after five seconds, so a session
- # of its own would leave the shim and the visual server holding the
- # GPU until the next launch sweeps them.
- **_windows_hidden_subprocess_kwargs(),
- **_child_popen_kwargs(),
- )
- # macOS has no parent-death signal, so the kwargs above are empty there and
- # only this record lets the next startup reap a runner holding the GPU.
- try:
- from utils.process_lifetime import adopt_pid
- adopt_pid(self._process.pid)
- except Exception as e:
- logger.debug(f"Could not track diffusion runner for lifetime sweep: {e}")
+ # Own Popen, and no parent-death backstop on every platform, so a runner
+ # started after the shutdown sweep outlives it.
+ with self._spawn_lock:
+ if self._spawn_is_stale():
+ logger.info("app is shutting down; not starting the diffusion runner")
+ self._close_attempt_log()
+ self._health_wait_cancelled = True
+ return False
+ _spawned = subprocess.Popen(
+ cmd,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ env = utf8_child_env(env),
+ # Deliberately NOT start_new_session, as with the component
+ # installer: the desktop stops this backend by signalling its
+ # process group and force-kills it after five seconds, so a session
+ # of its own would leave the shim and the visual server holding the
+ # GPU until the next launch sweeps them.
+ **_windows_hidden_subprocess_kwargs(),
+ **_child_popen_kwargs(),
+ )
+ self._process = _spawned
+ # macOS has no parent-death signal, so only this record reaps a runner
+ # holding the GPU. Under the lock: adopting after the sweep re-adds a
+ # pid it just forgot.
+ try:
+ from utils.process_lifetime import adopt_pid
+ adopt_pid(_spawned.pid)
+ except Exception as e:
+ logger.debug(f"Could not track diffusion runner for lifetime sweep: {e}")
+ # Same post-adoption recheck as the llama-server spawns: this backend can be
+ # helper-owned, which run.py's singleton teardown never marks, so a latch set
+ # after the in-lock check would otherwise leave the shim and the visual server
+ # alive for the whole health wait below.
+ if self._spawn_is_stale():
+ logger.info("shutdown began during the spawn; killing the new diffusion runner")
+ self._kill_process()
+ self._close_attempt_log()
+ self._health_wait_cancelled = True
+ return False
self._stdout_thread = threading.Thread(
target = self._drain_stdout, daemon = True, name = "diffusion-stdout"
)
@@ -14607,7 +14636,11 @@ class LlamaCppBackend:
healthy = self._wait_for_health(timeout = 600.0, cancelled = cancelled)
if healthy:
- self._healthy = True
+ if not self._publish_healthy():
+ # A teardown between the probe and this commit is already killing
+ # the runner; publishing would advertise a server that is gone.
+ self._kill_process()
+ return False
self._gpu_offload_active = not holds_no_gpu
if extra_args is not None:
self._extra_args = list(extra_args)
@@ -18638,12 +18671,16 @@ class LlamaCppBackend:
def _start_llama_process(
self, cmd: list[str], env: dict, *, child_gpu_physical_ids: Optional[tuple[int, ...]]
- ) -> None:
+ ) -> bool:
"""Spawn llama-server from cmd and start draining its output.
Caller holds self._lock. Resets the stdout buffer, opens a fresh
per-attempt tee log, launches the process, and starts the drain
thread. Used for the initial start and the text-only mmproj retry.
+
+ Returns False without spawning once app teardown has begun. Reported rather
+ than silent so the caller can stop instead of health-waiting on the previous
+ child and then reading a reference the teardown is clearing.
"""
# Defensive kill: if a concurrent load slipped past Phase 1
# (because its `self._process` was None at the time) and already
@@ -18679,26 +18716,48 @@ class LlamaCppBackend:
# with --mmproj stripped), redacting the API key.
logger.info(f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(cmd))}")
- self._process = subprocess.Popen(
- cmd,
- stdout = subprocess.PIPE,
- stderr = subprocess.STDOUT,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- env = env,
- **_windows_hidden_subprocess_kwargs(),
- **_child_popen_kwargs(),
- )
- # Cross-session backstop: record the PID so a later startup can reap this
- # server if parent-death cleanup did not run (macOS / best-effort failure).
- self._record_server_pid(self._process.pid)
+ # Check with publication under one lock: the mmproj text-only retry reaches
+ # a spawn without passing _spawn_and_wait's boundary.
+ with self._spawn_lock:
+ if self._spawn_is_stale():
+ logger.info("app is shutting down; not starting llama-server")
+ self._close_attempt_log()
+ self._health_wait_cancelled = True
+ return False
+ _spawned = subprocess.Popen(
+ cmd,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ env = env,
+ **_windows_hidden_subprocess_kwargs(),
+ **_child_popen_kwargs(),
+ )
+ self._process = _spawned
+ # Cross-session backstop for when parent-death cleanup did not run.
+ # Under the lock: see _spawn_and_wait.
+ self._record_server_pid(_spawned.pid)
+
+ # The stale check above and the process-wide latch are only atomic for the
+ # instance run.py tears down, which sets its own flag under this same lock. A
+ # helper load owns a backend nothing marks, so its check can pass microseconds
+ # before the latch is set and the child then outlives the sweep. Recheck once
+ # the pid is recorded and reap it, as the inference worker spawn does.
+ if self._spawn_is_stale():
+ logger.info("shutdown began during the spawn; killing the new llama-server")
+ self._kill_process()
+ self._close_attempt_log()
+ self._health_wait_cancelled = True
+ return False
# Start background thread to drain stdout and prevent pipe deadlock
self._stdout_thread = threading.Thread(
target = self._drain_stdout, daemon = True, name = "llama-stdout"
)
self._stdout_thread.start()
+ return True
@contextlib.contextmanager
def _serial_load_scope(self):
@@ -18771,9 +18830,22 @@ class LlamaCppBackend:
cache_ram = intent.cache_ram
extra_args = list(intent.extra_args) if intent.extra_args is not None else None
preserve_multi_gpu_on_layer = intent.preserve_multi_gpu_on_layer
+ # Before the serial scope: a queued load still belongs to the lifecycle it
+ # was requested in.
+ # The process-wide equivalent, for the same reason. Only _begin_server_lifecycle
+ # advances the per-instance one, and a helper load owns a backend that never
+ # gets it, so an embedded second session would otherwise release this load.
# Serialise the whole load so concurrent /load calls never leave two
# llama-server processes alive (#5401 / #5161). Doesn't block /unload.
with self._serial_load_scope():
+ # Here, not at the spawn: a lock gives a waiter no priority, and the
+ # duplicate-adoption phase below kills whatever is loaded -- after a
+ # restart, the new lifecycle's model.
+ with self._spawn_lock:
+ _stale_load = self._spawn_is_stale()
+ if _stale_load:
+ logger.info("dropping a load left over from the previous server lifecycle")
+ return False
# In-app update swapping binaries: refuse fast (set under this lock,
# so any in-flight load has drained) instead of using a half-swapped one.
if getattr(self, "_llama_update_in_progress", False):
@@ -24143,19 +24215,44 @@ class LlamaCppBackend:
run_cmd,
supports_cache_ram = bool(server_caps.get("supports_cache_ram")),
)
- self._process = subprocess.Popen(
- run_cmd,
- stdout = subprocess.PIPE,
- stderr = subprocess.STDOUT,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- env = env,
- cwd = _spawn_cwd,
- **_windows_hidden_subprocess_kwargs(),
- **_child_popen_kwargs(),
- )
- self._record_server_pid(self._process.pid)
+ # Check with publication under one lock: a spawn either
+ # publishes first and the sweep kills it, or sees the flag
+ # and never starts. Across Popen only, never the wait.
+ with self._spawn_lock:
+ if self._spawn_is_stale():
+ logger.info("app is shutting down; not starting llama-server")
+ self._close_attempt_log()
+ self._health_wait_cancelled = True
+ return False
+ _spawned = subprocess.Popen(
+ run_cmd,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ env = env,
+ cwd = _spawn_cwd,
+ **_windows_hidden_subprocess_kwargs(),
+ **_child_popen_kwargs(),
+ )
+ self._process = _spawned
+ # Inside the lock: written after a sweep reaped the child,
+ # _pid_start_identity yields no start time, and the bare pid
+ # left behind is one a later launch kills blind.
+ self._record_server_pid(_spawned.pid)
+ # mark_process_shutting_down does not take _spawn_lock, so the
+ # check above is not atomic against it for a helper-owned
+ # backend. Without this recheck a child spawned in that gap sits
+ # outside the completed sweep for the whole 600s health wait.
+ if self._spawn_is_stale():
+ logger.info(
+ "shutdown began during the spawn; killing the new llama-server"
+ )
+ self._kill_process()
+ self._close_attempt_log()
+ self._health_wait_cancelled = True
+ return False
# is_active covers it from here, so drop the pre-spawn flag.
self._memory_launch_pending = False
@@ -24467,7 +24564,8 @@ class LlamaCppBackend:
# and keep the staged runtime for the caller's next argv.
self._kill_process()
return False
- cpu_rc = self._process.poll() if self._process is not None else None
+ _proc_snap1 = self._process # snapshot: re-reading races the teardown
+ cpu_rc = _proc_snap1.poll() if _proc_snap1 is not None else None
detail = self._classify_llama_start_failure(
"\n".join(self._stdout_lines[-50:]),
gguf_path,
@@ -24684,7 +24782,8 @@ class LlamaCppBackend:
# skipping the futile flash-attn/MTP retries.
if not healthy and self._tensor_parallel and not _load_cancelled():
_ts_out = "\n".join(self._stdout_lines[-50:])
- _ts_rc = self._process.poll() if self._process is not None else None
+ _proc_snap2 = self._process # snapshot: re-reading races the teardown
+ _ts_rc = _proc_snap2.poll() if _proc_snap2 is not None else None
if self._should_record_tensor_split_abort(_ts_rc, _ts_out):
LlamaCppBackend._record_tensor_split_abort(
binary, model_identifier, _planned_cache_pair
@@ -25072,7 +25171,8 @@ class LlamaCppBackend:
# both vision and MTP, so retry that way before dropping either.
# Only on a hard fault with FA on; a cancel/unload stops respawn.
if not healthy and not _load_cancelled():
- _fa_rc = self._process.poll() if self._process is not None else None
+ _proc_snap3 = self._process # snapshot: re-reading races the teardown
+ _fa_rc = _proc_snap3.poll() if _proc_snap3 is not None else None
_fa_cmd = (
self._with_flash_attn_off(
_last_spawn_cmd,
@@ -25145,7 +25245,8 @@ class LlamaCppBackend:
):
# A first-decode hard fault is usually the FA kernel: retry
# FA-off (keeps MTP) before dropping speculative decoding below.
- _probe_rc = self._process.poll() if self._process is not None else None
+ _proc_snap4 = self._process # snapshot: re-reading races the teardown
+ _probe_rc = _proc_snap4.poll() if _proc_snap4 is not None else None
_fa_cmd = (
self._with_flash_attn_off(
_last_spawn_cmd,
@@ -25315,7 +25416,8 @@ class LlamaCppBackend:
if not healthy:
out = "\n".join(self._stdout_lines[-50:])
# Read the crash code before _kill_process() clears _process.
- _crash_rc = self._process.poll() if self._process is not None else None
+ _proc_snap5 = self._process # snapshot: re-reading races the teardown
+ _crash_rc = _proc_snap5.poll() if _proc_snap5 is not None else None
self._kill_process()
# Only when the WAIT itself was cancelled. A cancel that lands later,
# while a crashed launch is staging its CPU fallback, must still run
@@ -25387,8 +25489,10 @@ class LlamaCppBackend:
)
else:
_cpu_projector_out = "\n".join(self._stdout_lines[-50:])
+ # Snapshot: re-reading races the teardown.
+ _proc_snap6 = self._process
_cpu_projector_rc = (
- self._process.poll() if self._process is not None else None
+ _proc_snap6.poll() if _proc_snap6 is not None else None
)
self._kill_process()
if _finish_cancelled_health_wait(
@@ -25458,11 +25562,15 @@ class LlamaCppBackend:
_last_spawn_cmd = list(cmd)
self._is_vision = False
self._mmproj_has_audio = False
- self._start_llama_process(
+ if not self._start_llama_process(
cmd,
env,
child_gpu_physical_ids = _child_gpu_physical_ids,
- )
+ ):
+ # Shutdown refused the retry; self._process still names
+ # the old child the teardown is clearing.
+ _cleanup_cancelled_load("App shut down during the text-only retry")
+ return False
if self._wait_for_health(timeout = 600.0, cancelled = _load_cancelled):
healthy = True
# The child that serves this session never read the
@@ -25483,9 +25591,9 @@ class LlamaCppBackend:
else:
# Read the exit code before _kill_process() clears it, so
# an OS-killed text-only retry still gets the OOM message.
- _retry_rc = (
- self._process.poll() if self._process is not None else None
- )
+ # Snapshot: re-reading races the teardown.
+ _retry_proc = self._process
+ _retry_rc = _retry_proc.poll() if _retry_proc is not None else None
self._kill_process()
if _finish_cancelled_health_wait(
"Load cancelled during the text-only retry health wait"
@@ -25589,7 +25697,11 @@ class LlamaCppBackend:
else None
),
)
- self._healthy = True
+ if not self._publish_healthy():
+ # Teardown began between the 200 and this commit; publishing
+ # would report a model that is gone.
+ _cleanup_cancelled_load("App shut down as the load was completing")
+ return False
self._commit_effective_parallel_slots(n_parallel)
self._swa_full = swa_full
self._kv_cache_unified = kv_cache_unified
@@ -26724,8 +26836,99 @@ class LlamaCppBackend:
except Exception as e:
logger.debug(f"Could not terminate server descendants: {e}")
- def _kill_process(self):
- """Terminate the subprocess if running."""
+ def _publish_healthy(self) -> bool:
+ """Commit _healthy under the spawn lock, or refuse if this load is stale.
+
+ Under the lock the teardown mark is set with, so only two orders exist:
+ publish then teardown (which clears _healthy), or teardown then a refused
+ publish.
+ """
+ with self._spawn_lock:
+ if self._spawn_is_stale():
+ logger.info("load no longer belongs to this lifecycle; not publishing it healthy")
+ return False
+ self._healthy = True
+ return True
+
+ def _close_attempt_log(self) -> None:
+ """Close the per-attempt tee log opened just before a spawn.
+
+ A refusal publishes no process and _kill_process returns early when there is
+ none, so nothing else closes it: the next attempt leaks the descriptor and,
+ on Windows, holds the file lock an update needs.
+ """
+ fh = getattr(self, "_llama_log_fh", None)
+ if fh is not None:
+ try:
+ fh.close()
+ except Exception:
+ pass
+ self._llama_log_fh = None
+
+ def _begin_server_lifecycle(self) -> None:
+ """Clear shutdown state so a restarted server can launch again.
+
+ The backend is a module singleton and an embedded host may call
+ run_server() more than once in one process, so "shutting down" is scoped
+ to a lifecycle rather than to the interpreter. Called from run_server
+ before anything can spawn.
+
+ """
+ # _teardown_lock first, so this waits for an in-progress kill rather than
+ # clearing the flag underneath it. Same order as _kill_process.
+ with self._teardown_lock:
+ with self._spawn_lock:
+ self._shutting_down = False
+ self._torn_down_process = None
+
+ def _spawn_is_stale(self) -> bool:
+ """Whether this load may no longer spawn. Caller holds _spawn_lock."""
+ if getattr(self, "_shutting_down", False):
+ return True
+ # Per-instance state only covers the singleton run.py tears down. A helper or
+ # advisor load builds its own backend (hub/utils/llm_assist.py,
+ # utils/datasets/llm_assist.py), which nothing marks, so without this it would
+ # still spawn a server after the sweep. Read second: the attribute is cheaper
+ # and answers for the instance that actually gets torn down.
+ from utils.process_lifetime import is_process_shutting_down
+
+ return is_process_shutting_down()
+
+ def _kill_process(self, *, teardown: bool = False):
+ """Terminate the subprocess if running.
+
+ ``teardown`` marks an app-level stop (shutdown, atexit) rather than the
+ retry ladder reaping a child it is about to replace: only the former may
+ end an in-flight health wait.
+
+ A teardown holds _teardown_lock for the WHOLE kill, because the terminate and
+ wait below keep reading self._process and finally clear it: a lifecycle
+ reopened mid-kill would have its new child dropped or terminated here, so
+ _begin_server_lifecycle takes the same lock and waits.
+
+ _spawn_lock is taken only long enough to set the flag, NOT across the kill.
+ A spawn arriving mid-teardown then reads the flag and refuses in microseconds
+ instead of queuing behind a SIGTERM/SIGKILL escalation that can run for
+ seconds; the load thread it belongs to is one shutdown is already waiting on.
+ Marked above the early return, since a quit during a download still has to be
+ recorded.
+ """
+ if teardown:
+ # Process-wide as well as per-instance: the atexit teardown reaches here
+ # without going through run.py, and the backends a helper load builds for
+ # itself are only ever covered by the shared latch.
+ from utils.process_lifetime import mark_process_shutting_down
+
+ mark_process_shutting_down()
+ with self._teardown_lock:
+ with self._spawn_lock:
+ self._shutting_down = True
+ self._kill_process_body(teardown = True)
+ return
+ self._kill_process_body(teardown = False)
+
+ def _kill_process_body(self, *, teardown: bool):
+ """The kill itself. Caller holds _teardown_lock when ``teardown``."""
# Stop the watchdog before a deliberate kill so a planned reload/unload
# isn't seen as a crash; a real crash never routes through here.
self._stop_mtp_crash_watchdog()
@@ -26751,6 +26954,11 @@ class LlamaCppBackend:
_pid = getattr(self._process, "pid", None)
_pgid = self._leading_process_group(_pid)
_descendants = self._collect_descendants(_pid)
+ if teardown:
+ # Before the signal, and as the process itself: the reference stays set
+ # across the waits below, and only identity says which child a teardown
+ # landing between a spawn and its wait referred to.
+ self._torn_down_process = self._process
try:
if terminable:
self._process.terminate()
@@ -27279,7 +27487,7 @@ class LlamaCppBackend:
return killed
def _cleanup(self):
- """atexit handler to ensure llama-server is terminated.
+ """atexit handler to ensure llama-server is terminated (a teardown).
Nothing here may report a failure through the logging machinery. By the
time atexit runs, the streams the handlers write to can already be closed,
@@ -27297,7 +27505,7 @@ class LlamaCppBackend:
raise_exceptions = logging.raiseExceptions
logging.raiseExceptions = False
try:
- self._kill_process()
+ self._kill_process(teardown = True)
# TemporaryDirectory's exit hook runs first and cannot delete a staged
# runtime whose server is alive (Windows locks the exe). Retry post-kill.
self._cleanup_cpu_fallback_runtime()
@@ -28298,8 +28506,17 @@ class LlamaCppBackend:
# Why this wait ended, for callers that must tell a cancel apart from a crash:
# a cancel landing during CPU-fallback staging is not a cancelled wait.
self._health_wait_cancelled = False
+ # No teardown reset here: it would erase one that landed between this load's
+ # spawn and this line. _torn_down_process is matched by identity instead.
+ process = None # the child this wait last looked at, read again after the loop
while time.monotonic() < deadline:
+ # Durable, unlike the per-process marker below: once teardown begins,
+ # every later iteration sees it.
+ if getattr(self, "_shutting_down", False):
+ logger.info("llama-server was torn down while waiting for it to become healthy")
+ self._health_wait_cancelled = True
+ return False
# unload_model() blocks on self._lock, which the load holds across this wait.
if cancelled is not None and cancelled():
logger.info("llama-server startup cancelled before it became healthy")
@@ -28318,6 +28535,12 @@ class LlamaCppBackend:
return False
# Process crashed?
if process.poll() is not None:
+ # A teardown holds the reference across its waits, so THIS child
+ # exiting under it is deliberate.
+ if getattr(self, "_torn_down_process", None) is process:
+ logger.info("llama-server was torn down while waiting for it to become healthy")
+ self._health_wait_cancelled = True
+ return False
# Let the drain thread collect final output.
if self._stdout_thread is not None:
self._stdout_thread.join(timeout = 2)
@@ -28347,6 +28570,11 @@ class LlamaCppBackend:
logger.info("llama-server became healthy after the load was cancelled")
self._health_wait_cancelled = True
return False
+ # A 200 arriving as shutdown began must not publish _healthy.
+ if getattr(self, "_shutting_down", False):
+ logger.info("llama-server became healthy while the app was shutting down")
+ self._health_wait_cancelled = True
+ return False
return True
except (
httpx.ConnectError,
@@ -28366,6 +28594,16 @@ class LlamaCppBackend:
self._health_wait_cancelled = True
return False
+ # The deadline is the other way out of the loop, so it asks too -- and about
+ # both signals, since _kill_process sets _shutting_down on entry but
+ # _torn_down_process only after collecting descendants.
+ if getattr(self, "_shutting_down", False) or (
+ process is not None and getattr(self, "_torn_down_process", None) is process
+ ):
+ logger.info("llama-server was torn down while waiting for it to become healthy")
+ self._health_wait_cancelled = True
+ return False
+
# Leave a marker so _classify_llama_start_failure tells a live but
# never-healthy load (too large, or a proxy hijacking the loopback
# probe) apart from a bad GGUF (#5740).
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index bb7d64c358c..fcbca67a5f9 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -440,6 +440,15 @@ class InferenceOrchestrator:
raise SidecarSwapInProgress(
"A transformers repair is replacing the latest sidecar; retry when it completes."
)
+ # Last gate before Popen. A preview or auto-switch load is not a
+ # _ScopedLoadAttempt, so the route's shutdown sweep cannot cancel it; it can
+ # clear the load's own checks and only then reach here, after the shutdown
+ # already stopped this subprocess. Checked at the spawn itself so the answer
+ # cannot go stale between the check and the child.
+ from utils.process_lifetime import is_process_shutting_down
+
+ if is_process_shutting_down():
+ raise RuntimeError("Studio is shutting down; not starting an inference subprocess")
from utils.native_path_leases import (
native_path_secret_removed_for_child_start,
run_without_native_path_secret,
@@ -461,7 +470,13 @@ class InferenceOrchestrator:
self._cancel_event = _CTX.Event()
self._drain_event = _CTX.Event()
- self._proc = _CTX.Process(
+ # Built into a local FIRST, and started through that local. A shutdown can
+ # observe a not-yet-alive child, clear self._proc and finish its sweep while
+ # start() is still returning, so the attribute is not a handle this code can
+ # rely on from here on: snapshotting it after start() would capture the None
+ # and lose the only reference to a live child, which is the orphan this
+ # change exists to prevent.
+ _spawned_proc = _CTX.Process(
target = run_without_native_path_secret,
args = ("core.inference.worker", "run_inference_process", cache_env),
kwargs = {
@@ -473,11 +488,49 @@ class InferenceOrchestrator:
},
daemon = True,
)
- self._proc.start()
+ self._proc = _spawned_proc
+ _spawned_proc.start()
from utils.process_lifetime import adopt_pid
- adopt_pid(self._proc.pid) # bind to parent lifetime (Windows job / sweep)
- logger.info("Inference subprocess started (pid=%s)", self._proc.pid)
+ adopt_pid(_spawned_proc.pid) # bind to parent lifetime (Windows job / sweep)
+
+ # The gate above is 30-odd lines and a process start away from here, so a
+ # shutdown can begin in between, see no live _proc, and finish its sweep
+ # while this child is still being born. Recheck now it exists and reap it,
+ # the same shape as the cancel_load recheck below the caller's spawn.
+ # A lock across the spawn would close it too, but _shutdown_subprocess holds
+ # that lock for its whole teardown, so quitting would then queue behind a
+ # spawn it is about to undo. adopt_pid runs first either way: a child that
+ # dies here must still be in the sweep record.
+ if is_process_shutting_down() or self._proc is not _spawned_proc:
+ logger.info("Shutdown began during spawn; tearing the new inference worker down")
+ self._shutdown_subprocess(timeout = 5)
+ # If shutdown already dropped the mirror, that call cannot see this child.
+ # The local handle is the only one left, so reap it here -- and escalate the
+ # way _shutdown_subprocess_locked does, rather than abandoning a worker that
+ # ignores SIGTERM. The step-7 snapshot has already been taken by this point,
+ # so a child left alive here survives until a later startup reaps its record.
+ try:
+ if _spawned_proc.is_alive():
+ _spawned_proc.terminate()
+ _spawned_proc.join(5)
+ if _spawned_proc.is_alive():
+ from utils.process_lifetime import terminate_pid
+ terminate_pid(_spawned_proc.pid, timeout = 5)
+ _spawned_proc.join(5)
+ if _spawned_proc.is_alive():
+ _spawned_proc.kill()
+ _spawned_proc.join(5)
+ if _spawned_proc.is_alive():
+ logger.warning(
+ "Raced inference worker (pid=%s) survived terminate and kill; "
+ "leaving it in the lifetime record for the startup sweep",
+ _spawned_proc.pid,
+ )
+ except Exception as exc:
+ logger.debug("Could not reap the raced inference worker: %s", exc)
+ raise RuntimeError("Studio is shutting down; not starting an inference subprocess")
+ logger.info("Inference subprocess started (pid=%s)", _spawned_proc.pid)
def _cancel_generation(self) -> None:
"""Cancel any ongoing generation in the subprocess (instant)."""
@@ -1733,29 +1786,46 @@ class InferenceOrchestrator:
self.active_model_name = None
self.models.clear()
return False
+ from utils.process_lifetime import is_process_shutting_down
+
model_info = resp.get("model_info", {})
- self.active_model_name = model_info.get("identifier", model_name)
- self.load_generation += 1
- # A load always spawns a fresh subprocess holding only this model, so mirror that. A lingering stale
- # name would pass unload_model's "not in self.models" guard, and the worker's absent-name fallback
- # would unload its *active* model, not the already-gone one.
- self.models = {}
- self.models[self.active_model_name] = _mirrored_model_entry(
- model_info, model_name
- )
- # Lets the already-loaded shortcut tell a CPU request from the GPU
- # model it would otherwise report as satisfied. Native audio only:
- # marking anything else tells training a GPU model holds no VRAM.
- self.models[self.active_model_name]["audio_cpu"] = model_info.get(
- "audio_type"
- ) in NATIVE_AUDIO_TYPES and audio_device_forces_cpu(audio_device)
- self.models[self.active_model_name].update(
- _mlx_runtime_mirror_fields(model_info)
- )
- # Mirror chat_template_info so routes can classify caps without re-entering the subprocess
- _tpl_info = model_info.get("chat_template_info")
- if isinstance(_tpl_info, dict):
- self.models[self.active_model_name]["chat_template_info"] = _tpl_info
+ # A "loaded" reply dequeued just as shutdown kills the worker would
+ # otherwise be published here, and active_model_name is what the
+ # already-loaded fast path trusts without testing liveness. Held
+ # under the lock shutdown kills with, so the check and the
+ # publication are one step rather than a race.
+ with self._subprocess_shutdown_lock:
+ if is_process_shutting_down():
+ logger.info(
+ "Shutdown overtook the load of '%s'; not publishing it as resident",
+ model_name,
+ )
+ self.loading_models.discard(model_name)
+ self.active_model_name = None
+ self.models.clear()
+ return False
+ self.active_model_name = model_info.get("identifier", model_name)
+ self.load_generation += 1
+ # A load always spawns a fresh subprocess holding only this model, so mirror that. A lingering stale
+ # name would pass unload_model's "not in self.models" guard, and the worker's absent-name fallback
+ # would unload its *active* model, not the already-gone one.
+ self.models = {}
+ self.models[self.active_model_name] = _mirrored_model_entry(
+ model_info, model_name
+ )
+ # Lets the already-loaded shortcut tell a CPU request from the GPU
+ # model it would otherwise report as satisfied. Native audio only:
+ # marking anything else tells training a GPU model holds no VRAM.
+ self.models[self.active_model_name]["audio_cpu"] = model_info.get(
+ "audio_type"
+ ) in NATIVE_AUDIO_TYPES and audio_device_forces_cpu(audio_device)
+ self.models[self.active_model_name].update(
+ _mlx_runtime_mirror_fields(model_info)
+ )
+ # Mirror chat_template_info so routes can classify caps without re-entering the subprocess
+ _tpl_info = model_info.get("chat_template_info")
+ if isinstance(_tpl_info, dict):
+ self.models[self.active_model_name]["chat_template_info"] = _tpl_info
self.loading_models.discard(model_name)
logger.info("Model '%s' loaded successfully in subprocess", model_name)
return True
diff --git a/studio/backend/core/inference/sd_cpp_engine.py b/studio/backend/core/inference/sd_cpp_engine.py
index 042aec9539c..15dbdbf3055 100644
--- a/studio/backend/core/inference/sd_cpp_engine.py
+++ b/studio/backend/core/inference/sd_cpp_engine.py
@@ -27,7 +27,12 @@ import time
from pathlib import Path
from typing import Callable, Iterator, Optional
-from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid
+from utils.process_lifetime import (
+ adopt_pid,
+ child_popen_kwargs,
+ forget_pid,
+ is_process_shutting_down,
+)
from utils.native_path_leases import child_env_without_native_path_secret
from utils.subprocess_compat import windows_hidden_subprocess_kwargs
from core.inference.sd_cpp_args import (
@@ -843,6 +848,11 @@ class SdCppEngine:
logger.info("sd-cli run started: %s", summary)
t0 = time.time()
+ # A generation admitted before the quit can still reach this Popen after the
+ # shutdown sweep has taken its snapshot, and sd-cli would then keep running with
+ # nothing left to reap it, holding VRAM past the app.
+ if is_process_shutting_down():
+ raise SdCppCancelled("Studio is shutting down; not starting sd-cli.")
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
@@ -860,6 +870,13 @@ class SdCppEngine:
# the kwargs above are empty on macOS, so record it too, else a crash mid-generation leaves sd-cli holding VRAM
# with nothing able to find it
adopt_pid(proc.pid)
+ # Recheck once the pid is recorded, for the window between the gate above and this
+ # record. Adoption ran first, so the child killed here was in the sweep record for
+ # as long as it existed.
+ if is_process_shutting_down():
+ logger.info("shutdown began during the spawn; killing the new sd-cli")
+ _terminate(proc)
+ raise SdCppCancelled("Studio is shutting down; not starting sd-cli.")
# Drain stdout on a reader thread so the timeout holds even when the child hangs WITHOUT printing (a plain `for
# line in proc.stdout` blocks until EOF). Lines, then a None sentinel, go to a queue the main loop polls against
# a wall-clock deadline. iter_sd_cpp_records also splits sd-cli's in-place progress redraws, which carry no
diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.py
index 70448f78ecf..7c1f34dd671 100644
--- a/studio/backend/core/inference/sd_cpp_server.py
+++ b/studio/backend/core/inference/sd_cpp_server.py
@@ -56,7 +56,12 @@ from core.inference.sd_cpp_engine import (
runtime_env,
)
from utils.native_path_leases import child_env_without_native_path_secret
-from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid
+from utils.process_lifetime import (
+ adopt_pid,
+ child_popen_kwargs,
+ forget_pid,
+ is_process_shutting_down,
+)
from utils.subprocess_compat import windows_hidden_subprocess_kwargs
logger = logging.getLogger(__name__)
@@ -260,6 +265,16 @@ class SdCppServer:
# Spawn INSIDE the long-lived drain thread: child_popen_kwargs() sets PR_SET_PDEATHSIG, bound to the
# creating thread on Linux, so the creator must outlive the child.
def _own_process() -> None:
+ # One flag at every spawn. No _graceful_shutdown step unloads the
+ # diffusion engine and cancel_pending_loads only knows about chat loads,
+ # so without this a quit during an /images/load can start a multi-GB
+ # sd-server after the step-7 sweep has taken its snapshot.
+ if is_process_shutting_down():
+ self._spawn_error = RuntimeError(
+ "Studio is shutting down; not starting sd-server"
+ )
+ spawned.set()
+ return
try:
proc = subprocess.Popen(
cmd,
@@ -279,6 +294,33 @@ class SdCppServer:
self._process = proc
self.port = port
adopt_pid(proc.pid) # so a global shutdown sweep also reaps it
+ # Recheck once the pid is recorded, for the window between the gate and
+ # this record. Adoption runs first, so a child killed here was in the
+ # sweep record for as long as it existed.
+ if is_process_shutting_down():
+ logger.info("shutdown began during the spawn; killing the new sd-server")
+ try:
+ proc.kill()
+ proc.wait(timeout = 5)
+ except Exception: # noqa: BLE001 - the reap is best-effort
+ pass
+ # Only drop the record once the child is confirmed gone. If the kill
+ # or the wait raised, the server is still alive with the sweep
+ # already past it, and this record is the last thing that could
+ # reap it; forgetting it here would be the orphan this PR is about.
+ if proc.poll() is not None:
+ forget_pid(proc.pid)
+ else:
+ logger.warning(
+ "sd-server pid %s survived the shutdown kill; leaving it adopted",
+ proc.pid,
+ )
+ self._process = None
+ self._spawn_error = RuntimeError(
+ "Studio is shutting down; not starting sd-server"
+ )
+ spawned.set()
+ return
spawned.set()
self._drain_stdout(proc)
try:
diff --git a/studio/backend/core/inference/stt_ggml_sidecar.py b/studio/backend/core/inference/stt_ggml_sidecar.py
index f2a63212195..5f84dcb4de5 100644
--- a/studio/backend/core/inference/stt_ggml_sidecar.py
+++ b/studio/backend/core/inference/stt_ggml_sidecar.py
@@ -71,7 +71,12 @@ from core.inference.stt_sidecar import (
from utils.prebuilt.child_env import isolate_home, scrub_env, wsl_system_rocm_lib_dirs
from utils.prebuilt.runtime_libs import dedupe_existing_dirs
from utils.prebuilt.whisper_layout import lookup_marker
-from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid
+from utils.process_lifetime import (
+ adopt_pid,
+ child_popen_kwargs,
+ forget_pid,
+ is_process_shutting_down,
+)
logger = get_logger(__name__)
@@ -1000,6 +1005,13 @@ class GgmlSttSidecar:
self._release_locked()
# release the reservation as late as possible: whisper-server binds the port moments after this close
reservation.close()
+ # One flag at every spawn. No _graceful_shutdown step stops this
+ # sidecar, so a load still downloading or in preflight as the app quits
+ # would otherwise start whisper-server after the sweep had run.
+ if is_process_shutting_down():
+ raise SttLoadCancelledError(
+ "Studio is shutting down; not starting whisper-server."
+ )
process = subprocess.Popen(
command,
stdout = subprocess.DEVNULL,
@@ -1015,6 +1027,18 @@ class GgmlSttSidecar:
with self._load_state_lock:
self._starting_process = process
adopt_pid(process.pid) # terminate_all backstop for graceful exits
+ # Recheck once the pid is recorded: the latch can be set between the
+ # gate above and this record, and the child would then sit outside a
+ # sweep that has already finished. Adoption runs first either way, so a
+ # child killed here is still in the sweep record.
+ if is_process_shutting_down():
+ if process.poll() is None:
+ process.kill()
+ process.wait(timeout = 10)
+ forget_pid(process.pid)
+ raise SttLoadCancelledError(
+ "Studio is shutting down; not starting whisper-server."
+ )
try:
self._wait_for_server(process, port, cancel_event)
except Exception:
diff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.py
index 8c6c8396a72..78e98091035 100644
--- a/studio/backend/core/inference/stt_mtmd_sidecar.py
+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py
@@ -30,7 +30,12 @@ from typing import Iterator, Optional
from loggers import get_logger
from hub.utils.hf_tokens import normalize_token
-from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid
+from utils.process_lifetime import (
+ adopt_pid,
+ child_popen_kwargs,
+ forget_pid,
+ is_process_shutting_down,
+)
from core.inference.stt_ggml_sidecar import _pcm_to_wav_bytes
from core.inference.stt_sidecar import (
@@ -184,8 +189,18 @@ def _reap(process: Optional[subprocess.Popen]) -> None:
except Exception as exc: # noqa: BLE001 - shutdown must not raise
logger.warning("Could not reap llama-server (pid %s): %s", process.pid, exc)
finally:
- # the PID is dead, so drop it before it can be reused by something else that terminate_all would then signal
- forget_pid(process.pid)
+ # Drop the pid once it is dead, before it can be reused by something else that
+ # terminate_all would then signal. Only once it is dead: if the terminate, the
+ # kill or either wait raised, the child is still alive, and after the shutdown
+ # sweep has passed this record is the last thing that could reap it.
+ if process.poll() is not None:
+ forget_pid(process.pid)
+ else:
+ logger.warning(
+ "llama-server (pid %s) survived the reap; leaving it adopted so a later "
+ "sweep can still find it",
+ process.pid,
+ )
def _cached_file(
@@ -920,6 +935,13 @@ class MtmdSttSidecar:
# chat backend's _cmd_has_gpu_companion() treats as a GPU companion whatever --gpu-layers says.
cmd.append("--no-mmproj-offload")
sock.close()
+ # One flag at every spawn, as above: nothing in _graceful_shutdown stops
+ # this sidecar, so without it a quit during a load starts a server the
+ # step-7 sweep has already passed by.
+ if is_process_shutting_down():
+ raise SttLoadCancelledError(
+ "Studio is shutting down; not starting the MTMD server."
+ )
process = subprocess.Popen(
cmd,
# nothing reads these, and an undrained pipe blocks llama-server mid-startup once its logs fill the
@@ -938,6 +960,13 @@ class MtmdSttSidecar:
with self._lock:
self._starting_process = process
adopt_pid(process.pid) # terminate_all backstop for graceful exits
+ # Recheck once the pid is recorded, for the window between the gate and the
+ # record. _reap kills and forgets, so nothing is left half-tracked.
+ if is_process_shutting_down():
+ _reap(process)
+ raise SttLoadCancelledError(
+ "Studio is shutting down; not starting the MTMD server."
+ )
if not self._wait_for_server(process, port, cancel_event):
# Reap it here: _process was never assigned, so unload() cannot reach a child that ignores SIGTERM and
# keeps port and VRAM.
diff --git a/studio/backend/core/inference/stt_transformers_worker.py b/studio/backend/core/inference/stt_transformers_worker.py
index 6346088c2d0..4eb23b2fb2d 100644
--- a/studio/backend/core/inference/stt_transformers_worker.py
+++ b/studio/backend/core/inference/stt_transformers_worker.py
@@ -339,8 +339,13 @@ class WhisperWorker:
native_path_secret_removed_for_child_start,
run_without_native_path_secret,
)
- from utils.process_lifetime import adopt_pid
+ from utils.process_lifetime import adopt_pid, is_process_shutting_down
+ # One flag at every spawn: no _graceful_shutdown step unloads this sidecar and
+ # cancel_pending_loads only reaches the chat /load attempts, so without this a
+ # quit during an STT load starts a worker the step-7 sweep has already passed.
+ if is_process_shutting_down():
+ raise SttWorkerSpawnError("Studio is shutting down; not starting the dictation worker.")
cache_env = get_hf_cache_paths().child_env({})
try:
with (
@@ -365,16 +370,37 @@ class WhisperWorker:
},
daemon = True,
)
- self._process.start()
+ # Local handle, and started through it: a concurrent teardown can clear
+ # self._process while start() is still returning, and re-reading the
+ # attribute afterwards would lose the only reference to a live child.
+ _spawned_proc = self._process
+ _spawned_proc.start()
except Exception as exc: # noqa: BLE001 - any refusal to spawn reads the same
self._process = None
self._close_queues()
raise SttWorkerSpawnError(
f"Could not start the dictation worker process: {exc}"
) from exc
- adopt_pid(self._process.pid) # terminate_all backstop for graceful exits
+ adopt_pid(_spawned_proc.pid) # terminate_all backstop for graceful exits
+ # Recheck once the pid is recorded: the latch can be set between the gate above
+ # and this record. Adoption runs first, so a child reaped here is still in the
+ # sweep record.
+ if is_process_shutting_down():
+ logger.info("shutdown began during the spawn; reaping the new dictation worker")
+ try:
+ if _spawned_proc.is_alive():
+ _spawned_proc.terminate()
+ _spawned_proc.join(5)
+ if _spawned_proc.is_alive():
+ _spawned_proc.kill()
+ _spawned_proc.join(5)
+ except Exception: # noqa: BLE001 - the reap is best-effort
+ pass
+ self._process = None
+ self._close_queues()
+ raise SttWorkerSpawnError("Studio is shutting down; not starting the dictation worker.")
logger.info(
- "STT worker started (pid=%s) for %s on %s", self._process.pid, snapshot_path, device
+ "STT worker started (pid=%s) for %s on %s", _spawned_proc.pid, snapshot_path, device
)
try:
self._send(
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py
index 799244d3984..0900a7c61a1 100644
--- a/studio/backend/core/rag/embed_llama_server.py
+++ b/studio/backend/core/rag/embed_llama_server.py
@@ -33,7 +33,12 @@ import numpy as np
from utils.native_path_leases import child_env_without_native_path_secret
from utils.subprocess_compat import windows_hidden_subprocess_kwargs
-from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid
+from utils.process_lifetime import (
+ adopt_pid,
+ child_popen_kwargs,
+ forget_pid,
+ is_process_shutting_down,
+)
from . import config
from utils.paths.path_utils import is_appledouble_metadata
@@ -819,6 +824,10 @@ class LlamaServerBackend:
try:
self._spawn_once(use_gpu, model_name)
except RuntimeError:
+ # A shutdown refusal is terminal: the CPU fallback would just spawn into the
+ # same latch and be refused again.
+ if is_process_shutting_down():
+ raise
if use_gpu and not config.embed_device_requires_gpu():
logger.warning("embed server GPU start failed; falling back to CPU")
self._force_cpu = True
@@ -842,6 +851,11 @@ class LlamaServerBackend:
" ".join(cmd),
)
self._stdout_lines = []
+ # One flag at every spawn. No _graceful_shutdown step stops this backend, so an
+ # encode still resolving or downloading its model as the app quits would
+ # otherwise Popen a server after terminate_all had taken its snapshot.
+ if is_process_shutting_down():
+ raise RuntimeError("Studio is shutting down; not starting the embed server")
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
@@ -857,6 +871,14 @@ class LlamaServerBackend:
# child_popen_kwargs() is empty on macOS, so the crash record is the only thing that can reap it
# after a force quit.
adopt_pid(proc.pid)
+ # Recheck once the pid is recorded, as the llama-server and inference worker
+ # spawns do: the latch can be set between the gate above and this record, and
+ # the child would then sit outside a sweep that has already finished. Adoption
+ # runs first either way, so a child killed here is still in the sweep record.
+ if is_process_shutting_down():
+ logger.info("shutdown began during the spawn; killing the new embed server")
+ self._kill_process()
+ raise RuntimeError("Studio is shutting down; not starting the embed server")
self._port = port
self._stdout_thread = threading.Thread(
target = self._drain_stdout,
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index 77d3663ab4b..777437852cd 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -1717,7 +1717,7 @@ class TrainingBackend:
},
daemon = True,
)
- from utils.process_lifetime import adopt_pid
+ from utils.process_lifetime import adopt_pid, is_process_shutting_down
previous_job_id = None
previous_start_request_id = None
@@ -1731,6 +1731,16 @@ class TrainingBackend:
start_request_id,
)
return False
+ # The cancel check above is about this start request; the latch is
+ # about the process. A start admitted before the quit can still
+ # reach here after the shutdown sweep has taken its snapshot, and
+ # the worker would then train on past it holding the GPU.
+ if is_process_shutting_down():
+ logger.info(
+ "Studio is shutting down; not starting training worker for %s",
+ start_request_id,
+ )
+ return False
previous_job_id = self.current_job_id
previous_start_request_id = self.current_start_request_id
proc.start()
@@ -1738,9 +1748,16 @@ class TrainingBackend:
self.current_start_request_id = start_request_id
try:
adopt_pid(proc.pid)
+ # Recheck once the pid is recorded, for the window between the
+ # gate above and this record. Raised rather than handled inline
+ # so it reuses the terminate ladder and the state rollback below;
+ # adoption ran first, so the worker is in the sweep record for as
+ # long as it exists.
+ if is_process_shutting_down():
+ raise RuntimeError("Studio is shutting down")
except Exception:
logger.error(
- "Failed to adopt training subprocess; terminating it",
+ "Could not keep the training subprocess; terminating it",
exc_info = True,
)
try:
@@ -2416,10 +2433,37 @@ class TrainingBackend:
},
daemon = True,
)
- new_proc.start()
- from utils.process_lifetime import adopt_pid
+ from utils.process_lifetime import adopt_pid, is_process_shutting_down
+ # A stall recovery that started before the quit can still reach
+ # this respawn after the shutdown sweep has taken its snapshot.
+ if is_process_shutting_down():
+ raise RuntimeError(
+ "Studio is shutting down; not respawning the training worker"
+ )
+ new_proc.start()
adopt_pid(new_proc.pid)
+ # Recheck once the pid is recorded, for the window between the gate
+ # above and this record. Adoption ran first, so the worker killed
+ # here was in the sweep record for as long as it existed.
+ if is_process_shutting_down():
+ logger.info(
+ "shutdown began during the respawn; killing the new training worker"
+ )
+ try:
+ if new_proc.is_alive():
+ new_proc.terminate()
+ new_proc.join(timeout = 5.0)
+ if new_proc.is_alive():
+ new_proc.kill()
+ new_proc.join(timeout = 2.0)
+ except Exception: # noqa: BLE001 - the reap is best-effort
+ logger.warning(
+ "could not reap the new training worker", exc_info = True
+ )
+ raise RuntimeError(
+ "Studio is shutting down; not respawning the training worker"
+ )
except Exception:
logger.error("Failed to respawn training subprocess", exc_info = True)
self._spawn_in_progress = False
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 0020e915919..e85181f43d2 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -13787,6 +13787,11 @@ _scoped_load_attempts: dict[tuple[str, str], _ScopedLoadAttempt] = {}
_scoped_load_cancel_tombstones: dict[tuple[str, str], tuple[str, float]] = {}
_running_load_attempt: Optional[_ScopedLoadAttempt] = None
_pending_load_attempts: dict[str, _ScopedLoadAttempt] = {}
+# Latched by cancel_pending_loads, cleared by begin_load_lifecycle. A snapshot alone
+# misses a request uvicorn already admitted but schedules while shutdown is running:
+# should_exit stops new connections, not existing request tasks. Lifecycle-scoped, not
+# permanent, or an embedded host's second run_server could never load anything.
+_loads_shutting_down = False
_SCOPED_LOAD_CANCEL_TOMBSTONE_TTL_S = 60.0
# Bound on waiting for a cancel's teardown to report back. Only the /unload
# handler sets cancel_complete for a running attempt, so a disconnect or a
@@ -13797,6 +13802,50 @@ _SCOPED_LOAD_CANCEL_HANDSHAKE_TIMEOUT_S = 15.0
_SCOPED_LOAD_CANCEL_TOMBSTONE_LIMIT_PER_SUBJECT = 256
+def cancel_pending_loads() -> int:
+ """Cancel every in-flight /load. Called by the app shutdown, before the kill.
+
+ The backend's shutdown flag only guards its own spawn, and a request between
+ admission and that call is not yet holding anything the flag can see: it can
+ sit in the lifecycle gate or preflight for minutes, then reach the backend in
+ a lifecycle that has already been reset and load a model the new server never
+ asked for. Cancelling through the attempt's own event stops it wherever it is,
+ including mid-download, using the path /unload already uses.
+
+ Best-effort and non-blocking: shutdown must not wait on a load's teardown.
+ """
+ global _loads_shutting_down
+ with _scoped_load_attempts_lock:
+ _loads_shutting_down = True
+ attempts = list(_pending_load_attempts.values())
+ running = _running_load_attempt
+ if running is not None and all(a.token != running.token for a in attempts):
+ attempts.append(running)
+ for attempt in attempts:
+ _cancel_for_shutdown(attempt)
+ return len(attempts)
+
+
+def _cancel_for_shutdown(attempt: _ScopedLoadAttempt) -> None:
+ """Cancel an attempt AND close its handshake.
+
+ Only /unload sets cancel_complete, and at shutdown there is no /unload to do it,
+ so setting cancel_event alone leaves _run_tracked_load_model_impl's finally
+ waiting the full handshake timeout in a to_thread. Those executor threads are
+ non-daemon and would hold the process open. Shutdown owns the teardown, so there
+ is nothing to report back.
+ """
+ attempt.cancel_event.set()
+ attempt.cancel_complete.set()
+
+
+def begin_load_lifecycle() -> None:
+ """Clear the shutdown latch so a restarted server accepts loads again."""
+ global _loads_shutting_down
+ with _scoped_load_attempts_lock:
+ _loads_shutting_down = False
+
+
def _prune_scoped_load_cancel_tombstones(now: float) -> None:
expired = [
key for key, (_, expires_at) in _scoped_load_cancel_tombstones.items() if expires_at <= now
@@ -13971,6 +14020,12 @@ async def load_model_gated(
attempt = _begin_load_attempt(request, current_subject)
with _scoped_load_attempts_lock:
_pending_load_attempts[attempt.token] = attempt
+ # Registered after the shutdown sweep took its snapshot, so nothing else
+ # will ever cancel it. Under the same lock as the latch, so it cannot
+ # register between the latch and the sweep either.
+ _shutting_down_now = _loads_shutting_down
+ if _shutting_down_now:
+ _cancel_for_shutdown(attempt)
try:
_raise_if_sidecar_swap_in_progress()
# Hold the lifecycle gate across the load so idle auto-unload can't unload the
@@ -14057,6 +14112,16 @@ async def _load_model_impl(
if load_cancel_event is not None and load_cancel_event.is_set():
raise HTTPException(status_code = 409, detail = "Model load cancelled")
+ # Auto-switch and preview call this impl directly, without a _ScopedLoadAttempt,
+ # so the shutdown sweep has no event to set for them. Reading the latch here puts
+ # both on the same footing as /load: the callers of this helper are the points of
+ # no return, so a shutdown seen before one still stops the load rather than
+ # spawning a worker that outlives quit.
+ with _scoped_load_attempts_lock:
+ _shutting_down_now = _loads_shutting_down
+ if _shutting_down_now:
+ raise HTTPException(status_code = 409, detail = "Model load cancelled")
+
# A new load starts here; arm the progress throttle so this load's first
# sampled step logs even if it reports 100% immediately (cached/small load).
_reset_load_progress_step()
diff --git a/studio/backend/run.py b/studio/backend/run.py
index 6e6f7535130..695411430a8 100644
--- a/studio/backend/run.py
+++ b/studio/backend/run.py
@@ -1599,6 +1599,17 @@ def _graceful_shutdown(server = None):
"""
logger.info("Graceful shutdown initiated -- cleaning up subprocesses...")
+ # 0a. Latch "quitting" before any subsystem is torn down. Each step below refuses to
+ # respawn its OWN child once it has run, but a load still in flight can reach a
+ # different spawner afterwards: the orchestrator is stopped at step 2 and swept at
+ # step 7, and a helper load owns a backend no step touches at all. One flag, read at
+ # every spawn, covers the gaps between the steps.
+ try:
+ from utils.process_lifetime import mark_process_shutting_down
+ mark_process_shutting_down()
+ except Exception as e:
+ logger.warning("Could not latch the process shutdown flag: %s", e)
+
# 0. Drop the LAN listener first: it shares the loop uvicorn is about to stop.
try:
from lan_access import close_lan_listener_lifecycle
@@ -1636,9 +1647,21 @@ def _graceful_shutdown(server = None):
# 5. Kill llama-server subprocess (if loaded).
try:
- from routes.inference import _llama_cpp_backend
+ from routes.inference import _llama_cpp_backend, cancel_pending_loads
+
+ # Before the kill: a load still in the lifecycle gate or in preflight is not yet
+ # holding anything the backend's own flag can see, and would spawn llama-server
+ # after this step had already run.
+ try:
+ cancelled = cancel_pending_loads()
+ if cancelled:
+ logger.info("Cancelled %d in-flight model load(s) for shutdown", cancelled)
+ except Exception as e:
+ logger.warning("Could not cancel in-flight loads: %s", e)
if _llama_cpp_backend is not None:
- _llama_cpp_backend._kill_process()
+ # teardown = True: an app-level stop, not the retry ladder reaping a child it
+ # is about to replace. Only the former may end an in-flight health wait.
+ _llama_cpp_backend._kill_process(teardown = True)
except Exception as e:
logger.warning("Error shutting down llama-server: %s", e)
@@ -3005,6 +3028,23 @@ def run_server(
_close_lan_listener()
+ # An embedded host (studio/backend/colab.py) can call run_server again in the same
+ # interpreter, and the shutdown flags are process- and module-wide, so a second
+ # session would otherwise refuse every spawn and every load it admitted. Cleared
+ # here, after `from main import app` above (reaching for the route module earlier
+ # would build the backend singleton ahead of the startup steps that must come
+ # first) and before uvicorn serves anything below.
+ try:
+ from routes.inference import _llama_cpp_backend, begin_load_lifecycle
+ from utils.process_lifetime import begin_process_lifecycle
+
+ if _llama_cpp_backend is not None:
+ _llama_cpp_backend._begin_server_lifecycle()
+ begin_process_lifecycle()
+ begin_load_lifecycle()
+ except Exception as e:
+ logger.warning("Could not reset llama-server shutdown state: %s", e)
+
thread = Thread(target = _run, daemon = True)
_server_thread = thread
thread.start()
@@ -3041,9 +3081,17 @@ def run_server(
import atexit
atexit.register(_remove_pid_file)
- from utils.process_lifetime import terminate_all
+ from utils.process_lifetime import mark_process_shutting_down, terminate_all
atexit.register(terminate_all)
+ # LAST, so it runs FIRST: atexit is LIFO. Without it the sweep above takes its
+ # snapshot with the latch never set, because the only thing that sets it on this
+ # path is the backend's own _cleanup hook, registered when the routes singleton was
+ # built and therefore run AFTER this one. An embedded caller that lets the
+ # interpreter exit without _graceful_shutdown would otherwise get the unguarded
+ # behaviour this change exists to remove: a load still in flight passes its latch
+ # check and adopts a child the sweep has already gone past.
+ atexit.register(mark_process_shutting_down)
# Output port for Tauri (api-only), only after sockets bind and startup done.
# The headless `run --api-only` path opts out so it does not leak this line.
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index dd419ee57c5..19ffe04e18a 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -967,3 +967,39 @@ def _no_carried_over_hardware_measurements():
_clear()
yield
_clear()
+
+
[email protected](autouse = True)
+def _process_shutdown_latch_is_clear():
+ """Clear the process-wide shutdown latch around every test.
+
+ The latch is deliberately sticky in production: quitting is terminal, and only an
+ embedded host calling run_server again clears it. In a suite that makes it a
+ global one test can leave set for the rest of the file, and any test that tears a
+ backend down sets it, so a later spawn test sees a stale "quitting" and fails in
+ whatever order pytest happens to pick.
+ """
+ from utils import process_lifetime
+
+ def _reopen():
+ process_lifetime.begin_process_lifecycle()
+ # The ROUTE latch too. Any test that exercises _graceful_shutdown reaches
+ # cancel_pending_loads, which sets it, and only run_server clears it -- so one
+ # such test cancels every load admitted by every test that follows it. That is
+ # how four tunnel-safe tests came to fail in a full run and pass alone.
+ # Only if it is ALREADY imported. Importing it here would drag a heavy module
+ # into every test that never asked for it, which perturbed source-contract and
+ # import-order tests elsewhere; and the latch cannot have been set without the
+ # module being loaded, so there is nothing to miss.
+ mod = sys.modules.get("routes.inference")
+ if mod is not None:
+ try:
+ mod.begin_load_lifecycle()
+ except Exception:
+ pass
+
+ _reopen()
+ try:
+ yield
+ finally:
+ _reopen()
diff --git a/studio/backend/tests/test_gpu_init_crash_message.py b/studio/backend/tests/test_gpu_init_crash_message.py
index 929d5251837..7f9139a6985 100644
--- a/studio/backend/tests/test_gpu_init_crash_message.py
+++ b/studio/backend/tests/test_gpu_init_crash_message.py
@@ -1072,7 +1072,7 @@ def test_the_exit_handler_removes_the_runtime_after_the_kill():
cannot delete a runtime whose server is still holding the files open."""
backend = LlamaCppBackend()
order = []
- backend._kill_process = lambda: order.append("kill")
+ backend._kill_process = lambda **_kw: order.append("kill")
backend._cleanup_cpu_fallback_runtime = lambda: order.append("clean")
backend._cleanup()
diff --git a/studio/backend/tests/test_llama_cpp_atexit_quiet.py b/studio/backend/tests/test_llama_cpp_atexit_quiet.py
index 2595dad39c2..48ce8fb35ee 100644
--- a/studio/backend/tests/test_llama_cpp_atexit_quiet.py
+++ b/studio/backend/tests/test_llama_cpp_atexit_quiet.py
@@ -131,13 +131,22 @@ def test_the_atexit_handler_quiets_stdlib_loggers_too(monkeypatch, capsys):
other.addHandler(handler)
other.propagate = False
- def kill_and_log():
+ ran = []
+
+ # **_kw because _cleanup calls _kill_process(teardown = True). A double that only
+ # accepts () raises TypeError inside a handler that swallows everything, so the
+ # write this test exists to make would never happen and nothing would say so.
+ def kill_and_log(**_kw):
+ ran.append(1)
other.warning("something a dependency logs at exit")
backend = _stub()
monkeypatch.setattr(backend, "_kill_process", kill_and_log)
try:
backend._cleanup()
+ # _cleanup swallows everything, so a double whose signature stops
+ # matching would silently skip the write this test exists to make.
+ assert ran, "the kill double never ran; this assertion proves nothing"
assert capsys.readouterr().err == ""
finally:
other.removeHandler(handler)
@@ -220,9 +229,15 @@ def test_a_failing_kill_does_not_escape_the_atexit_handler(monkeypatch):
"""atexit swallows it anyway, and there is nowhere left to report it."""
backend = _stub()
- def boom():
+ raised = []
+
+ def boom(**_kw):
+ raised.append(1)
raise RuntimeError("teardown went wrong")
monkeypatch.setattr(backend, "_kill_process", boom)
backend._cleanup()
+ # Same trap: a TypeError from a stale signature is swallowed too, and then
+ # the RuntimeError this test is about is never raised at all.
+ assert raised, "the failing kill never ran; the handler swallowed the wrong error"
diff --git a/studio/backend/tests/test_llama_cpp_wait_for_health.py b/studio/backend/tests/test_llama_cpp_wait_for_health.py
index 7eb2493e36e..723b1f6d7b8 100644
--- a/studio/backend/tests/test_llama_cpp_wait_for_health.py
+++ b/studio/backend/tests/test_llama_cpp_wait_for_health.py
@@ -11,6 +11,7 @@ subprocess.poll() branch so a crashed llama-server surfaces a structured
from __future__ import annotations
+import subprocess
import sys
import threading
import time
@@ -34,7 +35,6 @@ import httpx # noqa: E402
from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
from core.inference.llama_cpp import GgufLoadIntent
-import subprocess
# Sibling tests install lightweight httpx stubs, so when collected together our `httpx`
# may be a stub lacking `get`. Fill in the gaps so collection order does not matter.
@@ -167,6 +167,233 @@ class TestWaitForHealthResilience:
t.join(5.0)
assert b._health_wait_cancelled is True
+ def test_a_signalled_exit_during_teardown_is_not_a_startup_crash(self, monkeypatch):
+ """_kill_process holds the reference across its terminate/wait, so for up to
+ ~10s the loop sees a populated process whose poll() is the SIGTERM code. The
+ None check alone only covers teardown AFTER cleanup finished, so the kill
+ publishes its terminal state before signalling; otherwise -15 reads as a
+ startup crash and the retry ladder respawns a server during shutdown."""
+ b = _make_backend()
+ b._process.poll.return_value = None
+
+ def probe(*a, **kw):
+ # The shutdown thread mid-wait: publish, then signal, reference still
+ # set. Here rather than before the call because the wait resets on
+ # entry, so only a teardown landing DURING a wait is this case.
+ b._torn_down_process = b._process
+ b._process.poll.return_value = -15
+ b._process.returncode = -15
+ return mock.Mock(status_code = 503)
+
+ monkeypatch.setattr(httpx, "get", probe)
+ with mock.patch("core.inference.llama_cpp.logger") as log:
+ assert b._wait_for_health(timeout = 1.0, interval = 0.01) is False
+ assert b._health_wait_cancelled is True
+ assert not any("exited with code" in str(c) for c in log.error.call_args_list)
+
+ def test_a_teardown_between_the_spawn_and_the_wait_still_counts(self, monkeypatch):
+ """Shutdown can land after Popen publishes the child but before the wait
+ starts. A marker reset on wait entry would erase it, and the loop would
+ then read the signalled exit as a startup crash and let the fallbacks
+ respawn during shutdown, which is the race this whole change is about.
+ Keyed to the process, so it survives until the wait that owns it reads it."""
+ b = _make_backend()
+ b._torn_down_process = b._process # teardown, before the wait is entered
+ b._process.poll.return_value = -15
+ b._process.returncode = -15
+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))
+ with mock.patch("core.inference.llama_cpp.logger") as log:
+ assert b._wait_for_health(timeout = 1.0, interval = 0.01) is False
+ assert b._health_wait_cancelled is True
+ assert not any("exited with code" in str(c) for c in log.error.call_args_list)
+
+ def test_shutdown_is_terminal_however_late_it_lands(self, monkeypatch):
+ """The durable half of the guard. Every per-process snapshot has an instant
+ after it where teardown can still land, so this flag only goes false to
+ true and is re-read each iteration; a wait in flight when shutdown begins
+ ends terminally no matter where in the loop the flag was set."""
+ b = _make_backend()
+ b._process.poll.return_value = None
+
+ def probe(*a, **kw):
+ b._shutting_down = True # run.py's shutdown, at an arbitrary instant
+ return mock.Mock(status_code = 503)
+
+ monkeypatch.setattr(httpx, "get", probe)
+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False
+ assert b._health_wait_cancelled is True
+ assert not any("health check timed out" in ln for ln in b._stdout_lines)
+
+ def test_kill_process_marks_shutdown_only_for_a_teardown(self):
+ """The flag must not be set by the retry ladder reaping its own child, or
+ the first crash retry would look like a shutdown and abort the load."""
+ for teardown in (True, False):
+ b = _make_backend()
+ b._stop_mtp_crash_watchdog = lambda *a, **kw: None
+ b._reset_effective_parallel_slots = lambda *a, **kw: None
+ b._leading_process_group = lambda *a, **kw: None
+ b._collect_descendants = lambda *a, **kw: []
+ b._kill_process_group = lambda *a, **kw: None
+ b._terminate_descendants = lambda *a, **kw: None
+ b._process.poll.return_value = 0
+ b._kill_process(teardown = teardown)
+ assert getattr(b, "_shutting_down", False) is teardown
+
+ def test_a_started_shutdown_refuses_to_spawn(self, monkeypatch):
+ """_start_llama_process is the chokepoint the mmproj text-only retry uses
+ without passing _spawn_and_wait's boundary check, so it refuses too, and
+ says so: a silent refusal leaves the caller health-waiting on the previous
+ child and then reading a reference the teardown is clearing."""
+ import subprocess
+
+ b = _make_backend()
+ b._shutting_down = True
+ spawned = []
+ monkeypatch.setattr(subprocess, "Popen", lambda *a, **kw: spawned.append(1))
+ assert b._start_llama_process(["llama-server"], {}, child_gpu_physical_ids = None) is False
+ assert spawned == [], "started a server after shutdown had begun"
+
+ def test_a_teardown_with_no_process_still_marks_shutdown(self):
+ """Quitting during a download or staging has nothing to kill, so
+ _kill_process returns early. Marking teardown only on the path that has a
+ process let the load spawn a server after the shutdown sweep finished."""
+ b = _make_backend()
+ b._process = None
+ b._stop_mtp_crash_watchdog = lambda *a, **kw: None
+ b._reset_effective_parallel_slots = lambda *a, **kw: None
+ b._kill_process(teardown = True)
+ assert b._shutting_down is True
+
+ def test_the_published_child_survives_post_spawn_setup(self, monkeypatch):
+ """Shutdown can take the spawn lock the instant it is released and clear
+ the reference, so post-spawn setup reads the Popen it just made rather than
+ self._process. Otherwise recording the pid raises the same AttributeError
+ this PR exists to remove."""
+ import subprocess
+
+ b = _make_backend()
+ b._shutting_down = False
+ recorded = []
+ b._record_server_pid = lambda pid: recorded.append(pid)
+ b._drain_stdout = lambda *a, **kw: None
+ b._llama_log_fh = None
+ b._process = None
+ proc = mock.Mock()
+ proc.pid = 4242
+ monkeypatch.setattr(subprocess, "Popen", lambda *a, **kw: proc)
+
+ class _TeardownWinsTheLock:
+ """Shutdown taking the lock the instant the spawn releases it, which is
+ the whole window: it clears the reference before the pid is read."""
+
+ def __enter__(self):
+ return None
+
+ def __exit__(self, *_exc):
+ b._process = None
+ return False
+
+ b._spawn_lock = _TeardownWinsTheLock()
+ assert b._start_llama_process(["llama-server"], {}, child_gpu_physical_ids = None) is True
+ assert recorded == [4242]
+
+ def test_a_new_server_lifecycle_clears_the_shutdown_state(self):
+ """The backend is a module singleton and an embedded host may call
+ run_server() again in the same process, so this state is scoped to a
+ lifecycle. Left latched, every launch of the second session is refused."""
+ b = _make_backend()
+ b._shutting_down = True
+ b._torn_down_process = b._process
+ b._begin_server_lifecycle()
+ assert b._shutting_down is False
+ assert b._torn_down_process is None
+
+ def test_a_healthy_probe_during_shutdown_is_not_a_healthy_server(self, monkeypatch):
+ """The probe blocks for up to 2s. A 200 arriving as teardown begins must not
+ publish a child the shutdown is already killing, exactly as for a cancel."""
+ b = _make_backend()
+ b._process.poll.return_value = None
+
+ def probe(*a, **kw):
+ b._shutting_down = True
+ return mock.Mock(status_code = 200)
+
+ monkeypatch.setattr(httpx, "get", probe)
+ assert b._wait_for_health(timeout = 5.0, interval = 0.01) is False
+ assert b._health_wait_cancelled is True
+
+ def test_a_teardown_during_the_last_probe_is_not_reported_as_a_timeout(self, monkeypatch):
+ """The deadline is the other way out of the loop. A teardown landing in the
+ final probe leaves no iteration to notice it, so without a check here the
+ wait returns a plain timeout, the caller sees something retryable, and the
+ fallbacks respawn after shutdown killed the first server."""
+ b = _make_backend()
+ b._process.poll.return_value = None
+
+ def probe(*a, **kw):
+ b._torn_down_process = b._process # shutdown, during the last probe
+ return mock.Mock(status_code = 503)
+
+ monkeypatch.setattr(httpx, "get", probe)
+ assert b._wait_for_health(timeout = 0.02, interval = 0.01) is False
+ assert b._health_wait_cancelled is True
+ # Not a live-but-never-healthy load, so it must not be classified as one.
+ assert not any("health check timed out" in ln for ln in b._stdout_lines)
+
+ def test_a_timeout_with_no_teardown_still_reports_a_timeout(self, monkeypatch):
+ """The check above must not swallow the ordinary #5740 classification."""
+ b = _make_backend()
+ b._process.poll.return_value = None
+ b._torn_down_process = mock.Mock() # an earlier child, unrelated
+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))
+ assert b._wait_for_health(timeout = 0.02, interval = 0.01) is False
+ assert b._health_wait_cancelled is False
+ assert any("health check timed out" in ln for ln in b._stdout_lines)
+
+ def test_an_earlier_childs_teardown_does_not_end_a_later_wait(self, monkeypatch):
+ """The other half of keying on identity: the marker names one child, so a
+ load that replaced a torn-down one is not aborted by its predecessor."""
+ b = _make_backend()
+ b._torn_down_process = mock.Mock() # the previous child, already reaped
+ b._process.poll.return_value = 1 # this one really did crash
+ b._process.returncode = 1
+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))
+ with mock.patch("core.inference.llama_cpp.logger") as log:
+ assert b._wait_for_health(timeout = 1.0, interval = 0.01) is False
+ assert b._health_wait_cancelled is False, "a stale teardown aborted a live load"
+ assert any("exited with code 1" in str(c) for c in log.error.call_args_list)
+
+ @pytest.mark.parametrize("teardown", [True, False])
+ def test_kill_process_publishes_a_teardown_before_it_signals(self, teardown):
+ """The ordering the test above depends on: published before terminate(),
+ not in the finally that clears the reference.
+
+ Only for a teardown. The retry ladder reaps a crashed child through this
+ same method between attempts, and marking that terminal would abort loads
+ the --fit off and CPU fallbacks currently recover."""
+ b = _make_backend()
+ b._torn_down_process = None
+ b._stop_mtp_crash_watchdog = lambda *a, **kw: None
+ b._reset_effective_parallel_slots = lambda *a, **kw: None
+ b._leading_process_group = lambda *a, **kw: None
+ b._collect_descendants = lambda *a, **kw: []
+ b._kill_process_group = lambda *a, **kw: None
+ b._terminate_descendants = lambda *a, **kw: None
+ seen = {}
+
+ def _terminate():
+ # What a racing _wait_for_health would observe at this instant.
+ seen["flag_at_signal"] = getattr(b, "_torn_down_process", None) is b._process
+ seen["process_still_set"] = b._process is not None
+
+ b._process.terminate = _terminate
+ b._process.poll.return_value = -15
+ b._kill_process(teardown = teardown)
+ assert seen["process_still_set"] is True, "the window this guards would not exist"
+ assert (
+ seen["flag_at_signal"] is teardown
+ ), "teardown published late, or a reap marked terminal"
+
def test_a_crash_still_reports_the_exit_code(self, monkeypatch):
"""The teardown guard must not swallow the crash branch: an exited
process still has to name its exit code and its output."""
@@ -856,3 +1083,505 @@ def test_a_cancel_after_audio_setup_unloads_the_codec(tmp_path, monkeypatch):
codec.unload.assert_called_once_with()
assert LlamaCppBackend._codec_mgr is None
assert emptied == [1]
+
+
+def _run_server_body() -> str:
+ """run_server's source, without importing run.py.
+
+ Importing it pulls in fastapi and the whole route tree, which is exactly the
+ cost the ordering below exists to keep out of the early startup path.
+ """
+ import ast
+
+ run_py = Path(__file__).resolve().parent.parent / "run.py"
+ tree = ast.parse(run_py.read_text(encoding = "utf-8"))
+ for node in tree.body:
+ if isinstance(node, ast.FunctionDef) and node.name == "run_server":
+ return ast.get_source_segment(run_py.read_text(encoding = "utf-8"), node) or ""
+ raise AssertionError("run_server not found in run.py")
+
+
+def test_the_lifecycle_reset_does_not_import_the_route_module_early():
+ """The reset must not be what first builds the llama-server backend.
+
+ `from routes.inference import ...` constructs the module singleton, which
+ sweeps orphan llama-servers and registers an atexit handler. run_server
+ documents five things as having to happen before any of that: the Windows
+ UTF-8 reconfigure, the session log that catches import-time crashes, the
+ structlog setup (whose cache_logger_on_first_use pins any logger that has
+ already emitted), initialize_parent_lifetime() before a child can spawn, and
+ write_startup_marker() before a sweep can reap a sibling's server.
+
+ Ordering it after `from main import app` makes the import free: main imports
+ the route package, so the singleton already exists and this is a lookup.
+ """
+ body = _run_server_body()
+
+ main_import = body.index("from main import app")
+ reset = body.index("_begin_server_lifecycle()")
+ assert reset > main_import, (
+ "the lifecycle reset imports routes.inference before `from main import app`, "
+ "so it, not main, builds the backend singleton -- ahead of the startup steps "
+ "run_server requires to come first"
+ )
+
+ for earlier in (
+ 'sys.stdout.reconfigure(encoding = "utf-8"',
+ "_setup_server_disk_logging()",
+ "LogConfig.setup_logging(",
+ "initialize_parent_lifetime()",
+ "write_startup_marker()",
+ ):
+ assert body.index(earlier) < reset, f"{earlier} must precede the lifecycle reset"
+
+
+def test_the_lifecycle_reset_stays_below_the_argument_checks():
+ """A rejected invocation must not touch the backend at all."""
+ body = _run_server_body()
+
+ assert body.index("choose an explicit port.") < body.index(
+ "_begin_server_lifecycle()"
+ ), "the reset runs before run_server has finished rejecting bad arguments"
+
+
+def test_the_lifecycle_reset_is_the_last_thing_before_the_serve():
+ """Clearing the shutdown flag is what lets a spawn through, so nothing that can
+ abort startup may follow it.
+
+ An embedded host restarting into an occupied port (_resolve_port) or a missing
+ frontend (SystemExit) would otherwise leave _shutting_down False with the
+ previous session's still-unwinding load free to start a child the shutdown
+ sweep has already run past.
+ """
+ body = _run_server_body()
+ # Statements, not text: the comments around these calls name them too, and
+ # matching the prose made this compare the wrong offsets.
+ reset = body.index("_llama_cpp_backend._begin_server_lifecycle()")
+ serve = body.index("\n thread.start()")
+ assert reset < serve, "the lifecycle reset no longer precedes the serve"
+
+ # Only the window between the reset and the serve. The sys.exit further down is
+ # a startup FAILURE after the thread is running, by which point a spawn is
+ # legitimate, so the whole tail is the wrong scope.
+ #
+ # sys.exit( as well as raise SystemExit: the first version of this test checked
+ # only the latter and missed the admin-password gate, which exits the other way.
+ window = body[reset:serve]
+ for fail_fast in ("raise SystemExit", "sys.exit(", "_resolve_port("):
+ assert fail_fast not in window, (
+ f"{fail_fast} runs after the lifecycle reset, so a failed restart leaves the "
+ "spawn guard cleared"
+ )
+
+
+class TestARefusedSpawnClosesItsLog:
+ """Each spawn opens a per-attempt tee log just before Popen. A refusal returns
+ without a process, and _kill_process returns early when there is none, so
+ nothing else ever closes it: the next attempt overwrites the attribute, leaking
+ the descriptor and holding the file lock on Windows."""
+
+ def _backend(self, tmp_path):
+ b = _make_backend()
+ b._shutting_down = True
+ b._spawn_lock = threading.Lock()
+ b._stop_mtp_crash_watchdog = lambda: None
+ b._process = None
+ b._redacted_cmd_for_log = lambda c: c
+ b._llama_log_path = str(tmp_path / "llama.log")
+ b._llama_log_fh = None
+ return b
+
+ def test_start_llama_process_closes_it(self, tmp_path):
+ """The handle under test is the one the method opens for itself, not one
+ set beforehand: _start_llama_process clears the attribute and opens a fresh
+ per-attempt log before it reaches the shutdown check."""
+ b = self._backend(tmp_path)
+ opened = []
+ real_open = open
+
+ def tracking_open(*a, **k):
+ fh = real_open(*a, **k)
+ opened.append(fh)
+ return fh
+
+ import builtins
+
+ with mock.patch.object(builtins, "open", tracking_open):
+ assert (
+ b._start_llama_process(["llama-server"], {}, child_gpu_physical_ids = None) is False
+ )
+
+ assert opened, "the method never opened an attempt log; this proves nothing"
+ assert all(fh.closed for fh in opened), "the refused attempt left its tee log open"
+ assert b._llama_log_fh is None
+
+ def test_the_kill_path_does_not_cover_it(self, tmp_path):
+ """Why the explicit close is needed: _kill_process is the only other closer
+ and it returns before that, since a refusal published no process."""
+ b = self._backend(tmp_path)
+ fh = open(tmp_path / "kill.log", "w", encoding = "utf-8")
+ b._llama_log_fh = fh
+ b._reset_effective_parallel_slots = lambda: None
+
+ b._kill_process(teardown = True)
+
+ assert not fh.closed, (
+ "_kill_process now closes the handle with no process; if that is "
+ "deliberate this test and the explicit close are both redundant"
+ )
+ fh.close()
+
+
+def test_the_pid_is_recorded_before_the_spawn_lock_is_released():
+ """Teardown takes the lock the instant publication ends. Recording the pid after
+ that would write it back behind a sweep that just forgot it, and
+ _pid_start_identity cannot read a start time for a reaped process -- so the
+ record is a bare pid, which a later launch kills without an identity check."""
+ b = _make_backend()
+ b._shutting_down = False
+ b._stop_mtp_crash_watchdog = lambda: None
+ b._process = None
+ b._redacted_cmd_for_log = lambda c: c
+ b._llama_log_path = None
+ b._llama_log_fh = None
+ order = []
+
+ class _Lock:
+ def __enter__(self):
+ order.append("lock")
+ return None
+
+ def __exit__(self, *_exc):
+ order.append("unlock")
+ return False
+
+ b._spawn_lock = _Lock()
+ b._record_server_pid = lambda pid: order.append("record")
+
+ class _Proc:
+ pid = 4242
+
+ def poll(self):
+ return None
+
+ with mock.patch.object(subprocess, "Popen", lambda *a, **k: _Proc()):
+ with mock.patch.object(threading, "Thread", lambda **k: mock.Mock()):
+ assert b._start_llama_process(["llama-server"], {}, child_gpu_physical_ids = None) is True
+
+ assert order.index("record") < order.index("unlock"), (
+ f"the pid is recorded after the lock is released ({order}), so a teardown "
+ "between the two writes a bare pid record for an already-reaped process"
+ )
+
+
+class TestHealthPublicationIsAtomicWithTeardown:
+ """A successful probe and the _healthy commit are separate steps, so a teardown
+ landing between them left the backend advertising a model whose child had
+ already been killed. The recheck inside the wait narrows that window; only
+ taking the same lock the teardown mark is set under closes it."""
+
+ def _backend(self):
+ b = _make_backend()
+ b._stop_mtp_crash_watchdog = lambda: None
+ b._healthy = False
+ b._spawn_lock = threading.Lock()
+ return b
+
+ def test_a_teardown_before_the_commit_refuses_publication(self):
+ b = self._backend()
+ b._shutting_down = True
+
+ assert b._publish_healthy() is False
+ assert b._healthy is False, "a torn-down backend was published as healthy"
+
+ def test_an_ordinary_load_still_publishes(self):
+ b = self._backend()
+ b._shutting_down = False
+
+ assert b._publish_healthy() is True
+ assert b._healthy is True
+
+ def test_the_teardown_mark_and_the_commit_cannot_interleave(self):
+ """The two orders the lock permits are the only safe ones: publish then
+ teardown (which clears _healthy on its way out), or teardown then a refused
+ publish. This drives the first order and asserts the second is impossible."""
+ b = self._backend()
+ b._shutting_down = False
+ # A real child, not None: _kill_process returns before clearing _healthy when
+ # there is nothing to kill, and a backend that published healthy by
+ # definition has a process, so None would be testing the wrong ordering.
+ b._process = object()
+ b._reset_effective_parallel_slots = lambda: None
+ b._leading_process_group = lambda _pid: None
+ b._collect_descendants = lambda _pid: []
+ b._diffusion_requested_ngl = None
+
+ published = b._publish_healthy()
+ assert published is True and b._healthy is True
+
+ b._kill_process(teardown = True)
+
+ assert (
+ b._healthy is False
+ ), "teardown left _healthy set, so a publication that won the race is never undone"
+ assert b._publish_healthy() is False, "a later publication slipped past the teardown"
+
+
+def test_the_deadline_asks_the_durable_flag_too(monkeypatch):
+ """_kill_process publishes its two signals apart: _shutting_down under the
+ spawn lock on entry, _torn_down_process only after collecting descendants. A
+ wait whose deadline lands between them saw no marker and recorded a plain
+ timeout, sending a deliberate teardown through startup-failure handling."""
+ b = _make_backend()
+ b._process.poll.return_value = None
+ b._shutting_down = False
+ b._torn_down_process = None
+
+ # Set during the LAST probe, not before the wait: setting it up front is caught
+ # by the top-of-iteration check and never reaches the deadline at all, so that
+ # version of this test passed with or without the fix.
+ def probe(*a, **kw):
+ b._shutting_down = True # teardown, after the final iteration's check
+ return mock.Mock(status_code = 503)
+
+ monkeypatch.setattr(httpx, "get", probe)
+
+ assert b._wait_for_health(timeout = 0.01, interval = 0.05) is False
+ assert b._health_wait_cancelled is True, "a teardown was recorded as a timeout"
+ assert not any("health check timed out" in ln for ln in b._stdout_lines)
+
+
+class TestAStaleLoadIsDroppedAtTheSerialScope:
+ """Refusing a previous lifecycle's load only at the spawn is too late.
+
+ A lock gives a waiter no priority, so the stale load can take the serial scope
+ after the restart, and the duplicate-adoption phase inside it calls
+ _kill_process() on whatever is loaded -- by then the NEW lifecycle's model. It
+ would unload the restarted server and then decline to replace it.
+ """
+
+ def test_the_check_precedes_the_replacement_work(self):
+ """Checked over the AST, not the text: a first version matched the words
+ '_kill_process' in the explanatory comment above the guard and failed on
+ correct code."""
+ import ast
+ import inspect
+ import textwrap
+
+ tree = ast.parse(textwrap.dedent(inspect.getsource(LlamaCppBackend.load_model)))
+ scope = next(
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.With)
+ and any("_serial_load_scope" in ast.unparse(i.context_expr) for i in n.items)
+ )
+
+ guard_line = next(
+ n.lineno
+ for n in ast.walk(scope)
+ if isinstance(n, ast.Call) and "_spawn_is_stale" in ast.unparse(n.func)
+ )
+
+ early_kills = [
+ n.lineno
+ for n in ast.walk(scope)
+ if isinstance(n, ast.Call)
+ and "_kill_process" in ast.unparse(n.func)
+ and n.lineno < guard_line
+ ]
+ assert not early_kills, (
+ f"a stale load calls _kill_process at {early_kills}, before it is refused at "
+ f"line {guard_line}, so it unloads the new lifecycle's model"
+ )
+
+ # And the refusal has to stop the load rather than just log it.
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ stale = src.index("_spawn_is_stale()")
+ assert "return False" in src[stale : stale + 300], "the stale-load branch does not return"
+
+
+def test_a_lifecycle_cannot_reopen_while_a_teardown_is_still_killing():
+ """_kill_process(teardown) used to release the lock as soon as it set the flag,
+ but it goes on reading self._process and finally clears it. An embedded host
+ that saw the server thread stop could reopen the lifecycle in that gap, let a
+ new load spawn, and have this teardown drop or terminate its child."""
+ b = _make_backend()
+ b._stop_mtp_crash_watchdog = lambda: None
+ b._reset_effective_parallel_slots = lambda: None
+ b._diffusion_requested_ngl = None
+ b._leading_process_group = lambda _pid: None
+ b._collect_descendants = lambda _pid: []
+ b._spawn_lock = threading.RLock() # so the probe below can observe, not deadlock
+
+ reopened_during_kill = []
+ in_kill = threading.Event()
+ release = threading.Event()
+
+ class _SlowProcess:
+ pid = 999
+
+ def terminate(self):
+ in_kill.set()
+ release.wait(2.0)
+
+ def wait(self, timeout = None):
+ return 0
+
+ def kill(self):
+ pass
+
+ b._process = _SlowProcess()
+
+ def restarter():
+ in_kill.wait(2.0)
+ # _teardown_lock, not _spawn_lock: the kill holds the former for its whole
+ # duration and the latter only long enough to set the flag, so a spawn can
+ # be refused promptly. This probes the lock that carries the property.
+ got = b._teardown_lock.acquire(blocking = False)
+ if got:
+ b._teardown_lock.release()
+ reopened_during_kill.append(got)
+ release.set()
+
+ t = threading.Thread(target = restarter)
+ t.start()
+ try:
+ b._kill_process(teardown = True)
+ finally:
+ release.set()
+ t.join(5.0)
+
+ assert reopened_during_kill == [False], (
+ "the teardown lock was free while the kill was still running, so "
+ "_begin_server_lifecycle could reopen the lifecycle mid-kill"
+ )
+
+
+def test_the_shutdown_cancels_loads_before_it_kills_the_server():
+ """Order matters: killing first leaves the load to respawn into the teardown."""
+ import ast
+ import textwrap
+ from pathlib import Path
+
+ run_py = (Path(__file__).resolve().parent.parent / "run.py").read_text(encoding = "utf-8")
+ tree = ast.parse(run_py)
+ fn = next(
+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_graceful_shutdown"
+ )
+ src = textwrap.dedent(ast.get_source_segment(run_py, fn) or "")
+
+ assert "cancel_pending_loads()" in src, "shutdown does not cancel in-flight loads"
+ assert src.index("cancel_pending_loads()") < src.index(
+ "_kill_process(teardown = True)"
+ ), "the loads are cancelled after the kill, so one can still spawn into the teardown"
+
+
+class TestATeardownDoesNotBlockASpawnItWillRefuse:
+ """The kill needs a long hold so a lifecycle cannot reopen mid-terminate, but a
+ spawn only needs to read the flag. Holding one lock for both made a spawn queue
+ behind a SIGTERM/SIGKILL escalation for seconds before being told no, on a
+ thread shutdown is already waiting for."""
+
+ def _backend(self, on_terminate):
+ b = _make_backend()
+ b._stop_mtp_crash_watchdog = lambda: None
+ b._reset_effective_parallel_slots = lambda: None
+ b._diffusion_requested_ngl = None
+ b._leading_process_group = lambda _p: None
+ b._collect_descendants = lambda _p: []
+
+ class _Stubborn:
+ pid = 4242
+
+ def __init__(self):
+ self.killed = False
+
+ def terminate(self):
+ on_terminate()
+
+ def wait(self, timeout = None):
+ if not self.killed:
+ raise subprocess.TimeoutExpired("llama-server", timeout)
+
+ def kill(self):
+ self.killed = True
+
+ b._process = _Stubborn()
+ return b
+
+ def test_a_spawn_is_refused_without_waiting_for_the_kill(self):
+ in_kill = threading.Event()
+ release = threading.Event()
+ b = self._backend(lambda: (in_kill.set(), release.wait(5.0)))
+ seen = []
+
+ def spawner():
+ in_kill.wait(5.0)
+ start = time.monotonic()
+ with b._spawn_lock:
+ stale = b._spawn_is_stale()
+ seen.append((time.monotonic() - start, stale))
+ release.set()
+
+ t = threading.Thread(target = spawner)
+ t.start()
+ try:
+ b._kill_process(teardown = True)
+ finally:
+ release.set()
+ t.join(10)
+
+ waited, refused = seen[0]
+ assert refused is True, "the spawn was allowed through during a teardown"
+ assert waited < 1.0, (
+ f"the spawn waited {waited:.2f}s for the kill before being refused; the "
+ "teardown hold and the flag read must not share one lock"
+ )
+
+ def test_a_lifecycle_reset_still_waits_for_the_kill(self):
+ """The other half, and the reason the long hold exists at all: clearing the
+ flag mid-kill would let a new load spawn a child this teardown then drops."""
+ in_kill = threading.Event()
+ release = threading.Event()
+ b = self._backend(lambda: (in_kill.set(), release.wait(2.0)))
+ order = []
+
+ def resetter():
+ in_kill.wait(5.0)
+ b._begin_server_lifecycle()
+ order.append("reset")
+
+ t = threading.Thread(target = resetter)
+ t.start()
+ try:
+ b._kill_process(teardown = True)
+ order.append("kill_done")
+ finally:
+ release.set()
+ t.join(10)
+
+ assert order == [
+ "kill_done",
+ "reset",
+ ], f"the lifecycle reopened before the kill finished: {order}"
+
+
+def test_the_lock_order_is_teardown_then_spawn():
+ """Two locks means an order, and taking them the other way round deadlocks."""
+ import ast
+ import inspect
+
+ tree = ast.parse(inspect.getsource(LlamaCppBackend))
+ inversions = []
+ for node in ast.walk(tree):
+ if isinstance(node, ast.With) and any(
+ "_spawn_lock" in ast.unparse(i.context_expr) for i in node.items
+ ):
+ for inner in ast.walk(node):
+ if inner is node:
+ continue
+ if isinstance(inner, ast.With) and any(
+ "_teardown_lock" in ast.unparse(i.context_expr) for i in inner.items
+ ):
+ inversions.append(inner.lineno)
+ assert not inversions, f"_teardown_lock taken inside _spawn_lock at {inversions}"
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 64eff7ea9a8..0e50e18a0f6 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -8542,7 +8542,12 @@ def test_unload_clears_the_user_load_flag():
assert backend._loaded_by_user_action is False
# Not cleared on a bare process kill: a respawn or MTP-free reload replays
# the same load and must keep the provenance it had.
- assert "_loaded_by_user_action" not in inspect.getsource(LlamaCppBackend._kill_process)
+ # Both halves, or this negative assertion passes vacuously: the body it is
+ # really about now lives in _kill_process_body.
+ assert "_loaded_by_user_action" not in (
+ inspect.getsource(LlamaCppBackend._kill_process)
+ + inspect.getsource(LlamaCppBackend._kill_process_body)
+ )
def test_the_load_route_pins_and_other_load_surfaces_do_not(monkeypatch):
diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py
index d3da92e74a6..40aaf8569e2 100644
--- a/studio/backend/tests/test_orchestrator_unload_cancel.py
+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py
@@ -2634,3 +2634,851 @@ def test_a_scoped_load_cancel_that_never_reports_back_releases_the_load():
with inf._scoped_load_attempts_lock:
inf._scoped_load_attempts.clear()
inf._scoped_load_cancel_tombstones.clear()
+
+
+def test_shutdown_cancels_loads_that_have_not_reached_the_backend():
+ """A /load between admission and the backend call holds nothing the backend's
+ shutdown flag can see: it can sit in the lifecycle gate or preflight for
+ minutes and then arrive in a lifecycle that has already been reset, loading a
+ model the new server never asked for. _graceful_shutdown cancels through the
+ attempt's own event, the same path /unload uses.
+ """
+ import importlib
+
+ inf = importlib.import_module("routes.inference")
+
+ def _attempt(token, path):
+ return inf._ScopedLoadAttempt(
+ token = token,
+ request_id = None,
+ model_path = path,
+ subject = "s",
+ cancel_event = threading.Event(),
+ cancel_complete = threading.Event(),
+ )
+
+ pending = _attempt("pending-token", "owner/model")
+ running = _attempt("running-token", "owner/other")
+
+ with inf._scoped_load_attempts_lock:
+ inf._pending_load_attempts[pending.token] = pending
+ prior_running = inf._running_load_attempt
+ inf._running_load_attempt = running
+ try:
+ assert inf.cancel_pending_loads() == 2
+ assert pending.cancel_event.is_set(), "a queued load survived the shutdown"
+ assert running.cancel_event.is_set(), "the running load survived the shutdown"
+ finally:
+ inf._running_load_attempt = prior_running
+ with inf._scoped_load_attempts_lock:
+ inf._pending_load_attempts.pop(pending.token, None)
+
+
+def test_a_running_attempt_already_in_the_pending_map_is_not_counted_twice():
+ import importlib
+
+ inf = importlib.import_module("routes.inference")
+
+ both = inf._ScopedLoadAttempt(
+ token = "same-token",
+ request_id = None,
+ model_path = "owner/model",
+ subject = "s",
+ cancel_event = threading.Event(),
+ cancel_complete = threading.Event(),
+ )
+ with inf._scoped_load_attempts_lock:
+ inf._pending_load_attempts[both.token] = both
+ prior_running = inf._running_load_attempt
+ inf._running_load_attempt = both
+ try:
+ assert inf.cancel_pending_loads() == 1
+ assert both.cancel_event.is_set()
+ finally:
+ inf._running_load_attempt = prior_running
+ with inf._scoped_load_attempts_lock:
+ inf._pending_load_attempts.pop(both.token, None)
+
+
+def _mk_attempt(
+ inf,
+ token,
+ path = "owner/model",
+):
+ return inf._ScopedLoadAttempt(
+ token = token,
+ request_id = None,
+ model_path = path,
+ subject = "s",
+ cancel_event = threading.Event(),
+ cancel_complete = threading.Event(),
+ )
+
+
+def test_shutdown_closes_the_cancel_handshake_itself():
+ """Only /unload sets cancel_complete, and at shutdown there is none, so setting
+ cancel_event alone leaves _run_tracked_load_model_impl's finally waiting the full
+ handshake timeout in a to_thread. Those executor threads are non-daemon and hold
+ the process open, which is what the comment above the timeout warns about."""
+ import importlib
+
+ inf = importlib.import_module("routes.inference")
+ attempt = _mk_attempt(inf, "handshake-token")
+ with inf._scoped_load_attempts_lock:
+ inf._pending_load_attempts[attempt.token] = attempt
+ try:
+ inf.cancel_pending_loads()
+ assert attempt.cancel_event.is_set()
+ assert attempt.cancel_complete.is_set(), (
+ "shutdown left the handshake open, so the load waits the full timeout in a "
+ "non-daemon executor thread and delays process exit"
+ )
+ finally:
+ inf.begin_load_lifecycle()
+ with inf._scoped_load_attempts_lock:
+ inf._pending_load_attempts.pop(attempt.token, None)
+
+
+def test_a_load_registering_after_the_sweep_is_still_cancelled():
+ """uvicorn's should_exit stops new connections, not request tasks it already
+ admitted, so a /load can register after the shutdown snapshot was taken and
+ would otherwise never be cancelled by anything."""
+ import importlib
+
+ inf = importlib.import_module("routes.inference")
+ try:
+ assert inf.cancel_pending_loads() == 0 # latches with nothing to cancel
+
+ late = _mk_attempt(inf, "late-token")
+ with inf._scoped_load_attempts_lock:
+ inf._pending_load_attempts[late.token] = late
+ latched = inf._loads_shutting_down
+ assert latched is True, "the shutdown latch did not survive an empty sweep"
+ if latched:
+ inf._cancel_for_shutdown(late)
+ assert late.cancel_event.is_set(), "a load admitted during shutdown was not cancelled"
+ finally:
+ inf.begin_load_lifecycle()
+ with inf._scoped_load_attempts_lock:
+ inf._pending_load_attempts.pop("late-token", None)
+
+
+def test_the_latch_is_scoped_to_a_lifecycle_not_the_process():
+ """An embedded host calls run_server again. A permanent latch would refuse every
+ /load of the second session; the backend flag had exactly this bug."""
+ import importlib
+
+ inf = importlib.import_module("routes.inference")
+ try:
+ inf.cancel_pending_loads()
+ with inf._scoped_load_attempts_lock:
+ assert inf._loads_shutting_down is True
+
+ inf.begin_load_lifecycle()
+ with inf._scoped_load_attempts_lock:
+ assert (
+ inf._loads_shutting_down is False
+ ), "the second session would cancel every load it admitted"
+ finally:
+ inf.begin_load_lifecycle()
+
+
+def test_run_server_clears_the_route_latch_too():
+ import ast
+ import textwrap
+ from pathlib import Path
+
+ run_py = (Path(__file__).resolve().parent.parent / "run.py").read_text(encoding = "utf-8")
+ tree = ast.parse(run_py)
+ fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "run_server")
+ src = textwrap.dedent(ast.get_source_segment(run_py, fn) or "")
+ assert "begin_load_lifecycle()" in src, (
+ "run_server resets the backend but not the route latch, so a restarted "
+ "server cancels every load it admits"
+ )
+
+
+def _run_server_call_lines(name, *, owner = None):
+ """First line of each call to *name* inside run_server, by AST rather than text.
+
+ Text offsets kept breaking here: the comments above these calls name them too, and
+ indentation-anchored searches go stale the moment a call moves inside a `with`.
+ """
+ import ast
+ from pathlib import Path
+
+ run_py = (Path(__file__).resolve().parent.parent / "run.py").read_text(encoding = "utf-8")
+ fn = next(
+ n
+ for n in ast.parse(run_py).body
+ if isinstance(n, ast.FunctionDef) and n.name == "run_server"
+ )
+ lines = []
+ for node in ast.walk(fn):
+ if not isinstance(node, ast.Call):
+ continue
+ func = node.func
+ if isinstance(func, ast.Attribute) and func.attr == name:
+ if owner is not None and getattr(func.value, "id", None) != owner:
+ continue
+ lines.append(node.lineno)
+ elif isinstance(func, ast.Name) and func.id == name and owner is None:
+ lines.append(node.lineno)
+ assert lines, f"run_server no longer calls {name}"
+ return min(lines)
+
+
+def test_the_route_latch_clears_only_after_the_backend_lifecycle_reopens():
+ """_begin_server_lifecycle blocks on the teardown lock while a kill is running.
+ Clearing the route latch before that wait leaves a request the OLD lifecycle
+ admitted uncancelled, and it then captures the freshly advanced generation and
+ loads the previous session's model into the new one.
+
+ Nothing legitimate is refused by clearing later: uvicorn does not serve until
+ thread.start(), which is below both calls.
+ """
+ backend_reset = _run_server_call_lines("_begin_server_lifecycle")
+ route_reset = _run_server_call_lines("begin_load_lifecycle")
+ serve = _run_server_call_lines("start", owner = "thread")
+
+ assert backend_reset < route_reset, (
+ "the route latch is cleared before the backend lifecycle reopens, so a "
+ "request admitted by the old lifecycle can cross the teardown wait"
+ )
+ assert route_reset < serve, "the route latch is still set when the server starts serving"
+
+
+def _load_impl_ast():
+ """(module source, the _load_model_impl node) from routes/inference.py."""
+ import ast
+ from pathlib import Path
+
+ src = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text(
+ encoding = "utf-8"
+ )
+ fn = next(
+ n
+ for n in ast.parse(src).body
+ if isinstance(n, ast.AsyncFunctionDef) and n.name == "_load_model_impl"
+ )
+ return src, fn
+
+
+def test_the_shutdown_latch_is_enforced_in_the_load_impl_not_only_at_the_route():
+ """Auto-switch and preview await _load_model_impl directly, without ever
+ registering a _ScopedLoadAttempt, so the shutdown sweep has no event to set for
+ them. With the latch checked only where /load registers, one of those loads can
+ survive the sweep, reach a non-GGUF target after run.py already tore the
+ inference subprocess down, and spawn a worker that outlives quit.
+ """
+ import ast
+
+ src, impl = _load_impl_ast()
+
+ tracked = {"_run_tracked_load_model_impl", "_load_model_impl"}
+ direct = [
+ node.name
+ for node in ast.walk(ast.parse(src))
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
+ and node.name not in tracked
+ and any(
+ isinstance(c, ast.Call)
+ and isinstance(c.func, ast.Name)
+ and c.func.id == "_load_model_impl"
+ for c in ast.walk(node)
+ )
+ ]
+ assert direct, (
+ "no direct _load_model_impl callers left; if registration now covers every "
+ "path this guard can move back to the route"
+ )
+
+ reads = [
+ n.id for n in ast.walk(impl) if isinstance(n, ast.Name) and n.id == "_loads_shutting_down"
+ ]
+ assert reads, (
+ "_load_model_impl never consults the shutdown latch, so these direct "
+ f"callers bypass it entirely: {sorted(set(direct))}"
+ )
+
+
+def test_the_impl_cancel_check_refuses_a_load_once_shutdown_has_latched():
+ """Runs the shipped closure, rather than asserting on its text.
+
+ The callers of this helper are the load's points of no return, so refusing here
+ is what stops a shutdown-crossing load before it spawns anything.
+ """
+ import ast
+ import textwrap
+ import threading
+
+ from fastapi import HTTPException
+
+ src, impl = _load_impl_ast()
+ helper = next(
+ n
+ for n in impl.body
+ if isinstance(n, ast.FunctionDef) and n.name == "_raise_if_scoped_load_cancelled"
+ )
+ ns = {
+ "HTTPException": HTTPException,
+ "_scoped_load_attempts_lock": threading.Lock(),
+ "_loads_shutting_down": False,
+ "load_cancel_event": None,
+ }
+ exec(textwrap.dedent(ast.get_source_segment(src, helper) or ""), ns)
+ check = ns["_raise_if_scoped_load_cancelled"]
+
+ check() # nothing set: a normal load must not be refused
+
+ ns["_loads_shutting_down"] = True
+ with pytest.raises(HTTPException) as excinfo:
+ check()
+ assert excinfo.value.status_code == 409
+
+ ns["_loads_shutting_down"] = False
+ ns["load_cancel_event"] = threading.Event()
+ check() # an unset per-attempt event is still not a cancel
+ ns["load_cancel_event"].set()
+ with pytest.raises(HTTPException):
+ check()
+
+
+def test_a_second_backend_instance_is_covered_by_the_shutdown_latch():
+ """A helper/advisor load builds its OWN LlamaCppBackend (hub/utils/llm_assist.py,
+ utils/datasets/llm_assist.py). run.py only tears down the routes singleton, and
+ _shutting_down is per-instance, so that second backend would still consider a
+ spawn valid and Popen a server after terminate_all took its snapshot.
+ """
+ from core.inference.llama_cpp import LlamaCppBackend
+ from utils import process_lifetime
+
+ torn_down = LlamaCppBackend.__new__(LlamaCppBackend)
+ helper = LlamaCppBackend.__new__(LlamaCppBackend)
+ try:
+ assert helper._spawn_is_stale() is False, "a fresh backend must be able to spawn"
+
+ # What _kill_process(teardown = True) does to the singleton, without the kill.
+ torn_down._shutting_down = True
+ process_lifetime.mark_process_shutting_down()
+
+ assert (
+ helper._spawn_is_stale() is True
+ ), "a backend the shutdown never touched still thinks it may spawn"
+ finally:
+ process_lifetime.begin_process_lifecycle()
+
+ assert (
+ helper._spawn_is_stale() is False
+ ), "the latch outlived the lifecycle, so an embedded second session cannot spawn"
+
+
+def test_the_worker_spawn_refuses_once_shutdown_has_latched():
+ """The preview path supplies no load_cancel_event and is not a _ScopedLoadAttempt,
+ so its latch read is one-shot: it can pass that check, spend time in the drain and
+ teardown, and only then reach the worker spawn. Guarding at the spawn is what makes
+ the answer un-stale.
+ """
+ from core.inference.orchestrator import InferenceOrchestrator
+ from utils import process_lifetime
+
+ orch = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ try:
+ process_lifetime.mark_process_shutting_down()
+ with pytest.raises(RuntimeError, match = "shutting down"):
+ orch._spawn_subprocess({})
+ finally:
+ process_lifetime.begin_process_lifecycle()
+
+
+def test_the_process_latch_clears_between_the_backend_and_the_route():
+ """Ordering, for the same reason the route latch clears late: the process latch is
+ what every other spawner reads, so it must outlast the teardown _begin_server_lifecycle
+ waits on, and be clear before uvicorn admits anything.
+ """
+ backend = _run_server_call_lines("_begin_server_lifecycle")
+ process = _run_server_call_lines("begin_process_lifecycle")
+ route = _run_server_call_lines("begin_load_lifecycle")
+ serve = _run_server_call_lines("start", owner = "thread")
+
+ assert backend < process < route < serve, (
+ "the process latch must clear after the backend teardown completes and "
+ "before the server starts serving"
+ )
+
+
+def test_shutdown_latches_the_process_before_any_subsystem_is_torn_down():
+ """The orchestrator is stopped at step 2 but loads are swept at step 5. A load in
+ that gap reaches a spawner nothing has marked yet, so the latch has to be set before
+ the first teardown step rather than alongside the llama-server kill.
+ """
+ import ast
+ import textwrap
+ from pathlib import Path
+
+ run_py = (Path(__file__).resolve().parent.parent / "run.py").read_text(encoding = "utf-8")
+ tree = ast.parse(run_py)
+ fn = next(
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "_graceful_shutdown"
+ )
+ src = textwrap.dedent(ast.get_source_segment(run_py, fn) or "")
+
+ latch = src.index("mark_process_shutting_down()")
+ orchestrator_stop = src.index("_shutdown_subprocess(timeout = 5.0)")
+ sweep = src.index("cancel_pending_loads()")
+
+ assert latch < orchestrator_stop, (
+ "the inference subprocess is stopped before anything latches, so a load in "
+ "flight can restart it"
+ )
+ assert latch < sweep
+
+
+def test_a_shutdown_that_begins_during_the_spawn_reaps_the_new_worker():
+ """The pre-spawn gate is a process start away from the child existing. A shutdown
+ landing in between sees no live _proc, no-ops, completes terminate_all, and would
+ leave this worker running after quit. The post-spawn recheck is what closes it.
+ """
+ from unittest import mock
+
+ from core.inference.orchestrator import InferenceOrchestrator
+ from utils import process_lifetime
+
+ orch = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ torn_down = []
+ orch._shutdown_subprocess = lambda timeout = None: torn_down.append(timeout)
+
+ started = mock.Mock()
+ started.pid = 4242
+
+ class _Ctx:
+ Queue = staticmethod(lambda: mock.Mock())
+ Event = staticmethod(lambda: mock.Mock())
+
+ @staticmethod
+ def Process(**kw):
+ # Shutdown begins while the child is being born, after the gate passed.
+ process_lifetime.mark_process_shutting_down()
+ return started
+
+ try:
+ with (
+ mock.patch.object(orch_mod, "_CTX", _Ctx),
+ mock.patch.object(orch_mod, "adopt_pid", lambda pid: None, create = True),
+ ):
+ with pytest.raises(RuntimeError, match = "shutting down"):
+ orch._spawn_subprocess({})
+ finally:
+ process_lifetime.begin_process_lifecycle()
+
+ assert torn_down, "the worker born during shutdown was never torn down"
+
+
+def test_a_llama_server_spawned_as_shutdown_began_is_reaped():
+ """The pre-spawn stale check and the process latch are only atomic for the instance
+ run.py tears down, which sets its own flag under the spawn lock. A helper backend's
+ check can pass microseconds before the latch is set, and its child would then
+ outlive the sweep. The recheck after the pid is recorded is what reaps it.
+ """
+ import ast
+ import textwrap
+ from pathlib import Path
+
+ src = (
+ Path(__file__).resolve().parent.parent / "core" / "inference" / "llama_cpp.py"
+ ).read_text(encoding = "utf-8")
+ fn = next(
+ n
+ for n in ast.walk(ast.parse(src))
+ if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
+ and n.name == "_start_llama_process"
+ )
+ body = textwrap.dedent(ast.get_source_segment(src, fn) or "")
+
+ publish = body.index("self._record_server_pid(")
+ recheck = body.index("_spawn_is_stale", publish)
+ kill = body.index("self._kill_process()", publish)
+ assert publish < recheck < kill, (
+ "nothing rechecks after the child is published, so a spawn that raced the "
+ "latch leaves a server the sweep has already passed"
+ )
+
+
+def test_a_load_that_finishes_during_shutdown_is_not_published_as_resident():
+ """The spawn checks stop once the worker exists. A "loaded" reply dequeued as
+ shutdown kills it would still reach the success branch, and active_model_name plus
+ models are exactly what the already-loaded fast path trusts. That path does not test
+ liveness, so the next session would report a dead worker as resident.
+ """
+ from core.inference.orchestrator import InferenceOrchestrator
+ from utils import process_lifetime
+
+ orch = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ orch.active_model_name = None
+ orch.models = {}
+ orch.loading_models = {"m"}
+ orch.load_generation = 0
+
+ try:
+ process_lifetime.mark_process_shutting_down()
+ stale = process_lifetime.is_process_shutting_down()
+ assert stale is True, "the load no longer belongs to a live session"
+
+ # What the success branch must do instead of publishing.
+ if stale:
+ orch.loading_models.discard("m")
+ orch.active_model_name = None
+ orch.models.clear()
+
+ assert (
+ orch.active_model_name is None and not orch.models
+ ), "a worker killed by shutdown was published as resident"
+ finally:
+ process_lifetime.begin_process_lifecycle()
+
+
+def test_the_publish_branch_rechecks_shutdown_before_recording_the_model():
+ """Pins the recheck in the shipped code, ahead of the first field it publishes."""
+ import ast
+ import textwrap
+ from pathlib import Path
+
+ src = (
+ Path(__file__).resolve().parent.parent / "core" / "inference" / "orchestrator.py"
+ ).read_text(encoding = "utf-8")
+ fn = next(
+ n
+ for n in ast.walk(ast.parse(src))
+ if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "load_model"
+ )
+ body = textwrap.dedent(ast.get_source_segment(src, fn) or "")
+
+ check = body.index("if is_process_shutting_down(")
+ publish = body.index("self.active_model_name = model_info.get(")
+ assert check < publish, (
+ "the load publishes active_model_name before rechecking shutdown, so a worker "
+ "killed mid-load is recorded as resident"
+ )
+
+
+def _fn_named(source, name):
+ import ast
+ return next(
+ n
+ for n in ast.walk(ast.parse(source))
+ if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == name
+ )
+
+
+def test_the_primary_llama_launch_rechecks_after_recording_the_pid():
+ """mark_process_shutting_down does not take _spawn_lock, and terminate_all does not
+ take it when it snapshots, so the in-lock check is not atomic against the
+ process-wide latch for a helper-owned backend. Without a recheck the child sits
+ outside the completed sweep for the whole 600s health wait.
+ """
+ import ast
+ from pathlib import Path
+
+ src = (
+ Path(__file__).resolve().parent.parent / "core" / "inference" / "llama_cpp.py"
+ ).read_text(encoding = "utf-8")
+ fn = _fn_named(src, "_spawn_and_wait")
+ record = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "_record_server_pid"
+ ]
+ stale = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "_spawn_is_stale"
+ ]
+ health = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "_wait_for_health"
+ ]
+ assert record and stale and health, "the spawn path no longer looks like itself"
+ assert any(max(record) < s < min(health) for s in stale), (
+ "no staleness recheck between recording the pid and the health wait: a spawn "
+ "that raced the latch is left running outside the sweep that already finished"
+ )
+
+
+def test_the_worker_mirrors_are_published_under_the_shutdown_lock():
+ """active_model_name is what the already-loaded fast path trusts, and it does not
+ test liveness. Checked and published apart, shutdown can kill the worker in between
+ and the next session reports a dead one as resident.
+ """
+ import ast
+ from pathlib import Path
+
+ src = (
+ Path(__file__).resolve().parent.parent / "core" / "inference" / "orchestrator.py"
+ ).read_text(encoding = "utf-8")
+ fn = _fn_named(src, "load_model")
+ holding = [
+ n
+ for n in ast.walk(fn)
+ if isinstance(n, ast.With)
+ and any(
+ getattr(item.context_expr, "attr", None) == "_subprocess_shutdown_lock"
+ for item in n.items
+ )
+ ]
+ assert holding, "load_model never holds the subprocess shutdown lock"
+ covered = False
+ for w in holding:
+ checks = [
+ n
+ for n in ast.walk(w)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "is_process_shutting_down"
+ ]
+ publishes = [
+ n
+ for n in ast.walk(w)
+ if isinstance(n, ast.Assign)
+ and any(
+ getattr(t, "attr", None) == "active_model_name"
+ and not isinstance(n.value, ast.Constant)
+ for t in n.targets
+ )
+ ]
+ if checks and publishes:
+ covered = True
+ assert covered, (
+ "the shutdown check and the active_model_name publication are not inside the "
+ "same held lock, so a kill can land between them"
+ )
+
+
+def test_the_worker_handle_is_captured_before_start_not_after():
+ """`self._proc` is not a handle this code can rely on once start() is running: a
+ concurrent _shutdown_subprocess can observe a not-yet-alive child and clear it.
+ Snapshotting the attribute AFTER start() therefore captures None and loses the only
+ reference to a live child, which is exactly the orphan this change prevents.
+ """
+ import ast
+ from pathlib import Path
+
+ src = (
+ Path(__file__).resolve().parent.parent / "core" / "inference" / "orchestrator.py"
+ ).read_text(encoding = "utf-8")
+ fn = _fn_named(src, "_spawn_subprocess")
+
+ # The local must be bound from the Process(...) construction, never from self._proc.
+ binds = [
+ n
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Assign)
+ and any(getattr(t, "id", None) == "_spawned_proc" for t in n.targets)
+ ]
+ assert binds, "_spawn_subprocess no longer keeps a local handle on the worker"
+ for n in binds:
+ assert not (isinstance(n.value, ast.Attribute) and n.value.attr == "_proc"), (
+ "the local handle is snapshotted from self._proc, which a concurrent "
+ "shutdown can have cleared by then; build it from Process(...) instead"
+ )
+
+ # And start() must be called through the local, not through the attribute.
+ starts = [
+ n
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "start"
+ ]
+ assert starts, "the worker is never started"
+ for n in starts:
+ owner = n.func.value
+ assert getattr(owner, "id", None) == "_spawned_proc", (
+ "start() is called on self._proc rather than the local handle, so a "
+ "shutdown clearing the attribute mid-start loses the child"
+ )
+
+
+def test_the_rag_embed_server_spawn_is_gated_and_rechecked():
+ """No _graceful_shutdown step stops this backend, so the process-wide latch is the
+ only thing standing between an in-flight encode and a server that outlives quit.
+ """
+ import ast
+ from pathlib import Path
+
+ src = (
+ Path(__file__).resolve().parent.parent / "core" / "rag" / "embed_llama_server.py"
+ ).read_text(encoding = "utf-8")
+ fn = _fn_named(src, "_spawn_once")
+
+ checks = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "is_process_shutting_down"
+ ]
+ popen = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "Popen"
+ ]
+ adopt = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "adopt_pid"
+ ]
+ assert popen and adopt, "the embed spawn no longer looks like itself"
+ assert any(c < min(popen) for c in checks), (
+ "the embed server spawns without consulting the shutdown latch, so a quit "
+ "during an encode can start one after the sweep has run"
+ )
+ assert any(c > max(adopt) for c in checks), (
+ "no recheck after the pid is recorded: a latch set during the spawn leaves "
+ "this child outside a sweep that has already finished"
+ )
+
+
+def test_the_stt_sidecar_spawns_are_gated_and_rechecked():
+ """Neither STT sidecar is stopped by any _graceful_shutdown step and neither is
+ covered by cancel_pending_loads, which only reaches the chat /load attempt maps.
+ The process-wide latch is the only thing that can stop a quit during an STT load
+ from leaving whisper-server or the MTMD llama-server behind.
+ """
+ import ast
+ from pathlib import Path
+
+ root = Path(__file__).resolve().parent.parent / "core" / "inference"
+ for filename, func in (
+ ("stt_ggml_sidecar.py", "load"),
+ ("stt_mtmd_sidecar.py", "_load_locked"),
+ ):
+ src = (root / filename).read_text(encoding = "utf-8")
+ fn = _fn_named(src, func)
+ checks = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "is_process_shutting_down"
+ ]
+ popen = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "Popen"
+ ]
+ adopt = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "adopt_pid"
+ ]
+ assert popen and adopt, f"{filename}:{func} no longer looks like a spawn"
+ assert any(c < min(popen) for c in checks), (
+ f"{filename} spawns without consulting the shutdown latch, so quitting "
+ "during an STT load can start a server after the sweep has run"
+ )
+ assert any(c > max(adopt) for c in checks), (
+ f"{filename} has no recheck after recording the pid, so a latch set during "
+ "the spawn leaves the child outside a sweep that already finished"
+ )
+
+
+def test_the_latch_is_set_before_the_atexit_sweep():
+ """atexit is LIFO, so the handler registered LAST runs FIRST.
+
+ The only thing that latched shutdown on the atexit path was the backend's own
+ _cleanup hook, registered when the routes singleton was built and therefore run
+ AFTER run_server's terminate_all. The sweep took its snapshot with the latch clear,
+ which is exactly the unguarded behaviour this change removes on the graceful path.
+ """
+ import ast
+ from pathlib import Path
+
+ run_py = (Path(__file__).resolve().parent.parent / "run.py").read_text(encoding = "utf-8")
+ tree = ast.parse(run_py)
+ registrations = [
+ (n.lineno, n.args[0].id)
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Call)
+ and getattr(n.func, "attr", None) == "register"
+ and getattr(n.func.value, "id", None) == "atexit"
+ and n.args
+ and isinstance(n.args[0], ast.Name)
+ ]
+ names = [name for _, name in registrations]
+ assert "terminate_all" in names, "run.py no longer registers the backstop sweep"
+ assert "mark_process_shutting_down" in names, (
+ "nothing latches shutdown on the atexit path, so the sweep snapshots while "
+ "spawners still believe they may start children"
+ )
+ sweep_line = max(l for l, name in registrations if name == "terminate_all")
+ latch_line = max(l for l, name in registrations if name == "mark_process_shutting_down")
+ assert latch_line > sweep_line, (
+ "the latch is registered before the sweep, so under LIFO it runs after it and "
+ "the snapshot is taken unguarded"
+ )
+
+
+def test_every_long_lived_spawner_consults_the_shutdown_latch():
+ """The premise of this change is one flag read at every spawn. A spawner that
+ adopts a long-lived child without consulting it is a hole in exactly the guarantee
+ the PR claims, so the set is pinned here rather than rediscovered one report at a
+ time.
+ """
+ import ast
+ from pathlib import Path
+
+ backend = Path(__file__).resolve().parent.parent
+ guarded = {
+ "core/export/orchestrator.py",
+ "core/inference/llama_cpp.py",
+ "core/inference/orchestrator.py",
+ "core/inference/sd_cpp_engine.py",
+ "core/inference/sd_cpp_server.py",
+ "core/inference/stt_ggml_sidecar.py",
+ "core/inference/stt_mtmd_sidecar.py",
+ "core/inference/stt_transformers_worker.py",
+ "core/rag/embed_llama_server.py",
+ "core/training/training.py",
+ }
+ # Adopters this change deliberately leaves ungated, listed so the completeness check
+ # below cannot pass by omission. They are a documented residual, not an oversight:
+ # gating them is the same six lines each, but each needs its own failure idiom and
+ # its own reaping, and this PR is scoped to the paths a quit during a model load
+ # actually reaches. A new adopter lands in neither set and fails the check, which is
+ # the point: the decision gets made once, here, instead of one report at a time.
+ not_gated_here = {
+ "cloudflare_tunnel.py",
+ "core/data_recipe/jobs/manager.py",
+ "core/inference/diffusion_transformer_quant.py",
+ "core/inference/stt_download_worker.py",
+ "core/inference/tools.py",
+ "core/training/diffusion_training_service.py",
+ "utils/prebuilt/update_flow.py",
+ "utils/process_lifetime.py",
+ "utils/torch_device_probe.py",
+ }
+ adopters = {
+ str(path.relative_to(backend)).replace("\\", "/")
+ for path in backend.rglob("*.py")
+ if "adopt_pid(" in path.read_text(encoding = "utf-8")
+ and not str(path.relative_to(backend)).replace("\\", "/").startswith("tests/")
+ and "vendor/" not in str(path.relative_to(backend)).replace("\\", "/")
+ }
+ assert adopters == guarded | not_gated_here, (
+ "a module adopts a child but is in neither set; decide whether it needs the "
+ f"shutdown gate and put it in one of them: {adopters ^ (guarded | not_gated_here)}"
+ )
+ for rel in sorted(guarded):
+ src = (backend / rel).read_text(encoding = "utf-8")
+ tree = ast.parse(src)
+ adopts = [
+ n.lineno
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "adopt_pid"
+ ]
+ checks = [
+ n.lineno
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "is_process_shutting_down"
+ ]
+ assert adopts, f"{rel} no longer adopts a child"
+ assert checks, f"{rel} adopts a child without ever consulting the shutdown latch"
+ assert any(c > min(adopts) for c in checks), (
+ f"{rel} never rechecks the latch after adopting, so a spawn that raced it "
+ "is left outside a sweep that has already finished"
+ )
diff --git a/studio/backend/tests/test_orphaned_children.py b/studio/backend/tests/test_orphaned_children.py
index ec66d735bc0..6fb0bace602 100644
--- a/studio/backend/tests/test_orphaned_children.py
+++ b/studio/backend/tests/test_orphaned_children.py
@@ -1450,7 +1450,11 @@ def test_the_visual_server_goes_down_with_an_ordinary_stop():
from core.inference.llama_cpp import LlamaCppBackend
- source = inspect.getsource(LlamaCppBackend._kill_process)
+ # The whole kill path: _kill_process is a thin wrapper that takes the spawn
+ # lock for a teardown and delegates the termination to _kill_process_body.
+ source = inspect.getsource(LlamaCppBackend._kill_process) + inspect.getsource(
+ LlamaCppBackend._kill_process_body
+ )
assert "_collect_descendants" in source
assert "_terminate_descendants" in source
# Named before the wait: the shim's children are reparented once it exits.
diff --git a/studio/backend/tests/test_parallel_slots_per_load.py b/studio/backend/tests/test_parallel_slots_per_load.py
index 0436e6db5c3..bd56d419759 100644
--- a/studio/backend/tests/test_parallel_slots_per_load.py
+++ b/studio/backend/tests/test_parallel_slots_per_load.py
@@ -227,7 +227,12 @@ def test_load_model_commits_requested_from_intent():
# come from the immutable pre-reduction intent.
src = inspect.getsource(LlamaCppBackend.load_model)
commit = src.find("self._requested_n_parallel = max(1, int(intent.n_parallel))")
- healthy = src.find("self._healthy = True\n", 0, commit if commit != -1 else None)
+ # The commit goes through _publish_healthy(), which sets _healthy under the
+ # spawn lock so a teardown cannot land between the successful probe and the
+ # commit. The invariant this test guards is unchanged: health is published
+ # first, so a failed start cannot poison the next inheritance check.
+ # Matched without the argument list, so adding one does not break this again.
+ healthy = src.find("self._publish_healthy(", 0, commit if commit != -1 else None)
snapshot = src.find("self._last_load_intent = replace(intent")
assert commit != -1, "load_model must commit the requested slot count"
assert healthy != -1 and healthy < commit < snapshot
diff --git a/studio/backend/tests/test_tensor_parallel.py b/studio/backend/tests/test_tensor_parallel.py
index ae765bfce6e..8c9ec975780 100644
--- a/studio/backend/tests/test_tensor_parallel.py
+++ b/studio/backend/tests/test_tensor_parallel.py
@@ -1136,7 +1136,11 @@ def test_kill_process_stops_crash_watchdog(monkeypatch):
def test_kill_process_stops_watchdog_before_terminate():
# Ordering matters: stop the watchdog before terminating so the watchdog's
# post-death stop re-check reliably sees a planned kill.
- src = inspect.getsource(LlamaCppBackend._kill_process)
+ # Both halves: the termination moved into _kill_process_body, and the ordering
+ # this asserts is within that body.
+ src = inspect.getsource(LlamaCppBackend._kill_process) + inspect.getsource(
+ LlamaCppBackend._kill_process_body
+ )
stop = src.find("_stop_mtp_crash_watchdog()")
term = src.find(".terminate(")
assert 0 <= stop < term, "must stop the watchdog before terminating"
@@ -1280,8 +1284,7 @@ def test_tensor_plan_leaves_the_floor_reserve_at_a_full_budget():
def test_split_mode_tensor_arch_failure_message():
msg = LlamaCppBackend._classify_llama_start_failure(
- "llama_model_create: LLAMA_SPLIT_MODE_TENSOR not implemented for "
- "architecture 'deepseek2'",
+ "llama_model_create: LLAMA_SPLIT_MODE_TENSOR not implemented for architecture 'deepseek2'",
None,
"unsloth/DeepSeek-V3-GGUF",
)
diff --git a/studio/backend/tests/test_training_pump_resilience.py b/studio/backend/tests/test_training_pump_resilience.py
index 5d9d406729e..c1a8a5faca3 100644
--- a/studio/backend/tests/test_training_pump_resilience.py
+++ b/studio/backend/tests/test_training_pump_resilience.py
@@ -506,6 +506,10 @@ def _stub_spawn(monkeypatch):
pl = _types.ModuleType("utils.process_lifetime")
pl.adopt_pid = lambda pid: None
+ # The spawn also reads the shutdown latch. These tests are about the pump, not about
+ # quitting, so the double answers "not shutting down" and the spawn proceeds; leaving
+ # it off makes the import fail and every start_training here return False.
+ pl.is_process_shutting_down = lambda: False
monkeypatch.setitem(sys.modules, "utils.process_lifetime", pl)
worker = _types.ModuleType("core.training.worker")
diff --git a/studio/backend/utils/process_lifetime.py b/studio/backend/utils/process_lifetime.py
index f0fcb58af74..c71eba75e40 100644
--- a/studio/backend/utils/process_lifetime.py
+++ b/studio/backend/utils/process_lifetime.py
@@ -357,7 +357,7 @@ _fork_reset_installed = False
def _reset_after_fork() -> None:
"""A fork child inherits both locks in whatever state they were in and a
_spawner whose thread does not exist here. Start clean instead of deadlocking."""
- global _spawner, _spawner_lock, _record_lock, _owner_identity
+ global _spawner, _spawner_lock, _record_lock, _owner_identity, _shutdown_latch
_spawner_lock = threading.Lock()
# A different pid here.
_owner_identity = None
@@ -366,6 +366,14 @@ def _reset_after_fork() -> None:
# A fork while another thread was inside adopt_pid / forget_pid leaves this held here with nobody to release it, and
# the first adoption blocks forever.
_record_lock = threading.Lock()
+ # Same hazard: Event carries an internal lock, so a fork taken while another thread
+ # was inside set() leaves the child unable to latch. Rebuilt holding the flag it had
+ # -- the child is still inside the lifecycle that forked it, and a cleared latch
+ # would read as permission to spawn.
+ _was_latched = _shutdown_latch.is_set()
+ _shutdown_latch = threading.Event()
+ if _was_latched:
+ _shutdown_latch.set()
_spawner = None
@@ -988,6 +996,37 @@ def adopt_pid(pid: Optional[int]) -> None:
pass
+# Set once when the app starts quitting, read by every spawner in the process.
+_shutdown_latch = threading.Event()
+
+
+def mark_process_shutting_down() -> None:
+ """Latch "this process is quitting" for every spawner in it.
+
+ Each subsystem already refuses to spawn during its OWN teardown, but that state
+ lives on the object being torn down: a second LlamaCppBackend built for a helper
+ load, or the inference orchestrator, never sees it and can Popen a child after
+ terminate_all has taken its snapshot. Set once here, read everywhere, so the answer
+ does not depend on which object a spawn happens to belong to.
+ """
+ _shutdown_latch.set()
+
+
+def is_process_shutting_down() -> bool:
+ """Whether a spawn must be refused because the app is quitting."""
+ return _shutdown_latch.is_set()
+
+
+def begin_process_lifecycle() -> None:
+ """Clear the latch for an embedded host that calls run_server again.
+
+ Quitting is terminal for a CLI run, but in-process callers (studio/backend/colab.py)
+ reuse the interpreter, and a latch that never cleared would refuse every spawn of
+ the second session.
+ """
+ _shutdown_latch.clear()
+
+
def terminate_all(timeout: float = 5.0) -> "list[int]":
"""Backstop sweep over adopted pids, after per-subsystem cleanup. SIGTERM,
then SIGKILL the survivors after `timeout`. Idempotent and teardown-safe.