Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop a quit during a model load from orphaning llama-server (#10592)
Daniel Hanfcaad20ef248
diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.pyindex da8b6314c2d..7ce9ef409e0 100644--- a/studio/backend/core/export/orchestrator.py+++ b/studio/backend/core/export/orchestrator.py@@ -230,9 +230,16 @@ class ExportOrchestrator:run_without_native_path_secret,)from utils.hf_cache_settings import child_environment_for_spawn, get_hf_cache_paths+ from utils.process_lifetime import is_process_shutting_downcache_env = get_hf_cache_paths().child_env({})+ # An export admitted before the quit can still reach this line after the shutdown+ # sweep has taken its snapshot, and the worker adopted below would then outlive+ # Studio holding the model in memory.+ if is_process_shutting_down():+ raise RuntimeError("Studio is shutting down; not starting an export subprocess")+with (child_environment_for_spawn(cache_env),native_path_secret_removed_for_child_start(),@@ -240,7 +247,11 @@ class ExportOrchestrator:self._cmd_queue = _CTX.Queue()self._resp_queue = _CTX.Queue()- self._proc = _CTX.Process(+ # Kept in a local as well as on self: a concurrent _shutdown_subprocess can+ # see a process that has not finished starting, decide it is not alive and+ # clear self._proc, and every read below would then be off a None while the+ # worker is alive and unadopted.+ _spawned_proc = _CTX.Process(target = run_without_native_path_secret,args = ("core.export.worker", "run_export_process", cache_env),kwargs = {@@ -250,11 +261,39 @@ class ExportOrchestrator:},daemon = True,)- self._proc.start()- from utils.process_lifetime import adopt_pid-- adopt_pid(self._proc.pid)- logger.info("Export subprocess started (pid=%s)", self._proc.pid)+ self._proc = _spawned_proc+ _spawned_proc.start()+ from utils.process_lifetime import adopt_pid, forget_pid++ adopt_pid(_spawned_proc.pid)+ # Recheck once the pid is recorded, for the window between the gate above and+ # this record. Adoption runs first, so a worker torn down here was in the sweep+ # record for as long as it existed. The handle check catches the other half of+ # the race: a teardown that already cleared or replaced self._proc leaves this+ # worker with no owner, so reap it here rather than let it run on.+ if is_process_shutting_down() or self._proc is not _spawned_proc:+ logger.info("shutdown began during the spawn; stopping the new export subprocess")+ if self._proc is _spawned_proc:+ self._shutdown_subprocess(timeout = 5)+ else:+ try:+ if _spawned_proc.is_alive():+ _spawned_proc.terminate()+ _spawned_proc.join(timeout = 5)+ if _spawned_proc.is_alive():+ _spawned_proc.kill()+ _spawned_proc.join(timeout = 3)+ except Exception: # noqa: BLE001 - the reap is best-effort+ logger.warning("could not reap the orphaned export worker", exc_info = True)+ if _spawned_proc.exitcode is not None:+ forget_pid(_spawned_proc.pid)+ else:+ logger.warning(+ "export worker (pid %s) survived the reap; leaving it adopted",+ _spawned_proc.pid,+ )+ raise RuntimeError("Studio is shutting down; not starting an export subprocess")+ logger.info("Export subprocess started (pid=%s)", _spawned_proc.pid)def _shutdown_subprocess(self, timeout: float = 10.0) -> bool:"""Gracefully shut down the export subprocess.diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 0c96e7271aa..8b65d4a94d2 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -6229,6 +6229,15 @@ class LlamaCppBackend:3. unload_model(): terminate the subprocess"""+ # Held across "is a teardown running?" and publishing the child, never across a+ # health wait. On the class: doubles built with __new__ never run __init__.+ _spawn_lock = threading.Lock()++ # Held across a whole teardown, so a lifecycle cannot reopen mid-kill. Separate+ # from _spawn_lock so that long hold does not also block a spawn, which only+ # needs to read the flag. Order is always _teardown_lock then _spawn_lock.+ _teardown_lock = threading.Lock()+def __init__(self, *, manages_processes: bool = True):"""``manages_processes = False`` builds an INERT probe.@@ -14494,29 +14503,49 @@ class LlamaCppBackend:# The shim (and its visual server) die with this backend process, so a# Unsloth crash/restart never orphans a GPU process.- self._process = subprocess.Popen(- cmd,- stdout = subprocess.PIPE,- stderr = subprocess.STDOUT,- text = True,- encoding = "utf-8",- errors = "replace",- env = utf8_child_env(env),- # Deliberately NOT start_new_session, as with the component- # installer: the desktop stops this backend by signalling its- # process group and force-kills it after five seconds, so a session- # of its own would leave the shim and the visual server holding the- # GPU until the next launch sweeps them.- **_windows_hidden_subprocess_kwargs(),- **_child_popen_kwargs(),- )- # macOS has no parent-death signal, so the kwargs above are empty there and- # only this record lets the next startup reap a runner holding the GPU.- try:- from utils.process_lifetime import adopt_pid- adopt_pid(self._process.pid)- except Exception as e:- logger.debug(f"Could not track diffusion runner for lifetime sweep: {e}")+ # Own Popen, and no parent-death backstop on every platform, so a runner+ # started after the shutdown sweep outlives it.+ with self._spawn_lock:+ if self._spawn_is_stale():+ logger.info("app is shutting down; not starting the diffusion runner")+ self._close_attempt_log()+ self._health_wait_cancelled = True+ return False+ _spawned = subprocess.Popen(+ cmd,+ stdout = subprocess.PIPE,+ stderr = subprocess.STDOUT,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ env = utf8_child_env(env),+ # Deliberately NOT start_new_session, as with the component+ # installer: the desktop stops this backend by signalling its+ # process group and force-kills it after five seconds, so a session+ # of its own would leave the shim and the visual server holding the+ # GPU until the next launch sweeps them.+ **_windows_hidden_subprocess_kwargs(),+ **_child_popen_kwargs(),+ )+ self._process = _spawned+ # macOS has no parent-death signal, so only this record reaps a runner+ # holding the GPU. Under the lock: adopting after the sweep re-adds a+ # pid it just forgot.+ try:+ from utils.process_lifetime import adopt_pid+ adopt_pid(_spawned.pid)+ except Exception as e:+ logger.debug(f"Could not track diffusion runner for lifetime sweep: {e}")+ # Same post-adoption recheck as the llama-server spawns: this backend can be+ # helper-owned, which run.py's singleton teardown never marks, so a latch set+ # after the in-lock check would otherwise leave the shim and the visual server+ # alive for the whole health wait below.+ if self._spawn_is_stale():+ logger.info("shutdown began during the spawn; killing the new diffusion runner")+ self._kill_process()+ self._close_attempt_log()+ self._health_wait_cancelled = True+ return Falseself._stdout_thread = threading.Thread(target = self._drain_stdout, daemon = True, name = "diffusion-stdout")@@ -14607,7 +14636,11 @@ class LlamaCppBackend:healthy = self._wait_for_health(timeout = 600.0, cancelled = cancelled)if healthy:- self._healthy = True+ if not self._publish_healthy():+ # A teardown between the probe and this commit is already killing+ # the runner; publishing would advertise a server that is gone.+ self._kill_process()+ return Falseself._gpu_offload_active = not holds_no_gpuif extra_args is not None:self._extra_args = list(extra_args)@@ -18638,12 +18671,16 @@ class LlamaCppBackend:def _start_llama_process(self, cmd: list[str], env: dict, *, child_gpu_physical_ids: Optional[tuple[int, ...]]- ) -> None:+ ) -> bool:"""Spawn llama-server from cmd and start draining its output.Caller holds self._lock. Resets the stdout buffer, opens a freshper-attempt tee log, launches the process, and starts the drainthread. Used for the initial start and the text-only mmproj retry.++ Returns False without spawning once app teardown has begun. Reported rather+ than silent so the caller can stop instead of health-waiting on the previous+ child and then reading a reference the teardown is clearing."""# Defensive kill: if a concurrent load slipped past Phase 1# (because its `self._process` was None at the time) and already@@ -18679,26 +18716,48 @@ class LlamaCppBackend:# with --mmproj stripped), redacting the API key.logger.info(f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(cmd))}")- self._process = subprocess.Popen(- cmd,- stdout = subprocess.PIPE,- stderr = subprocess.STDOUT,- text = True,- encoding = "utf-8",- errors = "replace",- env = env,- **_windows_hidden_subprocess_kwargs(),- **_child_popen_kwargs(),- )- # Cross-session backstop: record the PID so a later startup can reap this- # server if parent-death cleanup did not run (macOS / best-effort failure).- self._record_server_pid(self._process.pid)+ # Check with publication under one lock: the mmproj text-only retry reaches+ # a spawn without passing _spawn_and_wait's boundary.+ with self._spawn_lock:+ if self._spawn_is_stale():+ logger.info("app is shutting down; not starting llama-server")+ self._close_attempt_log()+ self._health_wait_cancelled = True+ return False+ _spawned = subprocess.Popen(+ cmd,+ stdout = subprocess.PIPE,+ stderr = subprocess.STDOUT,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ env = env,+ **_windows_hidden_subprocess_kwargs(),+ **_child_popen_kwargs(),+ )+ self._process = _spawned+ # Cross-session backstop for when parent-death cleanup did not run.+ # Under the lock: see _spawn_and_wait.+ self._record_server_pid(_spawned.pid)++ # The stale check above and the process-wide latch are only atomic for the+ # instance run.py tears down, which sets its own flag under this same lock. A+ # helper load owns a backend nothing marks, so its check can pass microseconds+ # before the latch is set and the child then outlives the sweep. Recheck once+ # the pid is recorded and reap it, as the inference worker spawn does.+ if self._spawn_is_stale():+ logger.info("shutdown began during the spawn; killing the new llama-server")+ self._kill_process()+ self._close_attempt_log()+ self._health_wait_cancelled = True+ return False# Start background thread to drain stdout and prevent pipe deadlockself._stdout_thread = threading.Thread(target = self._drain_stdout, daemon = True, name = "llama-stdout")self._stdout_thread.start()+ return True@contextlib.contextmanagerdef _serial_load_scope(self):@@ -18771,9 +18830,22 @@ class LlamaCppBackend:cache_ram = intent.cache_ramextra_args = list(intent.extra_args) if intent.extra_args is not None else Nonepreserve_multi_gpu_on_layer = intent.preserve_multi_gpu_on_layer+ # Before the serial scope: a queued load still belongs to the lifecycle it+ # was requested in.+ # The process-wide equivalent, for the same reason. Only _begin_server_lifecycle+ # advances the per-instance one, and a helper load owns a backend that never+ # gets it, so an embedded second session would otherwise release this load.# Serialise the whole load so concurrent /load calls never leave two# llama-server processes alive (#5401 / #5161). Doesn't block /unload.with self._serial_load_scope():+ # Here, not at the spawn: a lock gives a waiter no priority, and the+ # duplicate-adoption phase below kills whatever is loaded -- after a+ # restart, the new lifecycle's model.+ with self._spawn_lock:+ _stale_load = self._spawn_is_stale()+ if _stale_load:+ logger.info("dropping a load left over from the previous server lifecycle")+ return False# In-app update swapping binaries: refuse fast (set under this lock,# so any in-flight load has drained) instead of using a half-swapped one.if getattr(self, "_llama_update_in_progress", False):@@ -24143,19 +24215,44 @@ class LlamaCppBackend:run_cmd,supports_cache_ram = bool(server_caps.get("supports_cache_ram")),)- self._process = subprocess.Popen(- run_cmd,- stdout = subprocess.PIPE,- stderr = subprocess.STDOUT,- text = True,- encoding = "utf-8",- errors = "replace",- env = env,- cwd = _spawn_cwd,- **_windows_hidden_subprocess_kwargs(),- **_child_popen_kwargs(),- )- self._record_server_pid(self._process.pid)+ # Check with publication under one lock: a spawn either+ # publishes first and the sweep kills it, or sees the flag+ # and never starts. Across Popen only, never the wait.+ with self._spawn_lock:+ if self._spawn_is_stale():+ logger.info("app is shutting down; not starting llama-server")+ self._close_attempt_log()+ self._health_wait_cancelled = True+ return False+ _spawned = subprocess.Popen(+ run_cmd,+ stdout = subprocess.PIPE,+ stderr = subprocess.STDOUT,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ env = env,+ cwd = _spawn_cwd,+ **_windows_hidden_subprocess_kwargs(),+ **_child_popen_kwargs(),+ )+ self._process = _spawned+ # Inside the lock: written after a sweep reaped the child,+ # _pid_start_identity yields no start time, and the bare pid+ # left behind is one a later launch kills blind.+ self._record_server_pid(_spawned.pid)+ # mark_process_shutting_down does not take _spawn_lock, so the+ # check above is not atomic against it for a helper-owned+ # backend. Without this recheck a child spawned in that gap sits+ # outside the completed sweep for the whole 600s health wait.+ if self._spawn_is_stale():+ logger.info(+ "shutdown began during the spawn; killing the new llama-server"+ )+ self._kill_process()+ self._close_attempt_log()+ self._health_wait_cancelled = True+ return False# is_active covers it from here, so drop the pre-spawn flag.self._memory_launch_pending = False@@ -24467,7 +24564,8 @@ class LlamaCppBackend:# and keep the staged runtime for the caller's next argv.self._kill_process()return False- cpu_rc = self._process.poll() if self._process is not None else None+ _proc_snap1 = self._process # snapshot: re-reading races the teardown+ cpu_rc = _proc_snap1.poll() if _proc_snap1 is not None else Nonedetail = self._classify_llama_start_failure("\n".join(self._stdout_lines[-50:]),gguf_path,@@ -24684,7 +24782,8 @@ class LlamaCppBackend:# skipping the futile flash-attn/MTP retries.if not healthy and self._tensor_parallel and not _load_cancelled():_ts_out = "\n".join(self._stdout_lines[-50:])- _ts_rc = self._process.poll() if self._process is not None else None+ _proc_snap2 = self._process # snapshot: re-reading races the teardown+ _ts_rc = _proc_snap2.poll() if _proc_snap2 is not None else Noneif self._should_record_tensor_split_abort(_ts_rc, _ts_out):LlamaCppBackend._record_tensor_split_abort(binary, model_identifier, _planned_cache_pair@@ -25072,7 +25171,8 @@ class LlamaCppBackend:# both vision and MTP, so retry that way before dropping either.# Only on a hard fault with FA on; a cancel/unload stops respawn.if not healthy and not _load_cancelled():- _fa_rc = self._process.poll() if self._process is not None else None+ _proc_snap3 = self._process # snapshot: re-reading races the teardown+ _fa_rc = _proc_snap3.poll() if _proc_snap3 is not None else None_fa_cmd = (self._with_flash_attn_off(_last_spawn_cmd,@@ -25145,7 +25245,8 @@ class LlamaCppBackend:):# A first-decode hard fault is usually the FA kernel: retry# FA-off (keeps MTP) before dropping speculative decoding below.- _probe_rc = self._process.poll() if self._process is not None else None+ _proc_snap4 = self._process # snapshot: re-reading races the teardown+ _probe_rc = _proc_snap4.poll() if _proc_snap4 is not None else None_fa_cmd = (self._with_flash_attn_off(_last_spawn_cmd,@@ -25315,7 +25416,8 @@ class LlamaCppBackend:if not healthy:out = "\n".join(self._stdout_lines[-50:])# Read the crash code before _kill_process() clears _process.- _crash_rc = self._process.poll() if self._process is not None else None+ _proc_snap5 = self._process # snapshot: re-reading races the teardown+ _crash_rc = _proc_snap5.poll() if _proc_snap5 is not None else Noneself._kill_process()# Only when the WAIT itself was cancelled. A cancel that lands later,# while a crashed launch is staging its CPU fallback, must still run@@ -25387,8 +25489,10 @@ class LlamaCppBackend:)else:_cpu_projector_out = "\n".join(self._stdout_lines[-50:])+ # Snapshot: re-reading races the teardown.+ _proc_snap6 = self._process_cpu_projector_rc = (- self._process.poll() if self._process is not None else None+ _proc_snap6.poll() if _proc_snap6 is not None else None)self._kill_process()if _finish_cancelled_health_wait(@@ -25458,11 +25562,15 @@ class LlamaCppBackend:_last_spawn_cmd = list(cmd)self._is_vision = Falseself._mmproj_has_audio = False- self._start_llama_process(+ if not self._start_llama_process(cmd,env,child_gpu_physical_ids = _child_gpu_physical_ids,- )+ ):+ # Shutdown refused the retry; self._process still names+ # the old child the teardown is clearing.+ _cleanup_cancelled_load("App shut down during the text-only retry")+ return Falseif self._wait_for_health(timeout = 600.0, cancelled = _load_cancelled):healthy = True# The child that serves this session never read the@@ -25483,9 +25591,9 @@ class LlamaCppBackend:else:# Read the exit code before _kill_process() clears it, so# an OS-killed text-only retry still gets the OOM message.- _retry_rc = (- self._process.poll() if self._process is not None else None- )+ # Snapshot: re-reading races the teardown.+ _retry_proc = self._process+ _retry_rc = _retry_proc.poll() if _retry_proc is not None else Noneself._kill_process()if _finish_cancelled_health_wait("Load cancelled during the text-only retry health wait"@@ -25589,7 +25697,11 @@ class LlamaCppBackend:else None),)- self._healthy = True+ if not self._publish_healthy():+ # Teardown began between the 200 and this commit; publishing+ # would report a model that is gone.+ _cleanup_cancelled_load("App shut down as the load was completing")+ return Falseself._commit_effective_parallel_slots(n_parallel)self._swa_full = swa_fullself._kv_cache_unified = kv_cache_unified@@ -26724,8 +26836,99 @@ class LlamaCppBackend:except Exception as e:logger.debug(f"Could not terminate server descendants: {e}")- def _kill_process(self):- """Terminate the subprocess if running."""+ def _publish_healthy(self) -> bool:+ """Commit _healthy under the spawn lock, or refuse if this load is stale.++ Under the lock the teardown mark is set with, so only two orders exist:+ publish then teardown (which clears _healthy), or teardown then a refused+ publish.+ """+ with self._spawn_lock:+ if self._spawn_is_stale():+ logger.info("load no longer belongs to this lifecycle; not publishing it healthy")+ return False+ self._healthy = True+ return True++ def _close_attempt_log(self) -> None:+ """Close the per-attempt tee log opened just before a spawn.++ A refusal publishes no process and _kill_process returns early when there is+ none, so nothing else closes it: the next attempt leaks the descriptor and,+ on Windows, holds the file lock an update needs.+ """+ fh = getattr(self, "_llama_log_fh", None)+ if fh is not None:+ try:+ fh.close()+ except Exception:+ pass+ self._llama_log_fh = None++ def _begin_server_lifecycle(self) -> None:+ """Clear shutdown state so a restarted server can launch again.++ The backend is a module singleton and an embedded host may call+ run_server() more than once in one process, so "shutting down" is scoped+ to a lifecycle rather than to the interpreter. Called from run_server+ before anything can spawn.++ """+ # _teardown_lock first, so this waits for an in-progress kill rather than+ # clearing the flag underneath it. Same order as _kill_process.+ with self._teardown_lock:+ with self._spawn_lock:+ self._shutting_down = False+ self._torn_down_process = None++ def _spawn_is_stale(self) -> bool:+ """Whether this load may no longer spawn. Caller holds _spawn_lock."""+ if getattr(self, "_shutting_down", False):+ return True+ # Per-instance state only covers the singleton run.py tears down. A helper or+ # advisor load builds its own backend (hub/utils/llm_assist.py,+ # utils/datasets/llm_assist.py), which nothing marks, so without this it would+ # still spawn a server after the sweep. Read second: the attribute is cheaper+ # and answers for the instance that actually gets torn down.+ from utils.process_lifetime import is_process_shutting_down++ return is_process_shutting_down()++ def _kill_process(self, *, teardown: bool = False):+ """Terminate the subprocess if running.++ ``teardown`` marks an app-level stop (shutdown, atexit) rather than the+ retry ladder reaping a child it is about to replace: only the former may+ end an in-flight health wait.++ A teardown holds _teardown_lock for the WHOLE kill, because the terminate and+ wait below keep reading self._process and finally clear it: a lifecycle+ reopened mid-kill would have its new child dropped or terminated here, so+ _begin_server_lifecycle takes the same lock and waits.++ _spawn_lock is taken only long enough to set the flag, NOT across the kill.+ A spawn arriving mid-teardown then reads the flag and refuses in microseconds+ instead of queuing behind a SIGTERM/SIGKILL escalation that can run for+ seconds; the load thread it belongs to is one shutdown is already waiting on.+ Marked above the early return, since a quit during a download still has to be+ recorded.+ """+ if teardown:+ # Process-wide as well as per-instance: the atexit teardown reaches here+ # without going through run.py, and the backends a helper load builds for+ # itself are only ever covered by the shared latch.+ from utils.process_lifetime import mark_process_shutting_down++ mark_process_shutting_down()+ with self._teardown_lock:+ with self._spawn_lock:+ self._shutting_down = True+ self._kill_process_body(teardown = True)+ return+ self._kill_process_body(teardown = False)++ def _kill_process_body(self, *, teardown: bool):+ """The kill itself. Caller holds _teardown_lock when ``teardown``."""# Stop the watchdog before a deliberate kill so a planned reload/unload# isn't seen as a crash; a real crash never routes through here.self._stop_mtp_crash_watchdog()@@ -26751,6 +26954,11 @@ class LlamaCppBackend:_pid = getattr(self._process, "pid", None)_pgid = self._leading_process_group(_pid)_descendants = self._collect_descendants(_pid)+ if teardown:+ # Before the signal, and as the process itself: the reference stays set+ # across the waits below, and only identity says which child a teardown+ # landing between a spawn and its wait referred to.+ self._torn_down_process = self._processtry:if terminable:self._process.terminate()@@ -27279,7 +27487,7 @@ class LlamaCppBackend:return killeddef _cleanup(self):- """atexit handler to ensure llama-server is terminated.+ """atexit handler to ensure llama-server is terminated (a teardown).Nothing here may report a failure through the logging machinery. By thetime atexit runs, the streams the handlers write to can already be closed,@@ -27297,7 +27505,7 @@ class LlamaCppBackend:raise_exceptions = logging.raiseExceptionslogging.raiseExceptions = Falsetry:- self._kill_process()+ self._kill_process(teardown = True)# TemporaryDirectory's exit hook runs first and cannot delete a staged# runtime whose server is alive (Windows locks the exe). Retry post-kill.self._cleanup_cpu_fallback_runtime()@@ -28298,8 +28506,17 @@ class LlamaCppBackend:# Why this wait ended, for callers that must tell a cancel apart from a crash:# a cancel landing during CPU-fallback staging is not a cancelled wait.self._health_wait_cancelled = False+ # No teardown reset here: it would erase one that landed between this load's+ # spawn and this line. _torn_down_process is matched by identity instead.+ process = None # the child this wait last looked at, read again after the loopwhile time.monotonic() < deadline:+ # Durable, unlike the per-process marker below: once teardown begins,+ # every later iteration sees it.+ if getattr(self, "_shutting_down", False):+ logger.info("llama-server was torn down while waiting for it to become healthy")+ self._health_wait_cancelled = True+ return False# unload_model() blocks on self._lock, which the load holds across this wait.if cancelled is not None and cancelled():logger.info("llama-server startup cancelled before it became healthy")@@ -28318,6 +28535,12 @@ class LlamaCppBackend:return False# Process crashed?if process.poll() is not None:+ # A teardown holds the reference across its waits, so THIS child+ # exiting under it is deliberate.+ if getattr(self, "_torn_down_process", None) is process:+ logger.info("llama-server was torn down while waiting for it to become healthy")+ self._health_wait_cancelled = True+ return False# Let the drain thread collect final output.if self._stdout_thread is not None:self._stdout_thread.join(timeout = 2)@@ -28347,6 +28570,11 @@ class LlamaCppBackend:logger.info("llama-server became healthy after the load was cancelled")self._health_wait_cancelled = Truereturn False+ # A 200 arriving as shutdown began must not publish _healthy.+ if getattr(self, "_shutting_down", False):+ logger.info("llama-server became healthy while the app was shutting down")+ self._health_wait_cancelled = True+ return Falsereturn Trueexcept (httpx.ConnectError,@@ -28366,6 +28594,16 @@ class LlamaCppBackend:self._health_wait_cancelled = Truereturn False+ # The deadline is the other way out of the loop, so it asks too -- and about+ # both signals, since _kill_process sets _shutting_down on entry but+ # _torn_down_process only after collecting descendants.+ if getattr(self, "_shutting_down", False) or (+ process is not None and getattr(self, "_torn_down_process", None) is process+ ):+ logger.info("llama-server was torn down while waiting for it to become healthy")+ self._health_wait_cancelled = True+ return False+# Leave a marker so _classify_llama_start_failure tells a live but# never-healthy load (too large, or a proxy hijacking the loopback# probe) apart from a bad GGUF (#5740).diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex bb7d64c358c..fcbca67a5f9 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -440,6 +440,15 @@ class InferenceOrchestrator:raise SidecarSwapInProgress("A transformers repair is replacing the latest sidecar; retry when it completes.")+ # Last gate before Popen. A preview or auto-switch load is not a+ # _ScopedLoadAttempt, so the route's shutdown sweep cannot cancel it; it can+ # clear the load's own checks and only then reach here, after the shutdown+ # already stopped this subprocess. Checked at the spawn itself so the answer+ # cannot go stale between the check and the child.+ from utils.process_lifetime import is_process_shutting_down++ if is_process_shutting_down():+ raise RuntimeError("Studio is shutting down; not starting an inference subprocess")from utils.native_path_leases import (native_path_secret_removed_for_child_start,run_without_native_path_secret,@@ -461,7 +470,13 @@ class InferenceOrchestrator:self._cancel_event = _CTX.Event()self._drain_event = _CTX.Event()- self._proc = _CTX.Process(+ # Built into a local FIRST, and started through that local. A shutdown can+ # observe a not-yet-alive child, clear self._proc and finish its sweep while+ # start() is still returning, so the attribute is not a handle this code can+ # rely on from here on: snapshotting it after start() would capture the None+ # and lose the only reference to a live child, which is the orphan this+ # change exists to prevent.+ _spawned_proc = _CTX.Process(target = run_without_native_path_secret,args = ("core.inference.worker", "run_inference_process", cache_env),kwargs = {@@ -473,11 +488,49 @@ class InferenceOrchestrator:},daemon = True,)- self._proc.start()+ self._proc = _spawned_proc+ _spawned_proc.start()from utils.process_lifetime import adopt_pid- adopt_pid(self._proc.pid) # bind to parent lifetime (Windows job / sweep)- logger.info("Inference subprocess started (pid=%s)", self._proc.pid)+ adopt_pid(_spawned_proc.pid) # bind to parent lifetime (Windows job / sweep)++ # The gate above is 30-odd lines and a process start away from here, so a+ # shutdown can begin in between, see no live _proc, and finish its sweep+ # while this child is still being born. Recheck now it exists and reap it,+ # the same shape as the cancel_load recheck below the caller's spawn.+ # A lock across the spawn would close it too, but _shutdown_subprocess holds+ # that lock for its whole teardown, so quitting would then queue behind a+ # spawn it is about to undo. adopt_pid runs first either way: a child that+ # dies here must still be in the sweep record.+ if is_process_shutting_down() or self._proc is not _spawned_proc:+ logger.info("Shutdown began during spawn; tearing the new inference worker down")+ self._shutdown_subprocess(timeout = 5)+ # If shutdown already dropped the mirror, that call cannot see this child.+ # The local handle is the only one left, so reap it here -- and escalate the+ # way _shutdown_subprocess_locked does, rather than abandoning a worker that+ # ignores SIGTERM. The step-7 snapshot has already been taken by this point,+ # so a child left alive here survives until a later startup reaps its record.+ try:+ if _spawned_proc.is_alive():+ _spawned_proc.terminate()+ _spawned_proc.join(5)+ if _spawned_proc.is_alive():+ from utils.process_lifetime import terminate_pid+ terminate_pid(_spawned_proc.pid, timeout = 5)+ _spawned_proc.join(5)+ if _spawned_proc.is_alive():+ _spawned_proc.kill()+ _spawned_proc.join(5)+ if _spawned_proc.is_alive():+ logger.warning(+ "Raced inference worker (pid=%s) survived terminate and kill; "+ "leaving it in the lifetime record for the startup sweep",+ _spawned_proc.pid,+ )+ except Exception as exc:+ logger.debug("Could not reap the raced inference worker: %s", exc)+ raise RuntimeError("Studio is shutting down; not starting an inference subprocess")+ logger.info("Inference subprocess started (pid=%s)", _spawned_proc.pid)def _cancel_generation(self) -> None:"""Cancel any ongoing generation in the subprocess (instant)."""@@ -1733,29 +1786,46 @@ class InferenceOrchestrator:self.active_model_name = Noneself.models.clear()return False+ from utils.process_lifetime import is_process_shutting_down+model_info = resp.get("model_info", {})- self.active_model_name = model_info.get("identifier", model_name)- self.load_generation += 1- # A load always spawns a fresh subprocess holding only this model, so mirror that. A lingering stale- # name would pass unload_model's "not in self.models" guard, and the worker's absent-name fallback- # would unload its *active* model, not the already-gone one.- self.models = {}- self.models[self.active_model_name] = _mirrored_model_entry(- model_info, model_name- )- # Lets the already-loaded shortcut tell a CPU request from the GPU- # model it would otherwise report as satisfied. Native audio only:- # marking anything else tells training a GPU model holds no VRAM.- self.models[self.active_model_name]["audio_cpu"] = model_info.get(- "audio_type"- ) in NATIVE_AUDIO_TYPES and audio_device_forces_cpu(audio_device)- self.models[self.active_model_name].update(- _mlx_runtime_mirror_fields(model_info)- )- # Mirror chat_template_info so routes can classify caps without re-entering the subprocess- _tpl_info = model_info.get("chat_template_info")- if isinstance(_tpl_info, dict):- self.models[self.active_model_name]["chat_template_info"] = _tpl_info+ # A "loaded" reply dequeued just as shutdown kills the worker would+ # otherwise be published here, and active_model_name is what the+ # already-loaded fast path trusts without testing liveness. Held+ # under the lock shutdown kills with, so the check and the+ # publication are one step rather than a race.+ with self._subprocess_shutdown_lock:+ if is_process_shutting_down():+ logger.info(+ "Shutdown overtook the load of '%s'; not publishing it as resident",+ model_name,+ )+ self.loading_models.discard(model_name)+ self.active_model_name = None+ self.models.clear()+ return False+ self.active_model_name = model_info.get("identifier", model_name)+ self.load_generation += 1+ # A load always spawns a fresh subprocess holding only this model, so mirror that. A lingering stale+ # name would pass unload_model's "not in self.models" guard, and the worker's absent-name fallback+ # would unload its *active* model, not the already-gone one.+ self.models = {}+ self.models[self.active_model_name] = _mirrored_model_entry(+ model_info, model_name+ )+ # Lets the already-loaded shortcut tell a CPU request from the GPU+ # model it would otherwise report as satisfied. Native audio only:+ # marking anything else tells training a GPU model holds no VRAM.+ self.models[self.active_model_name]["audio_cpu"] = model_info.get(+ "audio_type"+ ) in NATIVE_AUDIO_TYPES and audio_device_forces_cpu(audio_device)+ self.models[self.active_model_name].update(+ _mlx_runtime_mirror_fields(model_info)+ )+ # Mirror chat_template_info so routes can classify caps without re-entering the subprocess+ _tpl_info = model_info.get("chat_template_info")+ if isinstance(_tpl_info, dict):+ self.models[self.active_model_name]["chat_template_info"] = _tpl_infoself.loading_models.discard(model_name)logger.info("Model '%s' loaded successfully in subprocess", model_name)return Truediff --git a/studio/backend/core/inference/sd_cpp_engine.py b/studio/backend/core/inference/sd_cpp_engine.pyindex 042aec9539c..15dbdbf3055 100644--- a/studio/backend/core/inference/sd_cpp_engine.py+++ b/studio/backend/core/inference/sd_cpp_engine.py@@ -27,7 +27,12 @@ import timefrom pathlib import Pathfrom typing import Callable, Iterator, Optional-from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid+from utils.process_lifetime import (+ adopt_pid,+ child_popen_kwargs,+ forget_pid,+ is_process_shutting_down,+)from utils.native_path_leases import child_env_without_native_path_secretfrom utils.subprocess_compat import windows_hidden_subprocess_kwargsfrom core.inference.sd_cpp_args import (@@ -843,6 +848,11 @@ class SdCppEngine:logger.info("sd-cli run started: %s", summary)t0 = time.time()+ # A generation admitted before the quit can still reach this Popen after the+ # shutdown sweep has taken its snapshot, and sd-cli would then keep running with+ # nothing left to reap it, holding VRAM past the app.+ if is_process_shutting_down():+ raise SdCppCancelled("Studio is shutting down; not starting sd-cli.")proc = subprocess.Popen(cmd,stdout = subprocess.PIPE,@@ -860,6 +870,13 @@ class SdCppEngine:# the kwargs above are empty on macOS, so record it too, else a crash mid-generation leaves sd-cli holding VRAM# with nothing able to find itadopt_pid(proc.pid)+ # Recheck once the pid is recorded, for the window between the gate above and this+ # record. Adoption ran first, so the child killed here was in the sweep record for+ # as long as it existed.+ if is_process_shutting_down():+ logger.info("shutdown began during the spawn; killing the new sd-cli")+ _terminate(proc)+ raise SdCppCancelled("Studio is shutting down; not starting sd-cli.")# Drain stdout on a reader thread so the timeout holds even when the child hangs WITHOUT printing (a plain `for# line in proc.stdout` blocks until EOF). Lines, then a None sentinel, go to a queue the main loop polls against# a wall-clock deadline. iter_sd_cpp_records also splits sd-cli's in-place progress redraws, which carry nodiff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.pyindex 70448f78ecf..7c1f34dd671 100644--- a/studio/backend/core/inference/sd_cpp_server.py+++ b/studio/backend/core/inference/sd_cpp_server.py@@ -56,7 +56,12 @@ from core.inference.sd_cpp_engine import (runtime_env,)from utils.native_path_leases import child_env_without_native_path_secret-from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid+from utils.process_lifetime import (+ adopt_pid,+ child_popen_kwargs,+ forget_pid,+ is_process_shutting_down,+)from utils.subprocess_compat import windows_hidden_subprocess_kwargslogger = logging.getLogger(__name__)@@ -260,6 +265,16 @@ class SdCppServer:# Spawn INSIDE the long-lived drain thread: child_popen_kwargs() sets PR_SET_PDEATHSIG, bound to the# creating thread on Linux, so the creator must outlive the child.def _own_process() -> None:+ # One flag at every spawn. No _graceful_shutdown step unloads the+ # diffusion engine and cancel_pending_loads only knows about chat loads,+ # so without this a quit during an /images/load can start a multi-GB+ # sd-server after the step-7 sweep has taken its snapshot.+ if is_process_shutting_down():+ self._spawn_error = RuntimeError(+ "Studio is shutting down; not starting sd-server"+ )+ spawned.set()+ returntry:proc = subprocess.Popen(cmd,@@ -279,6 +294,33 @@ class SdCppServer:self._process = procself.port = portadopt_pid(proc.pid) # so a global shutdown sweep also reaps it+ # Recheck once the pid is recorded, for the window between the gate and+ # this record. Adoption runs first, so a child killed here was in the+ # sweep record for as long as it existed.+ if is_process_shutting_down():+ logger.info("shutdown began during the spawn; killing the new sd-server")+ try:+ proc.kill()+ proc.wait(timeout = 5)+ except Exception: # noqa: BLE001 - the reap is best-effort+ pass+ # Only drop the record once the child is confirmed gone. If the kill+ # or the wait raised, the server is still alive with the sweep+ # already past it, and this record is the last thing that could+ # reap it; forgetting it here would be the orphan this PR is about.+ if proc.poll() is not None:+ forget_pid(proc.pid)+ else:+ logger.warning(+ "sd-server pid %s survived the shutdown kill; leaving it adopted",+ proc.pid,+ )+ self._process = None+ self._spawn_error = RuntimeError(+ "Studio is shutting down; not starting sd-server"+ )+ spawned.set()+ returnspawned.set()self._drain_stdout(proc)try:diff --git a/studio/backend/core/inference/stt_ggml_sidecar.py b/studio/backend/core/inference/stt_ggml_sidecar.pyindex f2a63212195..5f84dcb4de5 100644--- a/studio/backend/core/inference/stt_ggml_sidecar.py+++ b/studio/backend/core/inference/stt_ggml_sidecar.py@@ -71,7 +71,12 @@ from core.inference.stt_sidecar import (from utils.prebuilt.child_env import isolate_home, scrub_env, wsl_system_rocm_lib_dirsfrom utils.prebuilt.runtime_libs import dedupe_existing_dirsfrom utils.prebuilt.whisper_layout import lookup_marker-from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid+from utils.process_lifetime import (+ adopt_pid,+ child_popen_kwargs,+ forget_pid,+ is_process_shutting_down,+)logger = get_logger(__name__)@@ -1000,6 +1005,13 @@ class GgmlSttSidecar:self._release_locked()# release the reservation as late as possible: whisper-server binds the port moments after this closereservation.close()+ # One flag at every spawn. No _graceful_shutdown step stops this+ # sidecar, so a load still downloading or in preflight as the app quits+ # would otherwise start whisper-server after the sweep had run.+ if is_process_shutting_down():+ raise SttLoadCancelledError(+ "Studio is shutting down; not starting whisper-server."+ )process = subprocess.Popen(command,stdout = subprocess.DEVNULL,@@ -1015,6 +1027,18 @@ class GgmlSttSidecar:with self._load_state_lock:self._starting_process = processadopt_pid(process.pid) # terminate_all backstop for graceful exits+ # Recheck once the pid is recorded: the latch can be set between the+ # gate above and this record, and the child would then sit outside a+ # sweep that has already finished. Adoption runs first either way, so a+ # child killed here is still in the sweep record.+ if is_process_shutting_down():+ if process.poll() is None:+ process.kill()+ process.wait(timeout = 10)+ forget_pid(process.pid)+ raise SttLoadCancelledError(+ "Studio is shutting down; not starting whisper-server."+ )try:self._wait_for_server(process, port, cancel_event)except Exception:diff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.pyindex 8c6c8396a72..78e98091035 100644--- a/studio/backend/core/inference/stt_mtmd_sidecar.py+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py@@ -30,7 +30,12 @@ from typing import Iterator, Optionalfrom loggers import get_loggerfrom hub.utils.hf_tokens import normalize_token-from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid+from utils.process_lifetime import (+ adopt_pid,+ child_popen_kwargs,+ forget_pid,+ is_process_shutting_down,+)from core.inference.stt_ggml_sidecar import _pcm_to_wav_bytesfrom core.inference.stt_sidecar import (@@ -184,8 +189,18 @@ def _reap(process: Optional[subprocess.Popen]) -> None:except Exception as exc: # noqa: BLE001 - shutdown must not raiselogger.warning("Could not reap llama-server (pid %s): %s", process.pid, exc)finally:- # the PID is dead, so drop it before it can be reused by something else that terminate_all would then signal- forget_pid(process.pid)+ # Drop the pid once it is dead, before it can be reused by something else that+ # terminate_all would then signal. Only once it is dead: if the terminate, the+ # kill or either wait raised, the child is still alive, and after the shutdown+ # sweep has passed this record is the last thing that could reap it.+ if process.poll() is not None:+ forget_pid(process.pid)+ else:+ logger.warning(+ "llama-server (pid %s) survived the reap; leaving it adopted so a later "+ "sweep can still find it",+ process.pid,+ )def _cached_file(@@ -920,6 +935,13 @@ class MtmdSttSidecar:# chat backend's _cmd_has_gpu_companion() treats as a GPU companion whatever --gpu-layers says.cmd.append("--no-mmproj-offload")sock.close()+ # One flag at every spawn, as above: nothing in _graceful_shutdown stops+ # this sidecar, so without it a quit during a load starts a server the+ # step-7 sweep has already passed by.+ if is_process_shutting_down():+ raise SttLoadCancelledError(+ "Studio is shutting down; not starting the MTMD server."+ )process = subprocess.Popen(cmd,# nothing reads these, and an undrained pipe blocks llama-server mid-startup once its logs fill the@@ -938,6 +960,13 @@ class MtmdSttSidecar:with self._lock:self._starting_process = processadopt_pid(process.pid) # terminate_all backstop for graceful exits+ # Recheck once the pid is recorded, for the window between the gate and the+ # record. _reap kills and forgets, so nothing is left half-tracked.+ if is_process_shutting_down():+ _reap(process)+ raise SttLoadCancelledError(+ "Studio is shutting down; not starting the MTMD server."+ )if not self._wait_for_server(process, port, cancel_event):# Reap it here: _process was never assigned, so unload() cannot reach a child that ignores SIGTERM and# keeps port and VRAM.diff --git a/studio/backend/core/inference/stt_transformers_worker.py b/studio/backend/core/inference/stt_transformers_worker.pyindex 6346088c2d0..4eb23b2fb2d 100644--- a/studio/backend/core/inference/stt_transformers_worker.py+++ b/studio/backend/core/inference/stt_transformers_worker.py@@ -339,8 +339,13 @@ class WhisperWorker:native_path_secret_removed_for_child_start,run_without_native_path_secret,)- from utils.process_lifetime import adopt_pid+ from utils.process_lifetime import adopt_pid, is_process_shutting_down+ # One flag at every spawn: no _graceful_shutdown step unloads this sidecar and+ # cancel_pending_loads only reaches the chat /load attempts, so without this a+ # quit during an STT load starts a worker the step-7 sweep has already passed.+ if is_process_shutting_down():+ raise SttWorkerSpawnError("Studio is shutting down; not starting the dictation worker.")cache_env = get_hf_cache_paths().child_env({})try:with (@@ -365,16 +370,37 @@ class WhisperWorker:},daemon = True,)- self._process.start()+ # Local handle, and started through it: a concurrent teardown can clear+ # self._process while start() is still returning, and re-reading the+ # attribute afterwards would lose the only reference to a live child.+ _spawned_proc = self._process+ _spawned_proc.start()except Exception as exc: # noqa: BLE001 - any refusal to spawn reads the sameself._process = Noneself._close_queues()raise SttWorkerSpawnError(f"Could not start the dictation worker process: {exc}") from exc- adopt_pid(self._process.pid) # terminate_all backstop for graceful exits+ adopt_pid(_spawned_proc.pid) # terminate_all backstop for graceful exits+ # Recheck once the pid is recorded: the latch can be set between the gate above+ # and this record. Adoption runs first, so a child reaped here is still in the+ # sweep record.+ if is_process_shutting_down():+ logger.info("shutdown began during the spawn; reaping the new dictation worker")+ try:+ if _spawned_proc.is_alive():+ _spawned_proc.terminate()+ _spawned_proc.join(5)+ if _spawned_proc.is_alive():+ _spawned_proc.kill()+ _spawned_proc.join(5)+ except Exception: # noqa: BLE001 - the reap is best-effort+ pass+ self._process = None+ self._close_queues()+ raise SttWorkerSpawnError("Studio is shutting down; not starting the dictation worker.")logger.info(- "STT worker started (pid=%s) for %s on %s", self._process.pid, snapshot_path, device+ "STT worker started (pid=%s) for %s on %s", _spawned_proc.pid, snapshot_path, device)try:self._send(diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.pyindex 799244d3984..0900a7c61a1 100644--- a/studio/backend/core/rag/embed_llama_server.py+++ b/studio/backend/core/rag/embed_llama_server.py@@ -33,7 +33,12 @@ import numpy as npfrom utils.native_path_leases import child_env_without_native_path_secretfrom utils.subprocess_compat import windows_hidden_subprocess_kwargs-from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid+from utils.process_lifetime import (+ adopt_pid,+ child_popen_kwargs,+ forget_pid,+ is_process_shutting_down,+)from . import configfrom utils.paths.path_utils import is_appledouble_metadata@@ -819,6 +824,10 @@ class LlamaServerBackend:try:self._spawn_once(use_gpu, model_name)except RuntimeError:+ # A shutdown refusal is terminal: the CPU fallback would just spawn into the+ # same latch and be refused again.+ if is_process_shutting_down():+ raiseif use_gpu and not config.embed_device_requires_gpu():logger.warning("embed server GPU start failed; falling back to CPU")self._force_cpu = True@@ -842,6 +851,11 @@ class LlamaServerBackend:" ".join(cmd),)self._stdout_lines = []+ # One flag at every spawn. No _graceful_shutdown step stops this backend, so an+ # encode still resolving or downloading its model as the app quits would+ # otherwise Popen a server after terminate_all had taken its snapshot.+ if is_process_shutting_down():+ raise RuntimeError("Studio is shutting down; not starting the embed server")proc = subprocess.Popen(cmd,stdout = subprocess.PIPE,@@ -857,6 +871,14 @@ class LlamaServerBackend:# child_popen_kwargs() is empty on macOS, so the crash record is the only thing that can reap it# after a force quit.adopt_pid(proc.pid)+ # Recheck once the pid is recorded, as the llama-server and inference worker+ # spawns do: the latch can be set between the gate above and this record, and+ # the child would then sit outside a sweep that has already finished. Adoption+ # runs first either way, so a child killed here is still in the sweep record.+ if is_process_shutting_down():+ logger.info("shutdown began during the spawn; killing the new embed server")+ self._kill_process()+ raise RuntimeError("Studio is shutting down; not starting the embed server")self._port = portself._stdout_thread = threading.Thread(target = self._drain_stdout,diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.pyindex 77d3663ab4b..777437852cd 100644--- a/studio/backend/core/training/training.py+++ b/studio/backend/core/training/training.py@@ -1717,7 +1717,7 @@ class TrainingBackend:},daemon = True,)- from utils.process_lifetime import adopt_pid+ from utils.process_lifetime import adopt_pid, is_process_shutting_downprevious_job_id = Noneprevious_start_request_id = None@@ -1731,6 +1731,16 @@ class TrainingBackend:start_request_id,)return False+ # The cancel check above is about this start request; the latch is+ # about the process. A start admitted before the quit can still+ # reach here after the shutdown sweep has taken its snapshot, and+ # the worker would then train on past it holding the GPU.+ if is_process_shutting_down():+ logger.info(+ "Studio is shutting down; not starting training worker for %s",+ start_request_id,+ )+ return Falseprevious_job_id = self.current_job_idprevious_start_request_id = self.current_start_request_idproc.start()@@ -1738,9 +1748,16 @@ class TrainingBackend:self.current_start_request_id = start_request_idtry:adopt_pid(proc.pid)+ # Recheck once the pid is recorded, for the window between the+ # gate above and this record. Raised rather than handled inline+ # so it reuses the terminate ladder and the state rollback below;+ # adoption ran first, so the worker is in the sweep record for as+ # long as it exists.+ if is_process_shutting_down():+ raise RuntimeError("Studio is shutting down")except Exception:logger.error(- "Failed to adopt training subprocess; terminating it",+ "Could not keep the training subprocess; terminating it",exc_info = True,)try:@@ -2416,10 +2433,37 @@ class TrainingBackend:},daemon = True,)- new_proc.start()- from utils.process_lifetime import adopt_pid+ from utils.process_lifetime import adopt_pid, is_process_shutting_down+ # A stall recovery that started before the quit can still reach+ # this respawn after the shutdown sweep has taken its snapshot.+ if is_process_shutting_down():+ raise RuntimeError(+ "Studio is shutting down; not respawning the training worker"+ )+ new_proc.start()adopt_pid(new_proc.pid)+ # Recheck once the pid is recorded, for the window between the gate+ # above and this record. Adoption ran first, so the worker killed+ # here was in the sweep record for as long as it existed.+ if is_process_shutting_down():+ logger.info(+ "shutdown began during the respawn; killing the new training worker"+ )+ try:+ if new_proc.is_alive():+ new_proc.terminate()+ new_proc.join(timeout = 5.0)+ if new_proc.is_alive():+ new_proc.kill()+ new_proc.join(timeout = 2.0)+ except Exception: # noqa: BLE001 - the reap is best-effort+ logger.warning(+ "could not reap the new training worker", exc_info = True+ )+ raise RuntimeError(+ "Studio is shutting down; not respawning the training worker"+ )except Exception:logger.error("Failed to respawn training subprocess", exc_info = True)self._spawn_in_progress = Falsediff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 0020e915919..e85181f43d2 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -13787,6 +13787,11 @@ _scoped_load_attempts: dict[tuple[str, str], _ScopedLoadAttempt] = {}_scoped_load_cancel_tombstones: dict[tuple[str, str], tuple[str, float]] = {}_running_load_attempt: Optional[_ScopedLoadAttempt] = None_pending_load_attempts: dict[str, _ScopedLoadAttempt] = {}+# Latched by cancel_pending_loads, cleared by begin_load_lifecycle. A snapshot alone+# misses a request uvicorn already admitted but schedules while shutdown is running:+# should_exit stops new connections, not existing request tasks. Lifecycle-scoped, not+# permanent, or an embedded host's second run_server could never load anything.+_loads_shutting_down = False_SCOPED_LOAD_CANCEL_TOMBSTONE_TTL_S = 60.0# Bound on waiting for a cancel's teardown to report back. Only the /unload# handler sets cancel_complete for a running attempt, so a disconnect or a@@ -13797,6 +13802,50 @@ _SCOPED_LOAD_CANCEL_HANDSHAKE_TIMEOUT_S = 15.0_SCOPED_LOAD_CANCEL_TOMBSTONE_LIMIT_PER_SUBJECT = 256+def cancel_pending_loads() -> int:+ """Cancel every in-flight /load. Called by the app shutdown, before the kill.++ The backend's shutdown flag only guards its own spawn, and a request between+ admission and that call is not yet holding anything the flag can see: it can+ sit in the lifecycle gate or preflight for minutes, then reach the backend in+ a lifecycle that has already been reset and load a model the new server never+ asked for. Cancelling through the attempt's own event stops it wherever it is,+ including mid-download, using the path /unload already uses.++ Best-effort and non-blocking: shutdown must not wait on a load's teardown.+ """+ global _loads_shutting_down+ with _scoped_load_attempts_lock:+ _loads_shutting_down = True+ attempts = list(_pending_load_attempts.values())+ running = _running_load_attempt+ if running is not None and all(a.token != running.token for a in attempts):+ attempts.append(running)+ for attempt in attempts:+ _cancel_for_shutdown(attempt)+ return len(attempts)+++def _cancel_for_shutdown(attempt: _ScopedLoadAttempt) -> None:+ """Cancel an attempt AND close its handshake.++ Only /unload sets cancel_complete, and at shutdown there is no /unload to do it,+ so setting cancel_event alone leaves _run_tracked_load_model_impl's finally+ waiting the full handshake timeout in a to_thread. Those executor threads are+ non-daemon and would hold the process open. Shutdown owns the teardown, so there+ is nothing to report back.+ """+ attempt.cancel_event.set()+ attempt.cancel_complete.set()+++def begin_load_lifecycle() -> None:+ """Clear the shutdown latch so a restarted server accepts loads again."""+ global _loads_shutting_down+ with _scoped_load_attempts_lock:+ _loads_shutting_down = False++def _prune_scoped_load_cancel_tombstones(now: float) -> None:expired = [key for key, (_, expires_at) in _scoped_load_cancel_tombstones.items() if expires_at <= now@@ -13971,6 +14020,12 @@ async def load_model_gated(attempt = _begin_load_attempt(request, current_subject)with _scoped_load_attempts_lock:_pending_load_attempts[attempt.token] = attempt+ # Registered after the shutdown sweep took its snapshot, so nothing else+ # will ever cancel it. Under the same lock as the latch, so it cannot+ # register between the latch and the sweep either.+ _shutting_down_now = _loads_shutting_down+ if _shutting_down_now:+ _cancel_for_shutdown(attempt)try:_raise_if_sidecar_swap_in_progress()# Hold the lifecycle gate across the load so idle auto-unload can't unload the@@ -14057,6 +14112,16 @@ async def _load_model_impl(if load_cancel_event is not None and load_cancel_event.is_set():raise HTTPException(status_code = 409, detail = "Model load cancelled")+ # Auto-switch and preview call this impl directly, without a _ScopedLoadAttempt,+ # so the shutdown sweep has no event to set for them. Reading the latch here puts+ # both on the same footing as /load: the callers of this helper are the points of+ # no return, so a shutdown seen before one still stops the load rather than+ # spawning a worker that outlives quit.+ with _scoped_load_attempts_lock:+ _shutting_down_now = _loads_shutting_down+ if _shutting_down_now:+ raise HTTPException(status_code = 409, detail = "Model load cancelled")+# A new load starts here; arm the progress throttle so this load's first# sampled step logs even if it reports 100% immediately (cached/small load)._reset_load_progress_step()diff --git a/studio/backend/run.py b/studio/backend/run.pyindex 6e6f7535130..695411430a8 100644--- a/studio/backend/run.py+++ b/studio/backend/run.py@@ -1599,6 +1599,17 @@ def _graceful_shutdown(server = None):"""logger.info("Graceful shutdown initiated -- cleaning up subprocesses...")+ # 0a. Latch "quitting" before any subsystem is torn down. Each step below refuses to+ # respawn its OWN child once it has run, but a load still in flight can reach a+ # different spawner afterwards: the orchestrator is stopped at step 2 and swept at+ # step 7, and a helper load owns a backend no step touches at all. One flag, read at+ # every spawn, covers the gaps between the steps.+ try:+ from utils.process_lifetime import mark_process_shutting_down+ mark_process_shutting_down()+ except Exception as e:+ logger.warning("Could not latch the process shutdown flag: %s", e)+# 0. Drop the LAN listener first: it shares the loop uvicorn is about to stop.try:from lan_access import close_lan_listener_lifecycle@@ -1636,9 +1647,21 @@ def _graceful_shutdown(server = None):# 5. Kill llama-server subprocess (if loaded).try:- from routes.inference import _llama_cpp_backend+ from routes.inference import _llama_cpp_backend, cancel_pending_loads++ # Before the kill: a load still in the lifecycle gate or in preflight is not yet+ # holding anything the backend's own flag can see, and would spawn llama-server+ # after this step had already run.+ try:+ cancelled = cancel_pending_loads()+ if cancelled:+ logger.info("Cancelled %d in-flight model load(s) for shutdown", cancelled)+ except Exception as e:+ logger.warning("Could not cancel in-flight loads: %s", e)if _llama_cpp_backend is not None:- _llama_cpp_backend._kill_process()+ # teardown = True: an app-level stop, not the retry ladder reaping a child it+ # is about to replace. Only the former may end an in-flight health wait.+ _llama_cpp_backend._kill_process(teardown = True)except Exception as e:logger.warning("Error shutting down llama-server: %s", e)@@ -3005,6 +3028,23 @@ def run_server(_close_lan_listener()+ # An embedded host (studio/backend/colab.py) can call run_server again in the same+ # interpreter, and the shutdown flags are process- and module-wide, so a second+ # session would otherwise refuse every spawn and every load it admitted. Cleared+ # here, after `from main import app` above (reaching for the route module earlier+ # would build the backend singleton ahead of the startup steps that must come+ # first) and before uvicorn serves anything below.+ try:+ from routes.inference import _llama_cpp_backend, begin_load_lifecycle+ from utils.process_lifetime import begin_process_lifecycle++ if _llama_cpp_backend is not None:+ _llama_cpp_backend._begin_server_lifecycle()+ begin_process_lifecycle()+ begin_load_lifecycle()+ except Exception as e:+ logger.warning("Could not reset llama-server shutdown state: %s", e)+thread = Thread(target = _run, daemon = True)_server_thread = threadthread.start()@@ -3041,9 +3081,17 @@ def run_server(import atexitatexit.register(_remove_pid_file)- from utils.process_lifetime import terminate_all+ from utils.process_lifetime import mark_process_shutting_down, terminate_allatexit.register(terminate_all)+ # LAST, so it runs FIRST: atexit is LIFO. Without it the sweep above takes its+ # snapshot with the latch never set, because the only thing that sets it on this+ # path is the backend's own _cleanup hook, registered when the routes singleton was+ # built and therefore run AFTER this one. An embedded caller that lets the+ # interpreter exit without _graceful_shutdown would otherwise get the unguarded+ # behaviour this change exists to remove: a load still in flight passes its latch+ # check and adopts a child the sweep has already gone past.+ atexit.register(mark_process_shutting_down)# Output port for Tauri (api-only), only after sockets bind and startup done.# The headless `run --api-only` path opts out so it does not leak this line.diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex dd419ee57c5..19ffe04e18a 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -967,3 +967,39 @@ def _no_carried_over_hardware_measurements():_clear()yield_clear()++[email protected](autouse = True)+def _process_shutdown_latch_is_clear():+ """Clear the process-wide shutdown latch around every test.++ The latch is deliberately sticky in production: quitting is terminal, and only an+ embedded host calling run_server again clears it. In a suite that makes it a+ global one test can leave set for the rest of the file, and any test that tears a+ backend down sets it, so a later spawn test sees a stale "quitting" and fails in+ whatever order pytest happens to pick.+ """+ from utils import process_lifetime++ def _reopen():+ process_lifetime.begin_process_lifecycle()+ # The ROUTE latch too. Any test that exercises _graceful_shutdown reaches+ # cancel_pending_loads, which sets it, and only run_server clears it -- so one+ # such test cancels every load admitted by every test that follows it. That is+ # how four tunnel-safe tests came to fail in a full run and pass alone.+ # Only if it is ALREADY imported. Importing it here would drag a heavy module+ # into every test that never asked for it, which perturbed source-contract and+ # import-order tests elsewhere; and the latch cannot have been set without the+ # module being loaded, so there is nothing to miss.+ mod = sys.modules.get("routes.inference")+ if mod is not None:+ try:+ mod.begin_load_lifecycle()+ except Exception:+ pass++ _reopen()+ try:+ yield+ finally:+ _reopen()diff --git a/studio/backend/tests/test_gpu_init_crash_message.py b/studio/backend/tests/test_gpu_init_crash_message.pyindex 929d5251837..7f9139a6985 100644--- a/studio/backend/tests/test_gpu_init_crash_message.py+++ b/studio/backend/tests/test_gpu_init_crash_message.py@@ -1072,7 +1072,7 @@ def test_the_exit_handler_removes_the_runtime_after_the_kill():cannot delete a runtime whose server is still holding the files open."""backend = LlamaCppBackend()order = []- backend._kill_process = lambda: order.append("kill")+ backend._kill_process = lambda **_kw: order.append("kill")backend._cleanup_cpu_fallback_runtime = lambda: order.append("clean")backend._cleanup()diff --git a/studio/backend/tests/test_llama_cpp_atexit_quiet.py b/studio/backend/tests/test_llama_cpp_atexit_quiet.pyindex 2595dad39c2..48ce8fb35ee 100644--- a/studio/backend/tests/test_llama_cpp_atexit_quiet.py+++ b/studio/backend/tests/test_llama_cpp_atexit_quiet.py@@ -131,13 +131,22 @@ def test_the_atexit_handler_quiets_stdlib_loggers_too(monkeypatch, capsys):other.addHandler(handler)other.propagate = False- def kill_and_log():+ ran = []++ # **_kw because _cleanup calls _kill_process(teardown = True). A double that only+ # accepts () raises TypeError inside a handler that swallows everything, so the+ # write this test exists to make would never happen and nothing would say so.+ def kill_and_log(**_kw):+ ran.append(1)other.warning("something a dependency logs at exit")backend = _stub()monkeypatch.setattr(backend, "_kill_process", kill_and_log)try:backend._cleanup()+ # _cleanup swallows everything, so a double whose signature stops+ # matching would silently skip the write this test exists to make.+ assert ran, "the kill double never ran; this assertion proves nothing"assert capsys.readouterr().err == ""finally:other.removeHandler(handler)@@ -220,9 +229,15 @@ def test_a_failing_kill_does_not_escape_the_atexit_handler(monkeypatch):"""atexit swallows it anyway, and there is nowhere left to report it."""backend = _stub()- def boom():+ raised = []++ def boom(**_kw):+ raised.append(1)raise RuntimeError("teardown went wrong")monkeypatch.setattr(backend, "_kill_process", boom)backend._cleanup()+ # Same trap: a TypeError from a stale signature is swallowed too, and then+ # the RuntimeError this test is about is never raised at all.+ assert raised, "the failing kill never ran; the handler swallowed the wrong error"diff --git a/studio/backend/tests/test_llama_cpp_wait_for_health.py b/studio/backend/tests/test_llama_cpp_wait_for_health.pyindex 7eb2493e36e..723b1f6d7b8 100644--- a/studio/backend/tests/test_llama_cpp_wait_for_health.py+++ b/studio/backend/tests/test_llama_cpp_wait_for_health.py@@ -11,6 +11,7 @@ subprocess.poll() branch so a crashed llama-server surfaces a structuredfrom __future__ import annotations+import subprocessimport sysimport threadingimport time@@ -34,7 +35,6 @@ import httpx # noqa: E402from core.inference.llama_cpp import LlamaCppBackend # noqa: E402from core.inference.llama_cpp import GgufLoadIntent-import subprocess# Sibling tests install lightweight httpx stubs, so when collected together our `httpx`# may be a stub lacking `get`. Fill in the gaps so collection order does not matter.@@ -167,6 +167,233 @@ class TestWaitForHealthResilience:t.join(5.0)assert b._health_wait_cancelled is True+ def test_a_signalled_exit_during_teardown_is_not_a_startup_crash(self, monkeypatch):+ """_kill_process holds the reference across its terminate/wait, so for up to+ ~10s the loop sees a populated process whose poll() is the SIGTERM code. The+ None check alone only covers teardown AFTER cleanup finished, so the kill+ publishes its terminal state before signalling; otherwise -15 reads as a+ startup crash and the retry ladder respawns a server during shutdown."""+ b = _make_backend()+ b._process.poll.return_value = None++ def probe(*a, **kw):+ # The shutdown thread mid-wait: publish, then signal, reference still+ # set. Here rather than before the call because the wait resets on+ # entry, so only a teardown landing DURING a wait is this case.+ b._torn_down_process = b._process+ b._process.poll.return_value = -15+ b._process.returncode = -15+ return mock.Mock(status_code = 503)++ monkeypatch.setattr(httpx, "get", probe)+ with mock.patch("core.inference.llama_cpp.logger") as log:+ assert b._wait_for_health(timeout = 1.0, interval = 0.01) is False+ assert b._health_wait_cancelled is True+ assert not any("exited with code" in str(c) for c in log.error.call_args_list)++ def test_a_teardown_between_the_spawn_and_the_wait_still_counts(self, monkeypatch):+ """Shutdown can land after Popen publishes the child but before the wait+ starts. A marker reset on wait entry would erase it, and the loop would+ then read the signalled exit as a startup crash and let the fallbacks+ respawn during shutdown, which is the race this whole change is about.+ Keyed to the process, so it survives until the wait that owns it reads it."""+ b = _make_backend()+ b._torn_down_process = b._process # teardown, before the wait is entered+ b._process.poll.return_value = -15+ b._process.returncode = -15+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))+ with mock.patch("core.inference.llama_cpp.logger") as log:+ assert b._wait_for_health(timeout = 1.0, interval = 0.01) is False+ assert b._health_wait_cancelled is True+ assert not any("exited with code" in str(c) for c in log.error.call_args_list)++ def test_shutdown_is_terminal_however_late_it_lands(self, monkeypatch):+ """The durable half of the guard. Every per-process snapshot has an instant+ after it where teardown can still land, so this flag only goes false to+ true and is re-read each iteration; a wait in flight when shutdown begins+ ends terminally no matter where in the loop the flag was set."""+ b = _make_backend()+ b._process.poll.return_value = None++ def probe(*a, **kw):+ b._shutting_down = True # run.py's shutdown, at an arbitrary instant+ return mock.Mock(status_code = 503)++ monkeypatch.setattr(httpx, "get", probe)+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False+ assert b._health_wait_cancelled is True+ assert not any("health check timed out" in ln for ln in b._stdout_lines)++ def test_kill_process_marks_shutdown_only_for_a_teardown(self):+ """The flag must not be set by the retry ladder reaping its own child, or+ the first crash retry would look like a shutdown and abort the load."""+ for teardown in (True, False):+ b = _make_backend()+ b._stop_mtp_crash_watchdog = lambda *a, **kw: None+ b._reset_effective_parallel_slots = lambda *a, **kw: None+ b._leading_process_group = lambda *a, **kw: None+ b._collect_descendants = lambda *a, **kw: []+ b._kill_process_group = lambda *a, **kw: None+ b._terminate_descendants = lambda *a, **kw: None+ b._process.poll.return_value = 0+ b._kill_process(teardown = teardown)+ assert getattr(b, "_shutting_down", False) is teardown++ def test_a_started_shutdown_refuses_to_spawn(self, monkeypatch):+ """_start_llama_process is the chokepoint the mmproj text-only retry uses+ without passing _spawn_and_wait's boundary check, so it refuses too, and+ says so: a silent refusal leaves the caller health-waiting on the previous+ child and then reading a reference the teardown is clearing."""+ import subprocess++ b = _make_backend()+ b._shutting_down = True+ spawned = []+ monkeypatch.setattr(subprocess, "Popen", lambda *a, **kw: spawned.append(1))+ assert b._start_llama_process(["llama-server"], {}, child_gpu_physical_ids = None) is False+ assert spawned == [], "started a server after shutdown had begun"++ def test_a_teardown_with_no_process_still_marks_shutdown(self):+ """Quitting during a download or staging has nothing to kill, so+ _kill_process returns early. Marking teardown only on the path that has a+ process let the load spawn a server after the shutdown sweep finished."""+ b = _make_backend()+ b._process = None+ b._stop_mtp_crash_watchdog = lambda *a, **kw: None+ b._reset_effective_parallel_slots = lambda *a, **kw: None+ b._kill_process(teardown = True)+ assert b._shutting_down is True++ def test_the_published_child_survives_post_spawn_setup(self, monkeypatch):+ """Shutdown can take the spawn lock the instant it is released and clear+ the reference, so post-spawn setup reads the Popen it just made rather than+ self._process. Otherwise recording the pid raises the same AttributeError+ this PR exists to remove."""+ import subprocess++ b = _make_backend()+ b._shutting_down = False+ recorded = []+ b._record_server_pid = lambda pid: recorded.append(pid)+ b._drain_stdout = lambda *a, **kw: None+ b._llama_log_fh = None+ b._process = None+ proc = mock.Mock()+ proc.pid = 4242+ monkeypatch.setattr(subprocess, "Popen", lambda *a, **kw: proc)++ class _TeardownWinsTheLock:+ """Shutdown taking the lock the instant the spawn releases it, which is+ the whole window: it clears the reference before the pid is read."""++ def __enter__(self):+ return None++ def __exit__(self, *_exc):+ b._process = None+ return False++ b._spawn_lock = _TeardownWinsTheLock()+ assert b._start_llama_process(["llama-server"], {}, child_gpu_physical_ids = None) is True+ assert recorded == [4242]++ def test_a_new_server_lifecycle_clears_the_shutdown_state(self):+ """The backend is a module singleton and an embedded host may call+ run_server() again in the same process, so this state is scoped to a+ lifecycle. Left latched, every launch of the second session is refused."""+ b = _make_backend()+ b._shutting_down = True+ b._torn_down_process = b._process+ b._begin_server_lifecycle()+ assert b._shutting_down is False+ assert b._torn_down_process is None++ def test_a_healthy_probe_during_shutdown_is_not_a_healthy_server(self, monkeypatch):+ """The probe blocks for up to 2s. A 200 arriving as teardown begins must not+ publish a child the shutdown is already killing, exactly as for a cancel."""+ b = _make_backend()+ b._process.poll.return_value = None++ def probe(*a, **kw):+ b._shutting_down = True+ return mock.Mock(status_code = 200)++ monkeypatch.setattr(httpx, "get", probe)+ assert b._wait_for_health(timeout = 5.0, interval = 0.01) is False+ assert b._health_wait_cancelled is True++ def test_a_teardown_during_the_last_probe_is_not_reported_as_a_timeout(self, monkeypatch):+ """The deadline is the other way out of the loop. A teardown landing in the+ final probe leaves no iteration to notice it, so without a check here the+ wait returns a plain timeout, the caller sees something retryable, and the+ fallbacks respawn after shutdown killed the first server."""+ b = _make_backend()+ b._process.poll.return_value = None++ def probe(*a, **kw):+ b._torn_down_process = b._process # shutdown, during the last probe+ return mock.Mock(status_code = 503)++ monkeypatch.setattr(httpx, "get", probe)+ assert b._wait_for_health(timeout = 0.02, interval = 0.01) is False+ assert b._health_wait_cancelled is True+ # Not a live-but-never-healthy load, so it must not be classified as one.+ assert not any("health check timed out" in ln for ln in b._stdout_lines)++ def test_a_timeout_with_no_teardown_still_reports_a_timeout(self, monkeypatch):+ """The check above must not swallow the ordinary #5740 classification."""+ b = _make_backend()+ b._process.poll.return_value = None+ b._torn_down_process = mock.Mock() # an earlier child, unrelated+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))+ assert b._wait_for_health(timeout = 0.02, interval = 0.01) is False+ assert b._health_wait_cancelled is False+ assert any("health check timed out" in ln for ln in b._stdout_lines)++ def test_an_earlier_childs_teardown_does_not_end_a_later_wait(self, monkeypatch):+ """The other half of keying on identity: the marker names one child, so a+ load that replaced a torn-down one is not aborted by its predecessor."""+ b = _make_backend()+ b._torn_down_process = mock.Mock() # the previous child, already reaped+ b._process.poll.return_value = 1 # this one really did crash+ b._process.returncode = 1+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))+ with mock.patch("core.inference.llama_cpp.logger") as log:+ assert b._wait_for_health(timeout = 1.0, interval = 0.01) is False+ assert b._health_wait_cancelled is False, "a stale teardown aborted a live load"+ assert any("exited with code 1" in str(c) for c in log.error.call_args_list)++ @pytest.mark.parametrize("teardown", [True, False])+ def test_kill_process_publishes_a_teardown_before_it_signals(self, teardown):+ """The ordering the test above depends on: published before terminate(),+ not in the finally that clears the reference.++ Only for a teardown. The retry ladder reaps a crashed child through this+ same method between attempts, and marking that terminal would abort loads+ the --fit off and CPU fallbacks currently recover."""+ b = _make_backend()+ b._torn_down_process = None+ b._stop_mtp_crash_watchdog = lambda *a, **kw: None+ b._reset_effective_parallel_slots = lambda *a, **kw: None+ b._leading_process_group = lambda *a, **kw: None+ b._collect_descendants = lambda *a, **kw: []+ b._kill_process_group = lambda *a, **kw: None+ b._terminate_descendants = lambda *a, **kw: None+ seen = {}++ def _terminate():+ # What a racing _wait_for_health would observe at this instant.+ seen["flag_at_signal"] = getattr(b, "_torn_down_process", None) is b._process+ seen["process_still_set"] = b._process is not None++ b._process.terminate = _terminate+ b._process.poll.return_value = -15+ b._kill_process(teardown = teardown)+ assert seen["process_still_set"] is True, "the window this guards would not exist"+ assert (+ seen["flag_at_signal"] is teardown+ ), "teardown published late, or a reap marked terminal"+def test_a_crash_still_reports_the_exit_code(self, monkeypatch):"""The teardown guard must not swallow the crash branch: an exitedprocess still has to name its exit code and its output."""@@ -856,3 +1083,505 @@ def test_a_cancel_after_audio_setup_unloads_the_codec(tmp_path, monkeypatch):codec.unload.assert_called_once_with()assert LlamaCppBackend._codec_mgr is Noneassert emptied == [1]+++def _run_server_body() -> str:+ """run_server's source, without importing run.py.++ Importing it pulls in fastapi and the whole route tree, which is exactly the+ cost the ordering below exists to keep out of the early startup path.+ """+ import ast++ run_py = Path(__file__).resolve().parent.parent / "run.py"+ tree = ast.parse(run_py.read_text(encoding = "utf-8"))+ for node in tree.body:+ if isinstance(node, ast.FunctionDef) and node.name == "run_server":+ return ast.get_source_segment(run_py.read_text(encoding = "utf-8"), node) or ""+ raise AssertionError("run_server not found in run.py")+++def test_the_lifecycle_reset_does_not_import_the_route_module_early():+ """The reset must not be what first builds the llama-server backend.++ `from routes.inference import ...` constructs the module singleton, which+ sweeps orphan llama-servers and registers an atexit handler. run_server+ documents five things as having to happen before any of that: the Windows+ UTF-8 reconfigure, the session log that catches import-time crashes, the+ structlog setup (whose cache_logger_on_first_use pins any logger that has+ already emitted), initialize_parent_lifetime() before a child can spawn, and+ write_startup_marker() before a sweep can reap a sibling's server.++ Ordering it after `from main import app` makes the import free: main imports+ the route package, so the singleton already exists and this is a lookup.+ """+ body = _run_server_body()++ main_import = body.index("from main import app")+ reset = body.index("_begin_server_lifecycle()")+ assert reset > main_import, (+ "the lifecycle reset imports routes.inference before `from main import app`, "+ "so it, not main, builds the backend singleton -- ahead of the startup steps "+ "run_server requires to come first"+ )++ for earlier in (+ 'sys.stdout.reconfigure(encoding = "utf-8"',+ "_setup_server_disk_logging()",+ "LogConfig.setup_logging(",+ "initialize_parent_lifetime()",+ "write_startup_marker()",+ ):+ assert body.index(earlier) < reset, f"{earlier} must precede the lifecycle reset"+++def test_the_lifecycle_reset_stays_below_the_argument_checks():+ """A rejected invocation must not touch the backend at all."""+ body = _run_server_body()++ assert body.index("choose an explicit port.") < body.index(+ "_begin_server_lifecycle()"+ ), "the reset runs before run_server has finished rejecting bad arguments"+++def test_the_lifecycle_reset_is_the_last_thing_before_the_serve():+ """Clearing the shutdown flag is what lets a spawn through, so nothing that can+ abort startup may follow it.++ An embedded host restarting into an occupied port (_resolve_port) or a missing+ frontend (SystemExit) would otherwise leave _shutting_down False with the+ previous session's still-unwinding load free to start a child the shutdown+ sweep has already run past.+ """+ body = _run_server_body()+ # Statements, not text: the comments around these calls name them too, and+ # matching the prose made this compare the wrong offsets.+ reset = body.index("_llama_cpp_backend._begin_server_lifecycle()")+ serve = body.index("\n thread.start()")+ assert reset < serve, "the lifecycle reset no longer precedes the serve"++ # Only the window between the reset and the serve. The sys.exit further down is+ # a startup FAILURE after the thread is running, by which point a spawn is+ # legitimate, so the whole tail is the wrong scope.+ #+ # sys.exit( as well as raise SystemExit: the first version of this test checked+ # only the latter and missed the admin-password gate, which exits the other way.+ window = body[reset:serve]+ for fail_fast in ("raise SystemExit", "sys.exit(", "_resolve_port("):+ assert fail_fast not in window, (+ f"{fail_fast} runs after the lifecycle reset, so a failed restart leaves the "+ "spawn guard cleared"+ )+++class TestARefusedSpawnClosesItsLog:+ """Each spawn opens a per-attempt tee log just before Popen. A refusal returns+ without a process, and _kill_process returns early when there is none, so+ nothing else ever closes it: the next attempt overwrites the attribute, leaking+ the descriptor and holding the file lock on Windows."""++ def _backend(self, tmp_path):+ b = _make_backend()+ b._shutting_down = True+ b._spawn_lock = threading.Lock()+ b._stop_mtp_crash_watchdog = lambda: None+ b._process = None+ b._redacted_cmd_for_log = lambda c: c+ b._llama_log_path = str(tmp_path / "llama.log")+ b._llama_log_fh = None+ return b++ def test_start_llama_process_closes_it(self, tmp_path):+ """The handle under test is the one the method opens for itself, not one+ set beforehand: _start_llama_process clears the attribute and opens a fresh+ per-attempt log before it reaches the shutdown check."""+ b = self._backend(tmp_path)+ opened = []+ real_open = open++ def tracking_open(*a, **k):+ fh = real_open(*a, **k)+ opened.append(fh)+ return fh++ import builtins++ with mock.patch.object(builtins, "open", tracking_open):+ assert (+ b._start_llama_process(["llama-server"], {}, child_gpu_physical_ids = None) is False+ )++ assert opened, "the method never opened an attempt log; this proves nothing"+ assert all(fh.closed for fh in opened), "the refused attempt left its tee log open"+ assert b._llama_log_fh is None++ def test_the_kill_path_does_not_cover_it(self, tmp_path):+ """Why the explicit close is needed: _kill_process is the only other closer+ and it returns before that, since a refusal published no process."""+ b = self._backend(tmp_path)+ fh = open(tmp_path / "kill.log", "w", encoding = "utf-8")+ b._llama_log_fh = fh+ b._reset_effective_parallel_slots = lambda: None++ b._kill_process(teardown = True)++ assert not fh.closed, (+ "_kill_process now closes the handle with no process; if that is "+ "deliberate this test and the explicit close are both redundant"+ )+ fh.close()+++def test_the_pid_is_recorded_before_the_spawn_lock_is_released():+ """Teardown takes the lock the instant publication ends. Recording the pid after+ that would write it back behind a sweep that just forgot it, and+ _pid_start_identity cannot read a start time for a reaped process -- so the+ record is a bare pid, which a later launch kills without an identity check."""+ b = _make_backend()+ b._shutting_down = False+ b._stop_mtp_crash_watchdog = lambda: None+ b._process = None+ b._redacted_cmd_for_log = lambda c: c+ b._llama_log_path = None+ b._llama_log_fh = None+ order = []++ class _Lock:+ def __enter__(self):+ order.append("lock")+ return None++ def __exit__(self, *_exc):+ order.append("unlock")+ return False++ b._spawn_lock = _Lock()+ b._record_server_pid = lambda pid: order.append("record")++ class _Proc:+ pid = 4242++ def poll(self):+ return None++ with mock.patch.object(subprocess, "Popen", lambda *a, **k: _Proc()):+ with mock.patch.object(threading, "Thread", lambda **k: mock.Mock()):+ assert b._start_llama_process(["llama-server"], {}, child_gpu_physical_ids = None) is True++ assert order.index("record") < order.index("unlock"), (+ f"the pid is recorded after the lock is released ({order}), so a teardown "+ "between the two writes a bare pid record for an already-reaped process"+ )+++class TestHealthPublicationIsAtomicWithTeardown:+ """A successful probe and the _healthy commit are separate steps, so a teardown+ landing between them left the backend advertising a model whose child had+ already been killed. The recheck inside the wait narrows that window; only+ taking the same lock the teardown mark is set under closes it."""++ def _backend(self):+ b = _make_backend()+ b._stop_mtp_crash_watchdog = lambda: None+ b._healthy = False+ b._spawn_lock = threading.Lock()+ return b++ def test_a_teardown_before_the_commit_refuses_publication(self):+ b = self._backend()+ b._shutting_down = True++ assert b._publish_healthy() is False+ assert b._healthy is False, "a torn-down backend was published as healthy"++ def test_an_ordinary_load_still_publishes(self):+ b = self._backend()+ b._shutting_down = False++ assert b._publish_healthy() is True+ assert b._healthy is True++ def test_the_teardown_mark_and_the_commit_cannot_interleave(self):+ """The two orders the lock permits are the only safe ones: publish then+ teardown (which clears _healthy on its way out), or teardown then a refused+ publish. This drives the first order and asserts the second is impossible."""+ b = self._backend()+ b._shutting_down = False+ # A real child, not None: _kill_process returns before clearing _healthy when+ # there is nothing to kill, and a backend that published healthy by+ # definition has a process, so None would be testing the wrong ordering.+ b._process = object()+ b._reset_effective_parallel_slots = lambda: None+ b._leading_process_group = lambda _pid: None+ b._collect_descendants = lambda _pid: []+ b._diffusion_requested_ngl = None++ published = b._publish_healthy()+ assert published is True and b._healthy is True++ b._kill_process(teardown = True)++ assert (+ b._healthy is False+ ), "teardown left _healthy set, so a publication that won the race is never undone"+ assert b._publish_healthy() is False, "a later publication slipped past the teardown"+++def test_the_deadline_asks_the_durable_flag_too(monkeypatch):+ """_kill_process publishes its two signals apart: _shutting_down under the+ spawn lock on entry, _torn_down_process only after collecting descendants. A+ wait whose deadline lands between them saw no marker and recorded a plain+ timeout, sending a deliberate teardown through startup-failure handling."""+ b = _make_backend()+ b._process.poll.return_value = None+ b._shutting_down = False+ b._torn_down_process = None++ # Set during the LAST probe, not before the wait: setting it up front is caught+ # by the top-of-iteration check and never reaches the deadline at all, so that+ # version of this test passed with or without the fix.+ def probe(*a, **kw):+ b._shutting_down = True # teardown, after the final iteration's check+ return mock.Mock(status_code = 503)++ monkeypatch.setattr(httpx, "get", probe)++ assert b._wait_for_health(timeout = 0.01, interval = 0.05) is False+ assert b._health_wait_cancelled is True, "a teardown was recorded as a timeout"+ assert not any("health check timed out" in ln for ln in b._stdout_lines)+++class TestAStaleLoadIsDroppedAtTheSerialScope:+ """Refusing a previous lifecycle's load only at the spawn is too late.++ A lock gives a waiter no priority, so the stale load can take the serial scope+ after the restart, and the duplicate-adoption phase inside it calls+ _kill_process() on whatever is loaded -- by then the NEW lifecycle's model. It+ would unload the restarted server and then decline to replace it.+ """++ def test_the_check_precedes_the_replacement_work(self):+ """Checked over the AST, not the text: a first version matched the words+ '_kill_process' in the explanatory comment above the guard and failed on+ correct code."""+ import ast+ import inspect+ import textwrap++ tree = ast.parse(textwrap.dedent(inspect.getsource(LlamaCppBackend.load_model)))+ scope = next(+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.With)+ and any("_serial_load_scope" in ast.unparse(i.context_expr) for i in n.items)+ )++ guard_line = next(+ n.lineno+ for n in ast.walk(scope)+ if isinstance(n, ast.Call) and "_spawn_is_stale" in ast.unparse(n.func)+ )++ early_kills = [+ n.lineno+ for n in ast.walk(scope)+ if isinstance(n, ast.Call)+ and "_kill_process" in ast.unparse(n.func)+ and n.lineno < guard_line+ ]+ assert not early_kills, (+ f"a stale load calls _kill_process at {early_kills}, before it is refused at "+ f"line {guard_line}, so it unloads the new lifecycle's model"+ )++ # And the refusal has to stop the load rather than just log it.+ src = inspect.getsource(LlamaCppBackend.load_model)+ stale = src.index("_spawn_is_stale()")+ assert "return False" in src[stale : stale + 300], "the stale-load branch does not return"+++def test_a_lifecycle_cannot_reopen_while_a_teardown_is_still_killing():+ """_kill_process(teardown) used to release the lock as soon as it set the flag,+ but it goes on reading self._process and finally clears it. An embedded host+ that saw the server thread stop could reopen the lifecycle in that gap, let a+ new load spawn, and have this teardown drop or terminate its child."""+ b = _make_backend()+ b._stop_mtp_crash_watchdog = lambda: None+ b._reset_effective_parallel_slots = lambda: None+ b._diffusion_requested_ngl = None+ b._leading_process_group = lambda _pid: None+ b._collect_descendants = lambda _pid: []+ b._spawn_lock = threading.RLock() # so the probe below can observe, not deadlock++ reopened_during_kill = []+ in_kill = threading.Event()+ release = threading.Event()++ class _SlowProcess:+ pid = 999++ def terminate(self):+ in_kill.set()+ release.wait(2.0)++ def wait(self, timeout = None):+ return 0++ def kill(self):+ pass++ b._process = _SlowProcess()++ def restarter():+ in_kill.wait(2.0)+ # _teardown_lock, not _spawn_lock: the kill holds the former for its whole+ # duration and the latter only long enough to set the flag, so a spawn can+ # be refused promptly. This probes the lock that carries the property.+ got = b._teardown_lock.acquire(blocking = False)+ if got:+ b._teardown_lock.release()+ reopened_during_kill.append(got)+ release.set()++ t = threading.Thread(target = restarter)+ t.start()+ try:+ b._kill_process(teardown = True)+ finally:+ release.set()+ t.join(5.0)++ assert reopened_during_kill == [False], (+ "the teardown lock was free while the kill was still running, so "+ "_begin_server_lifecycle could reopen the lifecycle mid-kill"+ )+++def test_the_shutdown_cancels_loads_before_it_kills_the_server():+ """Order matters: killing first leaves the load to respawn into the teardown."""+ import ast+ import textwrap+ from pathlib import Path++ run_py = (Path(__file__).resolve().parent.parent / "run.py").read_text(encoding = "utf-8")+ tree = ast.parse(run_py)+ fn = next(+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_graceful_shutdown"+ )+ src = textwrap.dedent(ast.get_source_segment(run_py, fn) or "")++ assert "cancel_pending_loads()" in src, "shutdown does not cancel in-flight loads"+ assert src.index("cancel_pending_loads()") < src.index(+ "_kill_process(teardown = True)"+ ), "the loads are cancelled after the kill, so one can still spawn into the teardown"+++class TestATeardownDoesNotBlockASpawnItWillRefuse:+ """The kill needs a long hold so a lifecycle cannot reopen mid-terminate, but a+ spawn only needs to read the flag. Holding one lock for both made a spawn queue+ behind a SIGTERM/SIGKILL escalation for seconds before being told no, on a+ thread shutdown is already waiting for."""++ def _backend(self, on_terminate):+ b = _make_backend()+ b._stop_mtp_crash_watchdog = lambda: None+ b._reset_effective_parallel_slots = lambda: None+ b._diffusion_requested_ngl = None+ b._leading_process_group = lambda _p: None+ b._collect_descendants = lambda _p: []++ class _Stubborn:+ pid = 4242++ def __init__(self):+ self.killed = False++ def terminate(self):+ on_terminate()++ def wait(self, timeout = None):+ if not self.killed:+ raise subprocess.TimeoutExpired("llama-server", timeout)++ def kill(self):+ self.killed = True++ b._process = _Stubborn()+ return b++ def test_a_spawn_is_refused_without_waiting_for_the_kill(self):+ in_kill = threading.Event()+ release = threading.Event()+ b = self._backend(lambda: (in_kill.set(), release.wait(5.0)))+ seen = []++ def spawner():+ in_kill.wait(5.0)+ start = time.monotonic()+ with b._spawn_lock:+ stale = b._spawn_is_stale()+ seen.append((time.monotonic() - start, stale))+ release.set()++ t = threading.Thread(target = spawner)+ t.start()+ try:+ b._kill_process(teardown = True)+ finally:+ release.set()+ t.join(10)++ waited, refused = seen[0]+ assert refused is True, "the spawn was allowed through during a teardown"+ assert waited < 1.0, (+ f"the spawn waited {waited:.2f}s for the kill before being refused; the "+ "teardown hold and the flag read must not share one lock"+ )++ def test_a_lifecycle_reset_still_waits_for_the_kill(self):+ """The other half, and the reason the long hold exists at all: clearing the+ flag mid-kill would let a new load spawn a child this teardown then drops."""+ in_kill = threading.Event()+ release = threading.Event()+ b = self._backend(lambda: (in_kill.set(), release.wait(2.0)))+ order = []++ def resetter():+ in_kill.wait(5.0)+ b._begin_server_lifecycle()+ order.append("reset")++ t = threading.Thread(target = resetter)+ t.start()+ try:+ b._kill_process(teardown = True)+ order.append("kill_done")+ finally:+ release.set()+ t.join(10)++ assert order == [+ "kill_done",+ "reset",+ ], f"the lifecycle reopened before the kill finished: {order}"+++def test_the_lock_order_is_teardown_then_spawn():+ """Two locks means an order, and taking them the other way round deadlocks."""+ import ast+ import inspect++ tree = ast.parse(inspect.getsource(LlamaCppBackend))+ inversions = []+ for node in ast.walk(tree):+ if isinstance(node, ast.With) and any(+ "_spawn_lock" in ast.unparse(i.context_expr) for i in node.items+ ):+ for inner in ast.walk(node):+ if inner is node:+ continue+ if isinstance(inner, ast.With) and any(+ "_teardown_lock" in ast.unparse(i.context_expr) for i in inner.items+ ):+ inversions.append(inner.lineno)+ assert not inversions, f"_teardown_lock taken inside _spawn_lock at {inversions}"diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 64eff7ea9a8..0e50e18a0f6 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -8542,7 +8542,12 @@ def test_unload_clears_the_user_load_flag():assert backend._loaded_by_user_action is False# Not cleared on a bare process kill: a respawn or MTP-free reload replays# the same load and must keep the provenance it had.- assert "_loaded_by_user_action" not in inspect.getsource(LlamaCppBackend._kill_process)+ # Both halves, or this negative assertion passes vacuously: the body it is+ # really about now lives in _kill_process_body.+ assert "_loaded_by_user_action" not in (+ inspect.getsource(LlamaCppBackend._kill_process)+ + inspect.getsource(LlamaCppBackend._kill_process_body)+ )def test_the_load_route_pins_and_other_load_surfaces_do_not(monkeypatch):diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.pyindex d3da92e74a6..40aaf8569e2 100644--- a/studio/backend/tests/test_orchestrator_unload_cancel.py+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py@@ -2634,3 +2634,851 @@ def test_a_scoped_load_cancel_that_never_reports_back_releases_the_load():with inf._scoped_load_attempts_lock:inf._scoped_load_attempts.clear()inf._scoped_load_cancel_tombstones.clear()+++def test_shutdown_cancels_loads_that_have_not_reached_the_backend():+ """A /load between admission and the backend call holds nothing the backend's+ shutdown flag can see: it can sit in the lifecycle gate or preflight for+ minutes and then arrive in a lifecycle that has already been reset, loading a+ model the new server never asked for. _graceful_shutdown cancels through the+ attempt's own event, the same path /unload uses.+ """+ import importlib++ inf = importlib.import_module("routes.inference")++ def _attempt(token, path):+ return inf._ScopedLoadAttempt(+ token = token,+ request_id = None,+ model_path = path,+ subject = "s",+ cancel_event = threading.Event(),+ cancel_complete = threading.Event(),+ )++ pending = _attempt("pending-token", "owner/model")+ running = _attempt("running-token", "owner/other")++ with inf._scoped_load_attempts_lock:+ inf._pending_load_attempts[pending.token] = pending+ prior_running = inf._running_load_attempt+ inf._running_load_attempt = running+ try:+ assert inf.cancel_pending_loads() == 2+ assert pending.cancel_event.is_set(), "a queued load survived the shutdown"+ assert running.cancel_event.is_set(), "the running load survived the shutdown"+ finally:+ inf._running_load_attempt = prior_running+ with inf._scoped_load_attempts_lock:+ inf._pending_load_attempts.pop(pending.token, None)+++def test_a_running_attempt_already_in_the_pending_map_is_not_counted_twice():+ import importlib++ inf = importlib.import_module("routes.inference")++ both = inf._ScopedLoadAttempt(+ token = "same-token",+ request_id = None,+ model_path = "owner/model",+ subject = "s",+ cancel_event = threading.Event(),+ cancel_complete = threading.Event(),+ )+ with inf._scoped_load_attempts_lock:+ inf._pending_load_attempts[both.token] = both+ prior_running = inf._running_load_attempt+ inf._running_load_attempt = both+ try:+ assert inf.cancel_pending_loads() == 1+ assert both.cancel_event.is_set()+ finally:+ inf._running_load_attempt = prior_running+ with inf._scoped_load_attempts_lock:+ inf._pending_load_attempts.pop(both.token, None)+++def _mk_attempt(+ inf,+ token,+ path = "owner/model",+):+ return inf._ScopedLoadAttempt(+ token = token,+ request_id = None,+ model_path = path,+ subject = "s",+ cancel_event = threading.Event(),+ cancel_complete = threading.Event(),+ )+++def test_shutdown_closes_the_cancel_handshake_itself():+ """Only /unload sets cancel_complete, and at shutdown there is none, so setting+ cancel_event alone leaves _run_tracked_load_model_impl's finally waiting the full+ handshake timeout in a to_thread. Those executor threads are non-daemon and hold+ the process open, which is what the comment above the timeout warns about."""+ import importlib++ inf = importlib.import_module("routes.inference")+ attempt = _mk_attempt(inf, "handshake-token")+ with inf._scoped_load_attempts_lock:+ inf._pending_load_attempts[attempt.token] = attempt+ try:+ inf.cancel_pending_loads()+ assert attempt.cancel_event.is_set()+ assert attempt.cancel_complete.is_set(), (+ "shutdown left the handshake open, so the load waits the full timeout in a "+ "non-daemon executor thread and delays process exit"+ )+ finally:+ inf.begin_load_lifecycle()+ with inf._scoped_load_attempts_lock:+ inf._pending_load_attempts.pop(attempt.token, None)+++def test_a_load_registering_after_the_sweep_is_still_cancelled():+ """uvicorn's should_exit stops new connections, not request tasks it already+ admitted, so a /load can register after the shutdown snapshot was taken and+ would otherwise never be cancelled by anything."""+ import importlib++ inf = importlib.import_module("routes.inference")+ try:+ assert inf.cancel_pending_loads() == 0 # latches with nothing to cancel++ late = _mk_attempt(inf, "late-token")+ with inf._scoped_load_attempts_lock:+ inf._pending_load_attempts[late.token] = late+ latched = inf._loads_shutting_down+ assert latched is True, "the shutdown latch did not survive an empty sweep"+ if latched:+ inf._cancel_for_shutdown(late)+ assert late.cancel_event.is_set(), "a load admitted during shutdown was not cancelled"+ finally:+ inf.begin_load_lifecycle()+ with inf._scoped_load_attempts_lock:+ inf._pending_load_attempts.pop("late-token", None)+++def test_the_latch_is_scoped_to_a_lifecycle_not_the_process():+ """An embedded host calls run_server again. A permanent latch would refuse every+ /load of the second session; the backend flag had exactly this bug."""+ import importlib++ inf = importlib.import_module("routes.inference")+ try:+ inf.cancel_pending_loads()+ with inf._scoped_load_attempts_lock:+ assert inf._loads_shutting_down is True++ inf.begin_load_lifecycle()+ with inf._scoped_load_attempts_lock:+ assert (+ inf._loads_shutting_down is False+ ), "the second session would cancel every load it admitted"+ finally:+ inf.begin_load_lifecycle()+++def test_run_server_clears_the_route_latch_too():+ import ast+ import textwrap+ from pathlib import Path++ run_py = (Path(__file__).resolve().parent.parent / "run.py").read_text(encoding = "utf-8")+ tree = ast.parse(run_py)+ fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "run_server")+ src = textwrap.dedent(ast.get_source_segment(run_py, fn) or "")+ assert "begin_load_lifecycle()" in src, (+ "run_server resets the backend but not the route latch, so a restarted "+ "server cancels every load it admits"+ )+++def _run_server_call_lines(name, *, owner = None):+ """First line of each call to *name* inside run_server, by AST rather than text.++ Text offsets kept breaking here: the comments above these calls name them too, and+ indentation-anchored searches go stale the moment a call moves inside a `with`.+ """+ import ast+ from pathlib import Path++ run_py = (Path(__file__).resolve().parent.parent / "run.py").read_text(encoding = "utf-8")+ fn = next(+ n+ for n in ast.parse(run_py).body+ if isinstance(n, ast.FunctionDef) and n.name == "run_server"+ )+ lines = []+ for node in ast.walk(fn):+ if not isinstance(node, ast.Call):+ continue+ func = node.func+ if isinstance(func, ast.Attribute) and func.attr == name:+ if owner is not None and getattr(func.value, "id", None) != owner:+ continue+ lines.append(node.lineno)+ elif isinstance(func, ast.Name) and func.id == name and owner is None:+ lines.append(node.lineno)+ assert lines, f"run_server no longer calls {name}"+ return min(lines)+++def test_the_route_latch_clears_only_after_the_backend_lifecycle_reopens():+ """_begin_server_lifecycle blocks on the teardown lock while a kill is running.+ Clearing the route latch before that wait leaves a request the OLD lifecycle+ admitted uncancelled, and it then captures the freshly advanced generation and+ loads the previous session's model into the new one.++ Nothing legitimate is refused by clearing later: uvicorn does not serve until+ thread.start(), which is below both calls.+ """+ backend_reset = _run_server_call_lines("_begin_server_lifecycle")+ route_reset = _run_server_call_lines("begin_load_lifecycle")+ serve = _run_server_call_lines("start", owner = "thread")++ assert backend_reset < route_reset, (+ "the route latch is cleared before the backend lifecycle reopens, so a "+ "request admitted by the old lifecycle can cross the teardown wait"+ )+ assert route_reset < serve, "the route latch is still set when the server starts serving"+++def _load_impl_ast():+ """(module source, the _load_model_impl node) from routes/inference.py."""+ import ast+ from pathlib import Path++ src = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text(+ encoding = "utf-8"+ )+ fn = next(+ n+ for n in ast.parse(src).body+ if isinstance(n, ast.AsyncFunctionDef) and n.name == "_load_model_impl"+ )+ return src, fn+++def test_the_shutdown_latch_is_enforced_in_the_load_impl_not_only_at_the_route():+ """Auto-switch and preview await _load_model_impl directly, without ever+ registering a _ScopedLoadAttempt, so the shutdown sweep has no event to set for+ them. With the latch checked only where /load registers, one of those loads can+ survive the sweep, reach a non-GGUF target after run.py already tore the+ inference subprocess down, and spawn a worker that outlives quit.+ """+ import ast++ src, impl = _load_impl_ast()++ tracked = {"_run_tracked_load_model_impl", "_load_model_impl"}+ direct = [+ node.name+ for node in ast.walk(ast.parse(src))+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))+ and node.name not in tracked+ and any(+ isinstance(c, ast.Call)+ and isinstance(c.func, ast.Name)+ and c.func.id == "_load_model_impl"+ for c in ast.walk(node)+ )+ ]+ assert direct, (+ "no direct _load_model_impl callers left; if registration now covers every "+ "path this guard can move back to the route"+ )++ reads = [+ n.id for n in ast.walk(impl) if isinstance(n, ast.Name) and n.id == "_loads_shutting_down"+ ]+ assert reads, (+ "_load_model_impl never consults the shutdown latch, so these direct "+ f"callers bypass it entirely: {sorted(set(direct))}"+ )+++def test_the_impl_cancel_check_refuses_a_load_once_shutdown_has_latched():+ """Runs the shipped closure, rather than asserting on its text.++ The callers of this helper are the load's points of no return, so refusing here+ is what stops a shutdown-crossing load before it spawns anything.+ """+ import ast+ import textwrap+ import threading++ from fastapi import HTTPException++ src, impl = _load_impl_ast()+ helper = next(+ n+ for n in impl.body+ if isinstance(n, ast.FunctionDef) and n.name == "_raise_if_scoped_load_cancelled"+ )+ ns = {+ "HTTPException": HTTPException,+ "_scoped_load_attempts_lock": threading.Lock(),+ "_loads_shutting_down": False,+ "load_cancel_event": None,+ }+ exec(textwrap.dedent(ast.get_source_segment(src, helper) or ""), ns)+ check = ns["_raise_if_scoped_load_cancelled"]++ check() # nothing set: a normal load must not be refused++ ns["_loads_shutting_down"] = True+ with pytest.raises(HTTPException) as excinfo:+ check()+ assert excinfo.value.status_code == 409++ ns["_loads_shutting_down"] = False+ ns["load_cancel_event"] = threading.Event()+ check() # an unset per-attempt event is still not a cancel+ ns["load_cancel_event"].set()+ with pytest.raises(HTTPException):+ check()+++def test_a_second_backend_instance_is_covered_by_the_shutdown_latch():+ """A helper/advisor load builds its OWN LlamaCppBackend (hub/utils/llm_assist.py,+ utils/datasets/llm_assist.py). run.py only tears down the routes singleton, and+ _shutting_down is per-instance, so that second backend would still consider a+ spawn valid and Popen a server after terminate_all took its snapshot.+ """+ from core.inference.llama_cpp import LlamaCppBackend+ from utils import process_lifetime++ torn_down = LlamaCppBackend.__new__(LlamaCppBackend)+ helper = LlamaCppBackend.__new__(LlamaCppBackend)+ try:+ assert helper._spawn_is_stale() is False, "a fresh backend must be able to spawn"++ # What _kill_process(teardown = True) does to the singleton, without the kill.+ torn_down._shutting_down = True+ process_lifetime.mark_process_shutting_down()++ assert (+ helper._spawn_is_stale() is True+ ), "a backend the shutdown never touched still thinks it may spawn"+ finally:+ process_lifetime.begin_process_lifecycle()++ assert (+ helper._spawn_is_stale() is False+ ), "the latch outlived the lifecycle, so an embedded second session cannot spawn"+++def test_the_worker_spawn_refuses_once_shutdown_has_latched():+ """The preview path supplies no load_cancel_event and is not a _ScopedLoadAttempt,+ so its latch read is one-shot: it can pass that check, spend time in the drain and+ teardown, and only then reach the worker spawn. Guarding at the spawn is what makes+ the answer un-stale.+ """+ from core.inference.orchestrator import InferenceOrchestrator+ from utils import process_lifetime++ orch = InferenceOrchestrator.__new__(InferenceOrchestrator)+ try:+ process_lifetime.mark_process_shutting_down()+ with pytest.raises(RuntimeError, match = "shutting down"):+ orch._spawn_subprocess({})+ finally:+ process_lifetime.begin_process_lifecycle()+++def test_the_process_latch_clears_between_the_backend_and_the_route():+ """Ordering, for the same reason the route latch clears late: the process latch is+ what every other spawner reads, so it must outlast the teardown _begin_server_lifecycle+ waits on, and be clear before uvicorn admits anything.+ """+ backend = _run_server_call_lines("_begin_server_lifecycle")+ process = _run_server_call_lines("begin_process_lifecycle")+ route = _run_server_call_lines("begin_load_lifecycle")+ serve = _run_server_call_lines("start", owner = "thread")++ assert backend < process < route < serve, (+ "the process latch must clear after the backend teardown completes and "+ "before the server starts serving"+ )+++def test_shutdown_latches_the_process_before_any_subsystem_is_torn_down():+ """The orchestrator is stopped at step 2 but loads are swept at step 5. A load in+ that gap reaches a spawner nothing has marked yet, so the latch has to be set before+ the first teardown step rather than alongside the llama-server kill.+ """+ import ast+ import textwrap+ from pathlib import Path++ run_py = (Path(__file__).resolve().parent.parent / "run.py").read_text(encoding = "utf-8")+ tree = ast.parse(run_py)+ fn = next(+ n+ for n in ast.walk(tree)+ if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "_graceful_shutdown"+ )+ src = textwrap.dedent(ast.get_source_segment(run_py, fn) or "")++ latch = src.index("mark_process_shutting_down()")+ orchestrator_stop = src.index("_shutdown_subprocess(timeout = 5.0)")+ sweep = src.index("cancel_pending_loads()")++ assert latch < orchestrator_stop, (+ "the inference subprocess is stopped before anything latches, so a load in "+ "flight can restart it"+ )+ assert latch < sweep+++def test_a_shutdown_that_begins_during_the_spawn_reaps_the_new_worker():+ """The pre-spawn gate is a process start away from the child existing. A shutdown+ landing in between sees no live _proc, no-ops, completes terminate_all, and would+ leave this worker running after quit. The post-spawn recheck is what closes it.+ """+ from unittest import mock++ from core.inference.orchestrator import InferenceOrchestrator+ from utils import process_lifetime++ orch = InferenceOrchestrator.__new__(InferenceOrchestrator)+ torn_down = []+ orch._shutdown_subprocess = lambda timeout = None: torn_down.append(timeout)++ started = mock.Mock()+ started.pid = 4242++ class _Ctx:+ Queue = staticmethod(lambda: mock.Mock())+ Event = staticmethod(lambda: mock.Mock())++ @staticmethod+ def Process(**kw):+ # Shutdown begins while the child is being born, after the gate passed.+ process_lifetime.mark_process_shutting_down()+ return started++ try:+ with (+ mock.patch.object(orch_mod, "_CTX", _Ctx),+ mock.patch.object(orch_mod, "adopt_pid", lambda pid: None, create = True),+ ):+ with pytest.raises(RuntimeError, match = "shutting down"):+ orch._spawn_subprocess({})+ finally:+ process_lifetime.begin_process_lifecycle()++ assert torn_down, "the worker born during shutdown was never torn down"+++def test_a_llama_server_spawned_as_shutdown_began_is_reaped():+ """The pre-spawn stale check and the process latch are only atomic for the instance+ run.py tears down, which sets its own flag under the spawn lock. A helper backend's+ check can pass microseconds before the latch is set, and its child would then+ outlive the sweep. The recheck after the pid is recorded is what reaps it.+ """+ import ast+ import textwrap+ from pathlib import Path++ src = (+ Path(__file__).resolve().parent.parent / "core" / "inference" / "llama_cpp.py"+ ).read_text(encoding = "utf-8")+ fn = next(+ n+ for n in ast.walk(ast.parse(src))+ if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))+ and n.name == "_start_llama_process"+ )+ body = textwrap.dedent(ast.get_source_segment(src, fn) or "")++ publish = body.index("self._record_server_pid(")+ recheck = body.index("_spawn_is_stale", publish)+ kill = body.index("self._kill_process()", publish)+ assert publish < recheck < kill, (+ "nothing rechecks after the child is published, so a spawn that raced the "+ "latch leaves a server the sweep has already passed"+ )+++def test_a_load_that_finishes_during_shutdown_is_not_published_as_resident():+ """The spawn checks stop once the worker exists. A "loaded" reply dequeued as+ shutdown kills it would still reach the success branch, and active_model_name plus+ models are exactly what the already-loaded fast path trusts. That path does not test+ liveness, so the next session would report a dead worker as resident.+ """+ from core.inference.orchestrator import InferenceOrchestrator+ from utils import process_lifetime++ orch = InferenceOrchestrator.__new__(InferenceOrchestrator)+ orch.active_model_name = None+ orch.models = {}+ orch.loading_models = {"m"}+ orch.load_generation = 0++ try:+ process_lifetime.mark_process_shutting_down()+ stale = process_lifetime.is_process_shutting_down()+ assert stale is True, "the load no longer belongs to a live session"++ # What the success branch must do instead of publishing.+ if stale:+ orch.loading_models.discard("m")+ orch.active_model_name = None+ orch.models.clear()++ assert (+ orch.active_model_name is None and not orch.models+ ), "a worker killed by shutdown was published as resident"+ finally:+ process_lifetime.begin_process_lifecycle()+++def test_the_publish_branch_rechecks_shutdown_before_recording_the_model():+ """Pins the recheck in the shipped code, ahead of the first field it publishes."""+ import ast+ import textwrap+ from pathlib import Path++ src = (+ Path(__file__).resolve().parent.parent / "core" / "inference" / "orchestrator.py"+ ).read_text(encoding = "utf-8")+ fn = next(+ n+ for n in ast.walk(ast.parse(src))+ if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "load_model"+ )+ body = textwrap.dedent(ast.get_source_segment(src, fn) or "")++ check = body.index("if is_process_shutting_down(")+ publish = body.index("self.active_model_name = model_info.get(")+ assert check < publish, (+ "the load publishes active_model_name before rechecking shutdown, so a worker "+ "killed mid-load is recorded as resident"+ )+++def _fn_named(source, name):+ import ast+ return next(+ n+ for n in ast.walk(ast.parse(source))+ if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == name+ )+++def test_the_primary_llama_launch_rechecks_after_recording_the_pid():+ """mark_process_shutting_down does not take _spawn_lock, and terminate_all does not+ take it when it snapshots, so the in-lock check is not atomic against the+ process-wide latch for a helper-owned backend. Without a recheck the child sits+ outside the completed sweep for the whole 600s health wait.+ """+ import ast+ from pathlib import Path++ src = (+ Path(__file__).resolve().parent.parent / "core" / "inference" / "llama_cpp.py"+ ).read_text(encoding = "utf-8")+ fn = _fn_named(src, "_spawn_and_wait")+ record = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "_record_server_pid"+ ]+ stale = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "_spawn_is_stale"+ ]+ health = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "_wait_for_health"+ ]+ assert record and stale and health, "the spawn path no longer looks like itself"+ assert any(max(record) < s < min(health) for s in stale), (+ "no staleness recheck between recording the pid and the health wait: a spawn "+ "that raced the latch is left running outside the sweep that already finished"+ )+++def test_the_worker_mirrors_are_published_under_the_shutdown_lock():+ """active_model_name is what the already-loaded fast path trusts, and it does not+ test liveness. Checked and published apart, shutdown can kill the worker in between+ and the next session reports a dead one as resident.+ """+ import ast+ from pathlib import Path++ src = (+ Path(__file__).resolve().parent.parent / "core" / "inference" / "orchestrator.py"+ ).read_text(encoding = "utf-8")+ fn = _fn_named(src, "load_model")+ holding = [+ n+ for n in ast.walk(fn)+ if isinstance(n, ast.With)+ and any(+ getattr(item.context_expr, "attr", None) == "_subprocess_shutdown_lock"+ for item in n.items+ )+ ]+ assert holding, "load_model never holds the subprocess shutdown lock"+ covered = False+ for w in holding:+ checks = [+ n+ for n in ast.walk(w)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "is_process_shutting_down"+ ]+ publishes = [+ n+ for n in ast.walk(w)+ if isinstance(n, ast.Assign)+ and any(+ getattr(t, "attr", None) == "active_model_name"+ and not isinstance(n.value, ast.Constant)+ for t in n.targets+ )+ ]+ if checks and publishes:+ covered = True+ assert covered, (+ "the shutdown check and the active_model_name publication are not inside the "+ "same held lock, so a kill can land between them"+ )+++def test_the_worker_handle_is_captured_before_start_not_after():+ """`self._proc` is not a handle this code can rely on once start() is running: a+ concurrent _shutdown_subprocess can observe a not-yet-alive child and clear it.+ Snapshotting the attribute AFTER start() therefore captures None and loses the only+ reference to a live child, which is exactly the orphan this change prevents.+ """+ import ast+ from pathlib import Path++ src = (+ Path(__file__).resolve().parent.parent / "core" / "inference" / "orchestrator.py"+ ).read_text(encoding = "utf-8")+ fn = _fn_named(src, "_spawn_subprocess")++ # The local must be bound from the Process(...) construction, never from self._proc.+ binds = [+ n+ for n in ast.walk(fn)+ if isinstance(n, ast.Assign)+ and any(getattr(t, "id", None) == "_spawned_proc" for t in n.targets)+ ]+ assert binds, "_spawn_subprocess no longer keeps a local handle on the worker"+ for n in binds:+ assert not (isinstance(n.value, ast.Attribute) and n.value.attr == "_proc"), (+ "the local handle is snapshotted from self._proc, which a concurrent "+ "shutdown can have cleared by then; build it from Process(...) instead"+ )++ # And start() must be called through the local, not through the attribute.+ starts = [+ n+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "start"+ ]+ assert starts, "the worker is never started"+ for n in starts:+ owner = n.func.value+ assert getattr(owner, "id", None) == "_spawned_proc", (+ "start() is called on self._proc rather than the local handle, so a "+ "shutdown clearing the attribute mid-start loses the child"+ )+++def test_the_rag_embed_server_spawn_is_gated_and_rechecked():+ """No _graceful_shutdown step stops this backend, so the process-wide latch is the+ only thing standing between an in-flight encode and a server that outlives quit.+ """+ import ast+ from pathlib import Path++ src = (+ Path(__file__).resolve().parent.parent / "core" / "rag" / "embed_llama_server.py"+ ).read_text(encoding = "utf-8")+ fn = _fn_named(src, "_spawn_once")++ checks = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "is_process_shutting_down"+ ]+ popen = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "Popen"+ ]+ adopt = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "adopt_pid"+ ]+ assert popen and adopt, "the embed spawn no longer looks like itself"+ assert any(c < min(popen) for c in checks), (+ "the embed server spawns without consulting the shutdown latch, so a quit "+ "during an encode can start one after the sweep has run"+ )+ assert any(c > max(adopt) for c in checks), (+ "no recheck after the pid is recorded: a latch set during the spawn leaves "+ "this child outside a sweep that has already finished"+ )+++def test_the_stt_sidecar_spawns_are_gated_and_rechecked():+ """Neither STT sidecar is stopped by any _graceful_shutdown step and neither is+ covered by cancel_pending_loads, which only reaches the chat /load attempt maps.+ The process-wide latch is the only thing that can stop a quit during an STT load+ from leaving whisper-server or the MTMD llama-server behind.+ """+ import ast+ from pathlib import Path++ root = Path(__file__).resolve().parent.parent / "core" / "inference"+ for filename, func in (+ ("stt_ggml_sidecar.py", "load"),+ ("stt_mtmd_sidecar.py", "_load_locked"),+ ):+ src = (root / filename).read_text(encoding = "utf-8")+ fn = _fn_named(src, func)+ checks = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "is_process_shutting_down"+ ]+ popen = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "Popen"+ ]+ adopt = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "adopt_pid"+ ]+ assert popen and adopt, f"{filename}:{func} no longer looks like a spawn"+ assert any(c < min(popen) for c in checks), (+ f"{filename} spawns without consulting the shutdown latch, so quitting "+ "during an STT load can start a server after the sweep has run"+ )+ assert any(c > max(adopt) for c in checks), (+ f"{filename} has no recheck after recording the pid, so a latch set during "+ "the spawn leaves the child outside a sweep that already finished"+ )+++def test_the_latch_is_set_before_the_atexit_sweep():+ """atexit is LIFO, so the handler registered LAST runs FIRST.++ The only thing that latched shutdown on the atexit path was the backend's own+ _cleanup hook, registered when the routes singleton was built and therefore run+ AFTER run_server's terminate_all. The sweep took its snapshot with the latch clear,+ which is exactly the unguarded behaviour this change removes on the graceful path.+ """+ import ast+ from pathlib import Path++ run_py = (Path(__file__).resolve().parent.parent / "run.py").read_text(encoding = "utf-8")+ tree = ast.parse(run_py)+ registrations = [+ (n.lineno, n.args[0].id)+ for n in ast.walk(tree)+ if isinstance(n, ast.Call)+ and getattr(n.func, "attr", None) == "register"+ and getattr(n.func.value, "id", None) == "atexit"+ and n.args+ and isinstance(n.args[0], ast.Name)+ ]+ names = [name for _, name in registrations]+ assert "terminate_all" in names, "run.py no longer registers the backstop sweep"+ assert "mark_process_shutting_down" in names, (+ "nothing latches shutdown on the atexit path, so the sweep snapshots while "+ "spawners still believe they may start children"+ )+ sweep_line = max(l for l, name in registrations if name == "terminate_all")+ latch_line = max(l for l, name in registrations if name == "mark_process_shutting_down")+ assert latch_line > sweep_line, (+ "the latch is registered before the sweep, so under LIFO it runs after it and "+ "the snapshot is taken unguarded"+ )+++def test_every_long_lived_spawner_consults_the_shutdown_latch():+ """The premise of this change is one flag read at every spawn. A spawner that+ adopts a long-lived child without consulting it is a hole in exactly the guarantee+ the PR claims, so the set is pinned here rather than rediscovered one report at a+ time.+ """+ import ast+ from pathlib import Path++ backend = Path(__file__).resolve().parent.parent+ guarded = {+ "core/export/orchestrator.py",+ "core/inference/llama_cpp.py",+ "core/inference/orchestrator.py",+ "core/inference/sd_cpp_engine.py",+ "core/inference/sd_cpp_server.py",+ "core/inference/stt_ggml_sidecar.py",+ "core/inference/stt_mtmd_sidecar.py",+ "core/inference/stt_transformers_worker.py",+ "core/rag/embed_llama_server.py",+ "core/training/training.py",+ }+ # Adopters this change deliberately leaves ungated, listed so the completeness check+ # below cannot pass by omission. They are a documented residual, not an oversight:+ # gating them is the same six lines each, but each needs its own failure idiom and+ # its own reaping, and this PR is scoped to the paths a quit during a model load+ # actually reaches. A new adopter lands in neither set and fails the check, which is+ # the point: the decision gets made once, here, instead of one report at a time.+ not_gated_here = {+ "cloudflare_tunnel.py",+ "core/data_recipe/jobs/manager.py",+ "core/inference/diffusion_transformer_quant.py",+ "core/inference/stt_download_worker.py",+ "core/inference/tools.py",+ "core/training/diffusion_training_service.py",+ "utils/prebuilt/update_flow.py",+ "utils/process_lifetime.py",+ "utils/torch_device_probe.py",+ }+ adopters = {+ str(path.relative_to(backend)).replace("\\", "/")+ for path in backend.rglob("*.py")+ if "adopt_pid(" in path.read_text(encoding = "utf-8")+ and not str(path.relative_to(backend)).replace("\\", "/").startswith("tests/")+ and "vendor/" not in str(path.relative_to(backend)).replace("\\", "/")+ }+ assert adopters == guarded | not_gated_here, (+ "a module adopts a child but is in neither set; decide whether it needs the "+ f"shutdown gate and put it in one of them: {adopters ^ (guarded | not_gated_here)}"+ )+ for rel in sorted(guarded):+ src = (backend / rel).read_text(encoding = "utf-8")+ tree = ast.parse(src)+ adopts = [+ n.lineno+ for n in ast.walk(tree)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "adopt_pid"+ ]+ checks = [+ n.lineno+ for n in ast.walk(tree)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "is_process_shutting_down"+ ]+ assert adopts, f"{rel} no longer adopts a child"+ assert checks, f"{rel} adopts a child without ever consulting the shutdown latch"+ assert any(c > min(adopts) for c in checks), (+ f"{rel} never rechecks the latch after adopting, so a spawn that raced it "+ "is left outside a sweep that has already finished"+ )diff --git a/studio/backend/tests/test_orphaned_children.py b/studio/backend/tests/test_orphaned_children.pyindex ec66d735bc0..6fb0bace602 100644--- a/studio/backend/tests/test_orphaned_children.py+++ b/studio/backend/tests/test_orphaned_children.py@@ -1450,7 +1450,11 @@ def test_the_visual_server_goes_down_with_an_ordinary_stop():from core.inference.llama_cpp import LlamaCppBackend- source = inspect.getsource(LlamaCppBackend._kill_process)+ # The whole kill path: _kill_process is a thin wrapper that takes the spawn+ # lock for a teardown and delegates the termination to _kill_process_body.+ source = inspect.getsource(LlamaCppBackend._kill_process) + inspect.getsource(+ LlamaCppBackend._kill_process_body+ )assert "_collect_descendants" in sourceassert "_terminate_descendants" in source# Named before the wait: the shim's children are reparented once it exits.diff --git a/studio/backend/tests/test_parallel_slots_per_load.py b/studio/backend/tests/test_parallel_slots_per_load.pyindex 0436e6db5c3..bd56d419759 100644--- a/studio/backend/tests/test_parallel_slots_per_load.py+++ b/studio/backend/tests/test_parallel_slots_per_load.py@@ -227,7 +227,12 @@ def test_load_model_commits_requested_from_intent():# come from the immutable pre-reduction intent.src = inspect.getsource(LlamaCppBackend.load_model)commit = src.find("self._requested_n_parallel = max(1, int(intent.n_parallel))")- healthy = src.find("self._healthy = True\n", 0, commit if commit != -1 else None)+ # The commit goes through _publish_healthy(), which sets _healthy under the+ # spawn lock so a teardown cannot land between the successful probe and the+ # commit. The invariant this test guards is unchanged: health is published+ # first, so a failed start cannot poison the next inheritance check.+ # Matched without the argument list, so adding one does not break this again.+ healthy = src.find("self._publish_healthy(", 0, commit if commit != -1 else None)snapshot = src.find("self._last_load_intent = replace(intent")assert commit != -1, "load_model must commit the requested slot count"assert healthy != -1 and healthy < commit < snapshotdiff --git a/studio/backend/tests/test_tensor_parallel.py b/studio/backend/tests/test_tensor_parallel.pyindex ae765bfce6e..8c9ec975780 100644--- a/studio/backend/tests/test_tensor_parallel.py+++ b/studio/backend/tests/test_tensor_parallel.py@@ -1136,7 +1136,11 @@ def test_kill_process_stops_crash_watchdog(monkeypatch):def test_kill_process_stops_watchdog_before_terminate():# Ordering matters: stop the watchdog before terminating so the watchdog's# post-death stop re-check reliably sees a planned kill.- src = inspect.getsource(LlamaCppBackend._kill_process)+ # Both halves: the termination moved into _kill_process_body, and the ordering+ # this asserts is within that body.+ src = inspect.getsource(LlamaCppBackend._kill_process) + inspect.getsource(+ LlamaCppBackend._kill_process_body+ )stop = src.find("_stop_mtp_crash_watchdog()")term = src.find(".terminate(")assert 0 <= stop < term, "must stop the watchdog before terminating"@@ -1280,8 +1284,7 @@ def test_tensor_plan_leaves_the_floor_reserve_at_a_full_budget():def test_split_mode_tensor_arch_failure_message():msg = LlamaCppBackend._classify_llama_start_failure(- "llama_model_create: LLAMA_SPLIT_MODE_TENSOR not implemented for "- "architecture 'deepseek2'",+ "llama_model_create: LLAMA_SPLIT_MODE_TENSOR not implemented for architecture 'deepseek2'",None,"unsloth/DeepSeek-V3-GGUF",)diff --git a/studio/backend/tests/test_training_pump_resilience.py b/studio/backend/tests/test_training_pump_resilience.pyindex 5d9d406729e..c1a8a5faca3 100644--- a/studio/backend/tests/test_training_pump_resilience.py+++ b/studio/backend/tests/test_training_pump_resilience.py@@ -506,6 +506,10 @@ def _stub_spawn(monkeypatch):pl = _types.ModuleType("utils.process_lifetime")pl.adopt_pid = lambda pid: None+ # The spawn also reads the shutdown latch. These tests are about the pump, not about+ # quitting, so the double answers "not shutting down" and the spawn proceeds; leaving+ # it off makes the import fail and every start_training here return False.+ pl.is_process_shutting_down = lambda: Falsemonkeypatch.setitem(sys.modules, "utils.process_lifetime", pl)worker = _types.ModuleType("core.training.worker")diff --git a/studio/backend/utils/process_lifetime.py b/studio/backend/utils/process_lifetime.pyindex f0fcb58af74..c71eba75e40 100644--- a/studio/backend/utils/process_lifetime.py+++ b/studio/backend/utils/process_lifetime.py@@ -357,7 +357,7 @@ _fork_reset_installed = Falsedef _reset_after_fork() -> None:"""A fork child inherits both locks in whatever state they were in and a_spawner whose thread does not exist here. Start clean instead of deadlocking."""- global _spawner, _spawner_lock, _record_lock, _owner_identity+ global _spawner, _spawner_lock, _record_lock, _owner_identity, _shutdown_latch_spawner_lock = threading.Lock()# A different pid here._owner_identity = None@@ -366,6 +366,14 @@ def _reset_after_fork() -> None:# A fork while another thread was inside adopt_pid / forget_pid leaves this held here with nobody to release it, and# the first adoption blocks forever._record_lock = threading.Lock()+ # Same hazard: Event carries an internal lock, so a fork taken while another thread+ # was inside set() leaves the child unable to latch. Rebuilt holding the flag it had+ # -- the child is still inside the lifecycle that forked it, and a cleared latch+ # would read as permission to spawn.+ _was_latched = _shutdown_latch.is_set()+ _shutdown_latch = threading.Event()+ if _was_latched:+ _shutdown_latch.set()_spawner = None@@ -988,6 +996,37 @@ def adopt_pid(pid: Optional[int]) -> None:pass+# Set once when the app starts quitting, read by every spawner in the process.+_shutdown_latch = threading.Event()+++def mark_process_shutting_down() -> None:+ """Latch "this process is quitting" for every spawner in it.++ Each subsystem already refuses to spawn during its OWN teardown, but that state+ lives on the object being torn down: a second LlamaCppBackend built for a helper+ load, or the inference orchestrator, never sees it and can Popen a child after+ terminate_all has taken its snapshot. Set once here, read everywhere, so the answer+ does not depend on which object a spawn happens to belong to.+ """+ _shutdown_latch.set()+++def is_process_shutting_down() -> bool:+ """Whether a spawn must be refused because the app is quitting."""+ return _shutdown_latch.is_set()+++def begin_process_lifecycle() -> None:+ """Clear the latch for an embedded host that calls run_server again.++ Quitting is terminal for a CLI run, but in-process callers (studio/backend/colab.py)+ reuse the interpreter, and a latch that never cleared would refuse every spawn of+ the second session.+ """+ _shutdown_latch.clear()++def terminate_all(timeout: float = 5.0) -> "list[int]":"""Backstop sweep over adopted pids, after per-subsystem cleanup. SIGTERM,then SIGKILL the survivors after `timeout`. Idempotent and teardown-safe.