Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(server): stop API server when backend dies

jason-fxz711325d17edc
diff --git a/python/freetoken/server/api_server.py b/python/freetoken/server/api_server.py
index 64a814b8..85c94037 100644
--- a/python/freetoken/server/api_server.py
+++ b/python/freetoken/server/api_server.py
@@ -56,6 +56,7 @@ _MODEL_SAMPLING: Dict[str, Any] = {}
# signal handler). The backend supervisor reads this so a worker exiting AS PART of that
# shutdown is treated as expected — no ERROR log, no "failed" latch. See run_backend_supervisor.
_SHUTTING_DOWN = threading.Event()
+BACKEND_DEATH_EXIT_GRACE_S = 10.0
def get_global_state() -> FrontendManager:
@@ -83,6 +84,19 @@ def _terminate_backend_workers(processes: List[Any]) -> None:
continue
+def _exit_after_backend_death(grace_s: float) -> threading.Timer:
+ def _stop() -> None:
+ if _SHUTTING_DOWN.is_set():
+ return # an external stop got here first
+ logger.error("Backend worker is gone and cannot be restarted; stopping the API server")
+ os.kill(os.getpid(), signal.SIGTERM)
+
+ timer = threading.Timer(grace_s, _stop)
+ timer.daemon = True
+ timer.start()
+ return timer
+
+
def _reap_backend_workers(processes: List[Any], timeout: float = 5.0) -> None:
"""Wait out a preceding ``_terminate_backend_workers`` and SIGKILL whatever is still
standing. Only the shell path needs this: it owns the process lifetime end to end (no
@@ -979,6 +993,10 @@ def run_api_server(config: ServerArgs, start_backend: Callable[[], "Any"], run_s
# No CacheRebuildReply will ever arrive from a dead backend, so wake any caller blocked
# in dispatch_rebuild's await now — otherwise it strands until the full rebuild timeout.
_GLOBAL_STATE.fail_pending_rebuilds(message)
+ # Then take the whole serve down (see _exit_after_backend_death). Shell mode is excluded:
+ # a person is sitting at that TUI, the API is theirs alone, and its stop path is ^C.
+ if not run_shell:
+ _exit_after_backend_death(BACKEND_DEATH_EXIT_GRACE_S)
def _on_meta(meta: dict) -> None:
# Per-unit cache VRAM costs + the free-VRAM seed + per-pool floors + the actual pool