Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(launcher): reclaim verified orphaned listeners on supervised restart (#1795)

Bingxi Zhao (Frank)b7904fef3a46
diff --git a/deeptutor/runtime/launcher.py b/deeptutor/runtime/launcher.py
index 91e28997a..309430b33 100644
--- a/deeptutor/runtime/launcher.py
+++ b/deeptutor/runtime/launcher.py
@@ -517,7 +517,11 @@ def _prompt_new_ports(
return new_backend, new_frontend
-def _kill_port_listeners(listeners: dict[int, list[tuple[int, str]]]) -> None:
+def _kill_port_listeners(
+ listeners: dict[int, list[tuple[int, str]]],
+ *,
+ listener_guard: Callable[[int, int], bool] | None = None,
+) -> None:
"""Signal every port listener with the SIGTERM -> SIGKILL ladder.
One unreachable listener does not stop the others from being signalled,
@@ -526,6 +530,8 @@ def _kill_port_listeners(listeners: dict[int, list[tuple[int, str]]]) -> None:
"""
for port, entries in listeners.items():
for pid, command in entries:
+ if listener_guard is not None and not listener_guard(port, pid):
+ continue
_log(_t("start.port_killing", pid=pid, command=command))
target = f"port {port} listener (pid={pid})"
_signal_target(pid, None, signal.SIGTERM, target=target)
@@ -534,6 +540,8 @@ def _kill_port_listeners(listeners: dict[int, list[tuple[int, str]]]) -> None:
time.sleep(0.2)
if _port_accepts_connection(port):
for pid, _command in entries:
+ if listener_guard is not None and not listener_guard(port, pid):
+ continue
target = f"port {port} listener (pid={pid})"
_signal_target(pid, None, KILL_SIGNAL, target=target)
deadline = time.monotonic() + 3
@@ -546,18 +554,72 @@ def _kill_port_listeners(listeners: dict[int, list[tuple[int, str]]]) -> None:
_log(_t("start.port_freed", port=port))
+def _owned_orphan_started_at(
+ pid: int, *, role: str, runtime_home: Path, frontend_cwd: Path | None
+) -> float | None:
+ """Identify this deployment's abandoned listener without trusting its title.
+
+ #1795: an unattended restart may reclaim only a launcher-owned child whose
+ supervisor is gone. Unknown metadata, another home, or a live supervisor
+ must preserve the existing port-conflict failure.
+ """
+ import psutil
+
+ try:
+ process = psutil.Process(pid)
+ env = process.environ()
+ home = env.get(DEEPTUTOR_HOME_ENV)
+ if not home or Path(home).resolve() != runtime_home.resolve():
+ return None
+ supervisor = int(env.get(LAUNCHER_PID_ENV) or env.get(SUPERVISOR_PID_ENV) or "0")
+ if supervisor <= 0 or _is_pid_alive(supervisor):
+ return None
+ if role == "start.backend":
+ # Multiprocess uvicorn workers have spawn command lines; their
+ # parent carries the exact application command and interpreter.
+ for candidate in [process, *process.parents()]:
+ if candidate.pid <= 1:
+ break
+ args = candidate.cmdline()
+ if any(
+ args[i : i + 3] == ["-m", "uvicorn", "deeptutor.api.main:app"]
+ for i in range(len(args) - 2)
+ ):
+ if (
+ Path(candidate.exe()).resolve() == Path(sys.executable).resolve()
+ and Path(candidate.cwd()).resolve() == runtime_home.resolve()
+ ):
+ return process.create_time()
+ elif frontend_cwd is not None:
+ cwd = Path(process.cwd()).resolve()
+ args = process.cmdline()
+ if cwd != frontend_cwd.resolve() or Path(process.exe()).stem.lower() != "node":
+ return None
+ if any(arg.startswith("next-server (") for arg in args) or any(
+ Path(arg).name == "server.js" and (cwd / arg).resolve() == cwd / "server.js"
+ for arg in args
+ ):
+ return process.create_time()
+ except (psutil.Error, OSError, ValueError):
+ return None
+ return None
+
+
def _resolve_port_conflicts(
*,
backend_port: int,
frontend_port: int,
check_frontend: bool,
settings_dir: Path,
+ runtime_home: Path | None = None,
+ frontend_cwd: Path | None = None,
) -> tuple[int, int]:
"""Return free ``(backend_port, frontend_port)``, resolving conflicts interactively.
- When stdin is not a TTY (Docker, CI), falls back to exiting with the
- historical ``start.port_in_use`` message.
+ Without a TTY, reclaim verified orphaned children of this deployment once;
+ unknown or foreign listeners retain the historical port-conflict failure.
"""
+ reclaimed = False
while True:
roles = [("start.backend", backend_port)]
if check_frontend:
@@ -577,6 +639,38 @@ def _resolve_port_conflicts(
_log(_t("start.port_conflict_proc", pid=pid, command=command))
if sys.stdin is None or not sys.stdin.isatty():
+ if not reclaimed and runtime_home is not None:
+ roles_by_port = {port: key for key, port in occupied}
+ identities = {
+ (port, pid): _owned_orphan_started_at(
+ pid,
+ role=roles_by_port[port],
+ runtime_home=runtime_home,
+ frontend_cwd=frontend_cwd,
+ )
+ for port, entries in listeners.items()
+ for pid, _command in entries
+ }
+ if (
+ all(listeners.values())
+ and identities
+ and all(started is not None for started in identities.values())
+ ):
+
+ def still_owned(port: int, pid: int) -> bool:
+ return (
+ _owned_orphan_started_at(
+ pid,
+ role=roles_by_port[port],
+ runtime_home=runtime_home,
+ frontend_cwd=frontend_cwd,
+ )
+ == identities[(port, pid)]
+ )
+
+ _kill_port_listeners(listeners, listener_guard=still_owned)
+ reclaimed = True
+ continue
joined = ", ".join(str(port) for _key, port in occupied)
raise SystemExit(_t("start.port_in_use", ports=joined))
@@ -1386,6 +1480,8 @@ def start(
frontend_port=frontend_port,
check_frontend=existing_frontend is None,
settings_dir=settings.settings_dir,
+ runtime_home=runtime_home,
+ frontend_cwd=frontend.cwd,
)
if (resolved_backend, resolved_frontend) != (backend_port, frontend_port):
backend_port, frontend_port = resolved_backend, resolved_frontend
@@ -1530,8 +1626,10 @@ def start(
if cleanup_started:
return
cleanup_started = True
- _terminate(web)
- _terminate(backend)
+ try:
+ _terminate(backend)
+ finally:
+ _terminate(web)
if detached_paths is not None:
_clear_detached_runtime(detached_paths, detached_token)
diff --git a/tests/runtime/test_launcher.py b/tests/runtime/test_launcher.py
index bee8a9997..e5d45e37e 100644
--- a/tests/runtime/test_launcher.py
+++ b/tests/runtime/test_launcher.py
@@ -2,11 +2,15 @@ from __future__ import annotations
import builtins
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import json
import logging
+import os
from pathlib import Path
import signal
import subprocess
+import sys
from threading import Thread
+import time
from types import SimpleNamespace
import pytest
@@ -337,6 +341,196 @@ def test_resolve_port_conflicts_non_tty_exits_with_message(tmp_path: Path, monke
assert "8000" in str(excinfo.value)
[email protected]("role", ["start.backend", "start.frontend", "backend_worker"])
+def test_reclaim_requires_same_deployment_and_dead_launcher(tmp_path, monkeypatch, role):
+ import psutil
+
+ home = tmp_path / "home"
+ frontend = home / "web"
+ env = {launcher.DEEPTUTOR_HOME_ENV: str(home), launcher.LAUNCHER_PID_ENV: "4242"}
+ backend = SimpleNamespace(
+ pid=100,
+ cmdline=lambda: [sys.executable, "-m", "uvicorn", "deeptutor.api.main:app"],
+ exe=lambda: sys.executable,
+ cwd=lambda: str(home),
+ )
+ process = SimpleNamespace(
+ pid=101,
+ environ=lambda: env,
+ create_time=lambda: 123.0,
+ parents=lambda: [backend],
+ cmdline=backend.cmdline,
+ exe=backend.exe,
+ cwd=backend.cwd,
+ )
+ if role == "start.frontend":
+ process.cmdline = lambda: ["next-server (v16.2.3)"]
+ process.exe = lambda: "/usr/local/bin/node"
+ process.cwd = lambda: str(frontend)
+ elif role == "backend_worker":
+ process.cmdline = lambda: [
+ sys.executable,
+ "-c",
+ "from multiprocessing.spawn import spawn_main",
+ ]
+ resolved_role = "start.frontend" if role == "start.frontend" else "start.backend"
+ monkeypatch.setattr(psutil, "Process", lambda _pid: process)
+ monkeypatch.setattr(launcher, "_is_pid_alive", lambda _pid: False)
+
+ def inspect():
+ return launcher._owned_orphan_started_at(
+ 101, role=resolved_role, runtime_home=home, frontend_cwd=frontend
+ )
+
+ assert inspect() == 123.0
+ monkeypatch.setattr(launcher, "_is_pid_alive", lambda _pid: True)
+ assert inspect() is None
+ monkeypatch.setattr(launcher, "_is_pid_alive", lambda _pid: False)
+ env[launcher.DEEPTUTOR_HOME_ENV] = str(tmp_path / "another-install")
+ assert inspect() is None
+ env[launcher.DEEPTUTOR_HOME_ENV] = str(home)
+ env.pop(launcher.LAUNCHER_PID_ENV)
+ assert inspect() is None
+
+
[email protected]("fault", ["command", "interpreter", "cwd", "access_denied"])
+def test_reclaim_rejects_unverified_backend_identity(tmp_path, monkeypatch, fault):
+ import psutil
+
+ def environment():
+ if fault == "access_denied":
+ raise psutil.AccessDenied(101)
+ return {launcher.DEEPTUTOR_HOME_ENV: str(tmp_path), launcher.LAUNCHER_PID_ENV: "4242"}
+
+ process = SimpleNamespace(
+ pid=101,
+ environ=environment,
+ create_time=lambda: 123.0,
+ parents=lambda: [],
+ cmdline=lambda: [
+ sys.executable,
+ "-m",
+ "uvicorn",
+ "foreign.api:app" if fault == "command" else "deeptutor.api.main:app",
+ ],
+ exe=lambda: "/another/python" if fault == "interpreter" else sys.executable,
+ cwd=lambda: str(tmp_path / "other") if fault == "cwd" else str(tmp_path),
+ )
+ monkeypatch.setattr(psutil, "Process", lambda _pid: process)
+ monkeypatch.setattr(launcher, "_is_pid_alive", lambda _pid: False)
+ assert (
+ launcher._owned_orphan_started_at(
+ 101, role="start.backend", runtime_home=tmp_path, frontend_cwd=None
+ )
+ is None
+ )
+
+
[email protected]("case", ["owned", "unknown", "mixed", "pid_reused", "still_busy"])
+def test_noninteractive_restart_reclaims_only_verified_orphans_once(tmp_path, monkeypatch, case):
+ occupied = {8000}
+ attempts, signals, inspected = [], [], []
+ entries = [] if case == "unknown" else [(101, "backend")]
+ if case == "mixed":
+ entries.append((102, "foreign"))
+
+ def identity(pid, **_kwargs):
+ inspected.append(pid)
+ if pid == 102:
+ return None
+ return 456.0 if case == "pid_reused" and len(inspected) > 1 else 123.0
+
+ def reclaim(listeners, *, listener_guard):
+ attempts.append(listeners)
+ if listener_guard(8000, 101):
+ signals.append(101)
+ if case == "owned":
+ occupied.clear()
+
+ monkeypatch.setattr(launcher.sys, "stdin", None)
+ monkeypatch.setattr(launcher, "_port_accepts_connection", lambda port: port in occupied)
+ monkeypatch.setattr(launcher, "_port_listeners", lambda _port: entries)
+ monkeypatch.setattr(launcher, "_owned_orphan_started_at", identity)
+ monkeypatch.setattr(launcher, "_kill_port_listeners", reclaim)
+ kwargs = dict(
+ backend_port=8000,
+ frontend_port=3784,
+ check_frontend=False,
+ settings_dir=tmp_path,
+ runtime_home=tmp_path,
+ )
+ if case == "owned":
+ assert launcher._resolve_port_conflicts(**kwargs) == (8000, 3784)
+ else:
+ with pytest.raises(SystemExit):
+ launcher._resolve_port_conflicts(**kwargs)
+ assert len(attempts) == (0 if case in {"unknown", "mixed"} else 1)
+ assert signals == ([] if case in {"unknown", "mixed", "pid_reused"} else [101])
+
+
[email protected](os.name == "nt", reason="POSIX orphan lifecycle integration")
+def test_noninteractive_restart_recovers_an_actual_orphaned_backend_listener(tmp_path, monkeypatch):
+ """#1795: simulate launcher death with an isolated socket without starting the app."""
+ import psutil
+
+ (tmp_path / "uvicorn.py").write_text(
+ "import socket,time,json,os\ns=socket.socket()\ns.bind(('127.0.0.1',0))\n"
+ "s.listen()\nwith open('ready.json','w') as f: json.dump({'port':s.getsockname()[1]},f)\n"
+ "time.sleep(60)\n"
+ )
+ parent_code = (
+ "import os,subprocess,sys\n"
+ f"os.environ[{launcher.LAUNCHER_PID_ENV!r}]=str(os.getpid())\n"
+ "p=subprocess.Popen([sys.executable,'-m','uvicorn','deeptutor.api.main:app'],stdout=subprocess.DEVNULL)\n"
+ "print(p.pid,flush=True)\n"
+ )
+ env = os.environ.copy()
+ env[launcher.DEEPTUTOR_HOME_ENV] = str(tmp_path)
+ env.pop("PYTHONPATH", None)
+ child = psutil.Process(
+ int(
+ subprocess.check_output(
+ [sys.executable, "-c", parent_code],
+ cwd=tmp_path,
+ env=env,
+ text=True,
+ timeout=5,
+ )
+ )
+ )
+ try:
+ ready = tmp_path / "ready.json"
+ deadline = time.monotonic() + 5
+ while not ready.exists() and time.monotonic() < deadline:
+ time.sleep(0.05)
+ port = json.loads(ready.read_text())["port"]
+ assert launcher._port_accepts_connection(port)
+ assert (
+ launcher._owned_orphan_started_at(
+ child.pid, role="start.backend", runtime_home=tmp_path, frontend_cwd=None
+ )
+ == child.create_time()
+ )
+ monkeypatch.setattr(launcher.sys, "stdin", None)
+ # Enumerate only this test's listener, independent of lsof availability.
+ monkeypatch.setattr(
+ launcher, "_port_listeners", lambda _port: [(child.pid, "controlled uvicorn")]
+ )
+ assert launcher._resolve_port_conflicts(
+ backend_port=port,
+ frontend_port=3784,
+ check_frontend=False,
+ settings_dir=tmp_path,
+ runtime_home=tmp_path,
+ ) == (port, 3784)
+ assert not launcher._port_accepts_connection(port)
+ finally:
+ try:
+ child.kill()
+ except psutil.NoSuchProcess:
+ pass
+
+
def test_resolve_port_conflicts_kill_option_frees_port(tmp_path: Path, monkeypatch) -> None:
occupied = {8000}
killed: list[int] = []
@@ -627,6 +821,7 @@ def test_start_uses_ipv4_loopback_for_frontend_proxy(
system_json_path=settings_dir / "system.json",
)
captured_envs: dict[str, dict[str, str]] = {}
+ termination_order = []
monkeypatch.setattr(launcher, "_relax_console_encoding", lambda: None)
monkeypatch.setattr(launcher, "_reset_runtime_singletons", lambda: None)
@@ -659,8 +854,17 @@ def test_start_uses_ipv4_loopback_for_frontend_proxy(
)
monkeypatch.setattr(launcher, "_install_signal_handlers", lambda _callback, **_kwargs: None)
monkeypatch.setattr(launcher.atexit, "register", lambda _callback: None)
- monkeypatch.setattr(launcher, "_wait_for_http", lambda **_kwargs: None)
- monkeypatch.setattr(launcher, "_terminate", lambda _process: None)
+
+ def wait_for_http(**kwargs):
+ if kwargs["process"].name == "frontend":
+ raise RuntimeError("captured launch environment")
+
+ monkeypatch.setattr(launcher, "_wait_for_http", wait_for_http)
+ monkeypatch.setattr(
+ launcher,
+ "_terminate",
+ lambda process: termination_order.append(process.name) if process else None,
+ )
def _capture_spawn(_command, *, cwd, env, name):
assert cwd == tmp_path
@@ -668,7 +872,7 @@ def test_start_uses_ipv4_loopback_for_frontend_proxy(
if name == "backend":
return launcher.ManagedProcess("backend", object(), None)
assert name == "frontend"
- raise RuntimeError("captured launch environment")
+ return launcher.ManagedProcess("frontend", object(), None)
monkeypatch.setattr(launcher, "_spawn", _capture_spawn)
@@ -682,6 +886,7 @@ def test_start_uses_ipv4_loopback_for_frontend_proxy(
assert "DEEPTUTOR_NEXT_DIST_DIR" not in captured_envs["frontend"]
assert launcher.DETACHED_WORKER_ENV not in captured_envs["backend"]
assert launcher.DETACHED_TOKEN_ENV not in captured_envs["backend"]
+ assert termination_order == ["backend", "frontend"]
def test_foreground_signal_handlers_keep_windows_ctrl_c(monkeypatch) -> None:
diff --git a/tests/runtime/test_launcher_lifecycle.py b/tests/runtime/test_launcher_lifecycle.py
index 8e0ee8564..22777082f 100644
--- a/tests/runtime/test_launcher_lifecycle.py
+++ b/tests/runtime/test_launcher_lifecycle.py
@@ -214,6 +214,24 @@ def test_terminate_swallows_signal_errors_and_still_reaps(monkeypatch) -> None:
# --------------------------------------------------------------------------
+def test_guarded_reclaim_does_not_signal_a_replaced_process(monkeypatch, fast_clock) -> None:
+ """#1795: revalidate ownership before both TERM and KILL, not just once."""
+ owned = {"current": True}
+ signals = []
+
+ def signal_target(pid, pgid, sig):
+ signals.append(sig)
+ owned["current"] = False
+
+ monkeypatch.setattr(launcher, "_send_tree_signal", signal_target)
+ monkeypatch.setattr(launcher, "_port_accepts_connection", lambda _port: True)
+ launcher._kill_port_listeners(
+ {8000: [(4242, "old backend")]},
+ listener_guard=lambda _port, _pid: owned["current"],
+ )
+ assert signals == [signal.SIGTERM]
+
+
def test_kill_port_listeners_sigterm_frees_the_port(monkeypatch, fast_clock) -> None:
occupied = {8000}
signals = _SignalRecorder()