DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(launcher): reclaim verified orphaned listeners on supervised restart (#1795)
Bingxi Zhao (Frank)b7904fef3a46
diff --git a/deeptutor/runtime/launcher.py b/deeptutor/runtime/launcher.pyindex 91e28997a..309430b33 100644--- a/deeptutor/runtime/launcher.py+++ b/deeptutor/runtime/launcher.py@@ -517,7 +517,11 @@ def _prompt_new_ports(return new_backend, new_frontend-def _kill_port_listeners(listeners: dict[int, list[tuple[int, str]]]) -> None:+def _kill_port_listeners(+ listeners: dict[int, list[tuple[int, str]]],+ *,+ listener_guard: Callable[[int, int], bool] | None = None,+) -> None:"""Signal every port listener with the SIGTERM -> SIGKILL ladder.One unreachable listener does not stop the others from being signalled,@@ -526,6 +530,8 @@ def _kill_port_listeners(listeners: dict[int, list[tuple[int, str]]]) -> None:"""for port, entries in listeners.items():for pid, command in entries:+ if listener_guard is not None and not listener_guard(port, pid):+ continue_log(_t("start.port_killing", pid=pid, command=command))target = f"port {port} listener (pid={pid})"_signal_target(pid, None, signal.SIGTERM, target=target)@@ -534,6 +540,8 @@ def _kill_port_listeners(listeners: dict[int, list[tuple[int, str]]]) -> None:time.sleep(0.2)if _port_accepts_connection(port):for pid, _command in entries:+ if listener_guard is not None and not listener_guard(port, pid):+ continuetarget = f"port {port} listener (pid={pid})"_signal_target(pid, None, KILL_SIGNAL, target=target)deadline = time.monotonic() + 3@@ -546,18 +554,72 @@ def _kill_port_listeners(listeners: dict[int, list[tuple[int, str]]]) -> None:_log(_t("start.port_freed", port=port))+def _owned_orphan_started_at(+ pid: int, *, role: str, runtime_home: Path, frontend_cwd: Path | None+) -> float | None:+ """Identify this deployment's abandoned listener without trusting its title.++ #1795: an unattended restart may reclaim only a launcher-owned child whose+ supervisor is gone. Unknown metadata, another home, or a live supervisor+ must preserve the existing port-conflict failure.+ """+ import psutil++ try:+ process = psutil.Process(pid)+ env = process.environ()+ home = env.get(DEEPTUTOR_HOME_ENV)+ if not home or Path(home).resolve() != runtime_home.resolve():+ return None+ supervisor = int(env.get(LAUNCHER_PID_ENV) or env.get(SUPERVISOR_PID_ENV) or "0")+ if supervisor <= 0 or _is_pid_alive(supervisor):+ return None+ if role == "start.backend":+ # Multiprocess uvicorn workers have spawn command lines; their+ # parent carries the exact application command and interpreter.+ for candidate in [process, *process.parents()]:+ if candidate.pid <= 1:+ break+ args = candidate.cmdline()+ if any(+ args[i : i + 3] == ["-m", "uvicorn", "deeptutor.api.main:app"]+ for i in range(len(args) - 2)+ ):+ if (+ Path(candidate.exe()).resolve() == Path(sys.executable).resolve()+ and Path(candidate.cwd()).resolve() == runtime_home.resolve()+ ):+ return process.create_time()+ elif frontend_cwd is not None:+ cwd = Path(process.cwd()).resolve()+ args = process.cmdline()+ if cwd != frontend_cwd.resolve() or Path(process.exe()).stem.lower() != "node":+ return None+ if any(arg.startswith("next-server (") for arg in args) or any(+ Path(arg).name == "server.js" and (cwd / arg).resolve() == cwd / "server.js"+ for arg in args+ ):+ return process.create_time()+ except (psutil.Error, OSError, ValueError):+ return None+ return None++def _resolve_port_conflicts(*,backend_port: int,frontend_port: int,check_frontend: bool,settings_dir: Path,+ runtime_home: Path | None = None,+ frontend_cwd: Path | None = None,) -> tuple[int, int]:"""Return free ``(backend_port, frontend_port)``, resolving conflicts interactively.- When stdin is not a TTY (Docker, CI), falls back to exiting with the- historical ``start.port_in_use`` message.+ Without a TTY, reclaim verified orphaned children of this deployment once;+ unknown or foreign listeners retain the historical port-conflict failure."""+ reclaimed = Falsewhile True:roles = [("start.backend", backend_port)]if check_frontend:@@ -577,6 +639,38 @@ def _resolve_port_conflicts(_log(_t("start.port_conflict_proc", pid=pid, command=command))if sys.stdin is None or not sys.stdin.isatty():+ if not reclaimed and runtime_home is not None:+ roles_by_port = {port: key for key, port in occupied}+ identities = {+ (port, pid): _owned_orphan_started_at(+ pid,+ role=roles_by_port[port],+ runtime_home=runtime_home,+ frontend_cwd=frontend_cwd,+ )+ for port, entries in listeners.items()+ for pid, _command in entries+ }+ if (+ all(listeners.values())+ and identities+ and all(started is not None for started in identities.values())+ ):++ def still_owned(port: int, pid: int) -> bool:+ return (+ _owned_orphan_started_at(+ pid,+ role=roles_by_port[port],+ runtime_home=runtime_home,+ frontend_cwd=frontend_cwd,+ )+ == identities[(port, pid)]+ )++ _kill_port_listeners(listeners, listener_guard=still_owned)+ reclaimed = True+ continuejoined = ", ".join(str(port) for _key, port in occupied)raise SystemExit(_t("start.port_in_use", ports=joined))@@ -1386,6 +1480,8 @@ def start(frontend_port=frontend_port,check_frontend=existing_frontend is None,settings_dir=settings.settings_dir,+ runtime_home=runtime_home,+ frontend_cwd=frontend.cwd,)if (resolved_backend, resolved_frontend) != (backend_port, frontend_port):backend_port, frontend_port = resolved_backend, resolved_frontend@@ -1530,8 +1626,10 @@ def start(if cleanup_started:returncleanup_started = True- _terminate(web)- _terminate(backend)+ try:+ _terminate(backend)+ finally:+ _terminate(web)if detached_paths is not None:_clear_detached_runtime(detached_paths, detached_token)diff --git a/tests/runtime/test_launcher.py b/tests/runtime/test_launcher.pyindex bee8a9997..e5d45e37e 100644--- a/tests/runtime/test_launcher.py+++ b/tests/runtime/test_launcher.py@@ -2,11 +2,15 @@ from __future__ import annotationsimport builtinsfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+import jsonimport logging+import osfrom pathlib import Pathimport signalimport subprocess+import sysfrom threading import Thread+import timefrom types import SimpleNamespaceimport pytest@@ -337,6 +341,196 @@ def test_resolve_port_conflicts_non_tty_exits_with_message(tmp_path: Path, monkeassert "8000" in str(excinfo.value)[email protected]("role", ["start.backend", "start.frontend", "backend_worker"])+def test_reclaim_requires_same_deployment_and_dead_launcher(tmp_path, monkeypatch, role):+ import psutil++ home = tmp_path / "home"+ frontend = home / "web"+ env = {launcher.DEEPTUTOR_HOME_ENV: str(home), launcher.LAUNCHER_PID_ENV: "4242"}+ backend = SimpleNamespace(+ pid=100,+ cmdline=lambda: [sys.executable, "-m", "uvicorn", "deeptutor.api.main:app"],+ exe=lambda: sys.executable,+ cwd=lambda: str(home),+ )+ process = SimpleNamespace(+ pid=101,+ environ=lambda: env,+ create_time=lambda: 123.0,+ parents=lambda: [backend],+ cmdline=backend.cmdline,+ exe=backend.exe,+ cwd=backend.cwd,+ )+ if role == "start.frontend":+ process.cmdline = lambda: ["next-server (v16.2.3)"]+ process.exe = lambda: "/usr/local/bin/node"+ process.cwd = lambda: str(frontend)+ elif role == "backend_worker":+ process.cmdline = lambda: [+ sys.executable,+ "-c",+ "from multiprocessing.spawn import spawn_main",+ ]+ resolved_role = "start.frontend" if role == "start.frontend" else "start.backend"+ monkeypatch.setattr(psutil, "Process", lambda _pid: process)+ monkeypatch.setattr(launcher, "_is_pid_alive", lambda _pid: False)++ def inspect():+ return launcher._owned_orphan_started_at(+ 101, role=resolved_role, runtime_home=home, frontend_cwd=frontend+ )++ assert inspect() == 123.0+ monkeypatch.setattr(launcher, "_is_pid_alive", lambda _pid: True)+ assert inspect() is None+ monkeypatch.setattr(launcher, "_is_pid_alive", lambda _pid: False)+ env[launcher.DEEPTUTOR_HOME_ENV] = str(tmp_path / "another-install")+ assert inspect() is None+ env[launcher.DEEPTUTOR_HOME_ENV] = str(home)+ env.pop(launcher.LAUNCHER_PID_ENV)+ assert inspect() is None++[email protected]("fault", ["command", "interpreter", "cwd", "access_denied"])+def test_reclaim_rejects_unverified_backend_identity(tmp_path, monkeypatch, fault):+ import psutil++ def environment():+ if fault == "access_denied":+ raise psutil.AccessDenied(101)+ return {launcher.DEEPTUTOR_HOME_ENV: str(tmp_path), launcher.LAUNCHER_PID_ENV: "4242"}++ process = SimpleNamespace(+ pid=101,+ environ=environment,+ create_time=lambda: 123.0,+ parents=lambda: [],+ cmdline=lambda: [+ sys.executable,+ "-m",+ "uvicorn",+ "foreign.api:app" if fault == "command" else "deeptutor.api.main:app",+ ],+ exe=lambda: "/another/python" if fault == "interpreter" else sys.executable,+ cwd=lambda: str(tmp_path / "other") if fault == "cwd" else str(tmp_path),+ )+ monkeypatch.setattr(psutil, "Process", lambda _pid: process)+ monkeypatch.setattr(launcher, "_is_pid_alive", lambda _pid: False)+ assert (+ launcher._owned_orphan_started_at(+ 101, role="start.backend", runtime_home=tmp_path, frontend_cwd=None+ )+ is None+ )++[email protected]("case", ["owned", "unknown", "mixed", "pid_reused", "still_busy"])+def test_noninteractive_restart_reclaims_only_verified_orphans_once(tmp_path, monkeypatch, case):+ occupied = {8000}+ attempts, signals, inspected = [], [], []+ entries = [] if case == "unknown" else [(101, "backend")]+ if case == "mixed":+ entries.append((102, "foreign"))++ def identity(pid, **_kwargs):+ inspected.append(pid)+ if pid == 102:+ return None+ return 456.0 if case == "pid_reused" and len(inspected) > 1 else 123.0++ def reclaim(listeners, *, listener_guard):+ attempts.append(listeners)+ if listener_guard(8000, 101):+ signals.append(101)+ if case == "owned":+ occupied.clear()++ monkeypatch.setattr(launcher.sys, "stdin", None)+ monkeypatch.setattr(launcher, "_port_accepts_connection", lambda port: port in occupied)+ monkeypatch.setattr(launcher, "_port_listeners", lambda _port: entries)+ monkeypatch.setattr(launcher, "_owned_orphan_started_at", identity)+ monkeypatch.setattr(launcher, "_kill_port_listeners", reclaim)+ kwargs = dict(+ backend_port=8000,+ frontend_port=3784,+ check_frontend=False,+ settings_dir=tmp_path,+ runtime_home=tmp_path,+ )+ if case == "owned":+ assert launcher._resolve_port_conflicts(**kwargs) == (8000, 3784)+ else:+ with pytest.raises(SystemExit):+ launcher._resolve_port_conflicts(**kwargs)+ assert len(attempts) == (0 if case in {"unknown", "mixed"} else 1)+ assert signals == ([] if case in {"unknown", "mixed", "pid_reused"} else [101])++[email protected](os.name == "nt", reason="POSIX orphan lifecycle integration")+def test_noninteractive_restart_recovers_an_actual_orphaned_backend_listener(tmp_path, monkeypatch):+ """#1795: simulate launcher death with an isolated socket without starting the app."""+ import psutil++ (tmp_path / "uvicorn.py").write_text(+ "import socket,time,json,os\ns=socket.socket()\ns.bind(('127.0.0.1',0))\n"+ "s.listen()\nwith open('ready.json','w') as f: json.dump({'port':s.getsockname()[1]},f)\n"+ "time.sleep(60)\n"+ )+ parent_code = (+ "import os,subprocess,sys\n"+ f"os.environ[{launcher.LAUNCHER_PID_ENV!r}]=str(os.getpid())\n"+ "p=subprocess.Popen([sys.executable,'-m','uvicorn','deeptutor.api.main:app'],stdout=subprocess.DEVNULL)\n"+ "print(p.pid,flush=True)\n"+ )+ env = os.environ.copy()+ env[launcher.DEEPTUTOR_HOME_ENV] = str(tmp_path)+ env.pop("PYTHONPATH", None)+ child = psutil.Process(+ int(+ subprocess.check_output(+ [sys.executable, "-c", parent_code],+ cwd=tmp_path,+ env=env,+ text=True,+ timeout=5,+ )+ )+ )+ try:+ ready = tmp_path / "ready.json"+ deadline = time.monotonic() + 5+ while not ready.exists() and time.monotonic() < deadline:+ time.sleep(0.05)+ port = json.loads(ready.read_text())["port"]+ assert launcher._port_accepts_connection(port)+ assert (+ launcher._owned_orphan_started_at(+ child.pid, role="start.backend", runtime_home=tmp_path, frontend_cwd=None+ )+ == child.create_time()+ )+ monkeypatch.setattr(launcher.sys, "stdin", None)+ # Enumerate only this test's listener, independent of lsof availability.+ monkeypatch.setattr(+ launcher, "_port_listeners", lambda _port: [(child.pid, "controlled uvicorn")]+ )+ assert launcher._resolve_port_conflicts(+ backend_port=port,+ frontend_port=3784,+ check_frontend=False,+ settings_dir=tmp_path,+ runtime_home=tmp_path,+ ) == (port, 3784)+ assert not launcher._port_accepts_connection(port)+ finally:+ try:+ child.kill()+ except psutil.NoSuchProcess:+ pass++def test_resolve_port_conflicts_kill_option_frees_port(tmp_path: Path, monkeypatch) -> None:occupied = {8000}killed: list[int] = []@@ -627,6 +821,7 @@ def test_start_uses_ipv4_loopback_for_frontend_proxy(system_json_path=settings_dir / "system.json",)captured_envs: dict[str, dict[str, str]] = {}+ termination_order = []monkeypatch.setattr(launcher, "_relax_console_encoding", lambda: None)monkeypatch.setattr(launcher, "_reset_runtime_singletons", lambda: None)@@ -659,8 +854,17 @@ def test_start_uses_ipv4_loopback_for_frontend_proxy()monkeypatch.setattr(launcher, "_install_signal_handlers", lambda _callback, **_kwargs: None)monkeypatch.setattr(launcher.atexit, "register", lambda _callback: None)- monkeypatch.setattr(launcher, "_wait_for_http", lambda **_kwargs: None)- monkeypatch.setattr(launcher, "_terminate", lambda _process: None)++ def wait_for_http(**kwargs):+ if kwargs["process"].name == "frontend":+ raise RuntimeError("captured launch environment")++ monkeypatch.setattr(launcher, "_wait_for_http", wait_for_http)+ monkeypatch.setattr(+ launcher,+ "_terminate",+ lambda process: termination_order.append(process.name) if process else None,+ )def _capture_spawn(_command, *, cwd, env, name):assert cwd == tmp_path@@ -668,7 +872,7 @@ def test_start_uses_ipv4_loopback_for_frontend_proxy(if name == "backend":return launcher.ManagedProcess("backend", object(), None)assert name == "frontend"- raise RuntimeError("captured launch environment")+ return launcher.ManagedProcess("frontend", object(), None)monkeypatch.setattr(launcher, "_spawn", _capture_spawn)@@ -682,6 +886,7 @@ def test_start_uses_ipv4_loopback_for_frontend_proxy(assert "DEEPTUTOR_NEXT_DIST_DIR" not in captured_envs["frontend"]assert launcher.DETACHED_WORKER_ENV not in captured_envs["backend"]assert launcher.DETACHED_TOKEN_ENV not in captured_envs["backend"]+ assert termination_order == ["backend", "frontend"]def test_foreground_signal_handlers_keep_windows_ctrl_c(monkeypatch) -> None:diff --git a/tests/runtime/test_launcher_lifecycle.py b/tests/runtime/test_launcher_lifecycle.pyindex 8e0ee8564..22777082f 100644--- a/tests/runtime/test_launcher_lifecycle.py+++ b/tests/runtime/test_launcher_lifecycle.py@@ -214,6 +214,24 @@ def test_terminate_swallows_signal_errors_and_still_reaps(monkeypatch) -> None:# --------------------------------------------------------------------------+def test_guarded_reclaim_does_not_signal_a_replaced_process(monkeypatch, fast_clock) -> None:+ """#1795: revalidate ownership before both TERM and KILL, not just once."""+ owned = {"current": True}+ signals = []++ def signal_target(pid, pgid, sig):+ signals.append(sig)+ owned["current"] = False++ monkeypatch.setattr(launcher, "_send_tree_signal", signal_target)+ monkeypatch.setattr(launcher, "_port_accepts_connection", lambda _port: True)+ launcher._kill_port_listeners(+ {8000: [(4242, "old backend")]},+ listener_guard=lambda _port, _pid: owned["current"],+ )+ assert signals == [signal.SIGTERM]++def test_kill_port_listeners_sigterm_frees_the_port(monkeypatch, fast_clock) -> None:occupied = {8000}signals = _SignalRecorder()