DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(runtime): surface launcher termination and port-kill signal failures (#1715)
evan188199-tech6c5a6fa1c98e
diff --git a/deeptutor/runtime/launcher.py b/deeptutor/runtime/launcher.pyindex 76cb6564c..968d8eab7 100644--- a/deeptutor/runtime/launcher.py+++ b/deeptutor/runtime/launcher.py@@ -6,6 +6,7 @@ import atexitfrom dataclasses import dataclassimport hashlibimport json+import loggingimport osfrom pathlib import Pathimport secrets@@ -35,6 +36,8 @@ from deeptutor.services.app_update import LAUNCHER_PID_ENVBACKEND_READY_TIMEOUT_ENV = "DEEPTUTOR_BACKEND_READY_TIMEOUT"FRONTEND_READY_TIMEOUT_ENV = "DEEPTUTOR_FRONTEND_READY_TIMEOUT"+logger = logging.getLogger(__name__)+def _ready_timeout(env_name: str, default: int) -> int:"""Seconds to wait for a child to answer, overridable per deployment.@@ -237,13 +240,17 @@ def _send_tree_signal(pid: int | None, pgid: int | None, sig: signal.Signals | icmd = ["taskkill", "/PID", str(pid), "/T"]if sig == KILL_SIGNAL:cmd.append("/F")- subprocess.run(+ completed = subprocess.run(cmd,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=False,**_no_window_kwargs(),)+ # Exit code 128 is taskkill's "process not found": the tree is+ # already gone, which is what the caller wanted anyway.+ if completed.returncode not in (0, 128):+ raise RuntimeError(f"taskkill {pid} failed with exit code {completed.returncode}")returnif os.name != "nt" and pgid is not None:os.killpg(pgid, sig)@@ -251,21 +258,50 @@ def _send_tree_signal(pid: int | None, pgid: int | None, sig: signal.Signals | ios.kill(pid, sig)+def _signal_target(+ pid: int | None,+ pgid: int | None,+ sig: signal.Signals | int,+ *,+ target: str,+) -> None:+ """Deliver ``sig`` to a process tree, logging failures instead of+ swallowing them.++ A target that is already gone (ESRCH on POSIX, taskkill's "process not+ found" on Windows) has reached the intended end state, so it is not a+ delivery failure. Any other failure is logged with its cause; the+ caller's escalation ladder is unchanged.+ """+ try:+ _send_tree_signal(pid, pgid, sig)+ except ProcessLookupError:+ return+ except Exception:+ logger.warning(+ "failed to send %s to %s",+ getattr(sig, "name", sig),+ target,+ exc_info=True,+ )++def _terminate(proc: ManagedProcess | None) -> None:+ """Stop ``proc`` with the SIGTERM -> SIGKILL ladder.++ A signal that could not be delivered is logged rather than silently+ dropped, so a surviving child can be traced back to its failed+ termination.+ """if proc is None or proc.process.poll() is not None:return_log(_t("start.stopping", name=proc.name, pid=proc.process.pid))- try:- _send_tree_signal(proc.process.pid, proc.pgid, signal.SIGTERM)- except Exception:- pass+ target = f"{proc.name} (pid={proc.process.pid})"+ _signal_target(proc.process.pid, proc.pgid, signal.SIGTERM, target=target)try:proc.process.wait(timeout=8)except subprocess.TimeoutExpired:- try:- _send_tree_signal(proc.process.pid, proc.pgid, KILL_SIGNAL)- except Exception:- pass+ _signal_target(proc.process.pid, proc.pgid, KILL_SIGNAL, target=target)def _relax_console_encoding(streams: tuple[object, ...] | None = None) -> None:@@ -497,22 +533,24 @@ def _prompt_new_ports(def _kill_port_listeners(listeners: dict[int, list[tuple[int, str]]]) -> None:+ """Signal every port listener with the SIGTERM -> SIGKILL ladder.++ One unreachable listener does not stop the others from being signalled,+ and each failed delivery leaves a warning in the log instead of a silent+ zombie.+ """for port, entries in listeners.items():for pid, command in entries:_log(_t("start.port_killing", pid=pid, command=command))- try:- _send_tree_signal(pid, None, signal.SIGTERM)- except Exception:- pass+ target = f"port {port} listener (pid={pid})"+ _signal_target(pid, None, signal.SIGTERM, target=target)deadline = time.monotonic() + 5while time.monotonic() < deadline and _port_accepts_connection(port):time.sleep(0.2)if _port_accepts_connection(port):for pid, _command in entries:- try:- _send_tree_signal(pid, None, KILL_SIGNAL)- except Exception:- pass+ target = f"port {port} listener (pid={pid})"+ _signal_target(pid, None, KILL_SIGNAL, target=target)deadline = time.monotonic() + 3while time.monotonic() < deadline and _port_accepts_connection(port):time.sleep(0.2)diff --git a/tests/runtime/test_launcher.py b/tests/runtime/test_launcher.pyindex 23729cabc..f95eec796 100644--- a/tests/runtime/test_launcher.py+++ b/tests/runtime/test_launcher.py@@ -2,7 +2,10 @@ from __future__ import annotationsimport builtinsfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+import loggingfrom pathlib import Path+import signal+import subprocessfrom threading import Threadfrom types import SimpleNamespace@@ -927,3 +930,181 @@ def test_port_listeners_tolerate_missing_stdout(monkeypatch, platform: str) -> Nlauncher.subprocess, "run", lambda *args, **kwargs: SimpleNamespace(stdout=None))assert launcher._port_listeners(3782) == []+++class _StubbornPopen:+ """Popen stand-in that stays alive until a fake signal lets it exit."""++ def __init__(self, *, pid: int = 4242) -> None:+ self.pid = pid+ self.exited = False++ def poll(self) -> int | None:+ return 0 if self.exited else None++ def wait(self, timeout: float | None = None) -> int:+ if not self.exited:+ raise subprocess.TimeoutExpired(cmd="backend", timeout=timeout)+ return 0+++def test_terminate_warns_when_both_signal_rounds_fail(+ monkeypatch, caplog: pytest.LogCaptureFixture+) -> None:+ """A child no signal reaches must surface every failed round."""+ child = _StubbornPopen()+ proc = launcher.ManagedProcess(name="backend", process=child, pgid=4242)+ sent: list[int] = []++ def fake_send(pid, pgid, sig):+ sent.append(sig)+ # Neither round is delivered: SIGTERM is refused, SIGKILL vanishes.+ raise PermissionError("operation not permitted")++ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)++ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):+ launcher._terminate(proc)++ assert sent == [signal.SIGTERM, launcher.KILL_SIGNAL]+ warnings = [record.getMessage() for record in caplog.records]+ assert any("SIGTERM" in message for message in warnings)+ assert any("SIGKILL" in message for message in warnings)+++def test_terminate_warns_on_sigterm_but_accepts_kill_escalation(+ monkeypatch, caplog: pytest.LogCaptureFixture+) -> None:+ """The ladder keeps its shape: refused SIGTERM, working SIGKILL, done."""+ child = _StubbornPopen()+ proc = launcher.ManagedProcess(name="backend", process=child, pgid=4242)++ def fake_send(pid, pgid, sig):+ if sig == launcher.KILL_SIGNAL:+ child.exited = True+ return+ raise PermissionError("operation not permitted")++ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)++ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):+ launcher._terminate(proc)++ assert child.exited is True+ warnings = [record.getMessage() for record in caplog.records]+ assert any("SIGTERM" in message for message in warnings)+ assert not any("SIGKILL" in message for message in warnings)+++def test_terminate_skips_absent_or_exited_child(+ monkeypatch, caplog: pytest.LogCaptureFixture+) -> None:+ """Nothing to stop means no signal and no warning."""+ sent: list[int] = []++ def fake_send(pid, pgid, sig):+ sent.append(sig)++ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)+ exited = _StubbornPopen()+ exited.exited = True++ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):+ launcher._terminate(None)+ launcher._terminate(launcher.ManagedProcess(name="web", process=exited, pgid=None))++ assert sent == []+ assert caplog.records == []+++def test_signal_target_treats_already_gone_target_as_terminated(+ monkeypatch, caplog: pytest.LogCaptureFixture+) -> None:+ """ESRCH means the target exited on its own; that is not a failure."""++ def fake_send(pid, pgid, sig):+ raise ProcessLookupError()++ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)++ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):+ launcher._signal_target(123, None, signal.SIGTERM, target="port 8000 listener")++ assert caplog.records == []+++def test_windows_taskkill_failures_surface_but_missing_processes_do_not(+ monkeypatch, caplog: pytest.LogCaptureFixture+) -> None:+ """A failing taskkill must be logged; one that found no process must not."""+ monkeypatch.setattr(launcher, "os", SimpleNamespace(name="nt"))+ monkeypatch.setattr(launcher, "_no_window_kwargs", lambda: {})+ codes: list[int] = []+ monkeypatch.setattr(+ launcher.subprocess,+ "run",+ lambda *args, **kwargs: SimpleNamespace(returncode=codes.pop(0)),+ )++ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):+ for code in (1, 0, 128):+ codes.append(code)+ launcher._signal_target(7, None, signal.SIGTERM, target="backend (pid=7)")++ warnings = [record.getMessage() for record in caplog.records]+ assert len(warnings) == 1+ assert "SIGTERM" in warnings[0]+ assert "taskkill" in (caplog.records[0].exc_text or "")+++def test_kill_port_listeners_isolates_one_targets_sigterm_failure(+ monkeypatch, caplog: pytest.LogCaptureFixture+) -> None:+ """One unreachable listener must not spare the others or hide itself."""+ occupied = {8000}++ def fake_send(pid, pgid, sig):+ if pid == 123:+ raise PermissionError("operation not permitted")+ occupied.discard(8000)++ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)+ monkeypatch.setattr(launcher, "_port_accepts_connection", lambda port: port in occupied)++ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):+ launcher._kill_port_listeners({8000: [(123, "python uvicorn"), (456, "node")]})++ assert occupied == set()+ assert any("123" in record.getMessage() for record in caplog.records)+++def test_kill_port_listeners_reports_kill_round_failures_per_target(+ monkeypatch, caplog: pytest.LogCaptureFixture+) -> None:+ """SIGTERM-ignoring listeners escalate; KILL failures stay per-target."""+ occupied = {8000}+ sent: list[tuple[int, int]] = []+ clock = {"now": 0.0}++ def fake_send(pid, pgid, sig):+ sent.append((pid, sig))+ if sig == launcher.KILL_SIGNAL and pid == 123:+ raise OSError("no such process")+ # pid 456 accepts both signals; zombie 123 keeps holding the port.++ def fake_sleep(seconds: float) -> None:+ clock["now"] += seconds++ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)+ monkeypatch.setattr(launcher, "_port_accepts_connection", lambda port: port in occupied)+ monkeypatch.setattr(launcher.time, "monotonic", lambda: clock["now"])+ monkeypatch.setattr(launcher.time, "sleep", fake_sleep)++ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):+ launcher._kill_port_listeners({8000: [(123, "python uvicorn"), (456, "node")]})++ assert (123, signal.SIGTERM) in sent+ assert (456, signal.SIGTERM) in sent+ assert (123, launcher.KILL_SIGNAL) in sent+ assert (456, launcher.KILL_SIGNAL) in sent+ assert any("SIGKILL" in record.getMessage() for record in caplog.records)