Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(runtime): surface launcher termination and port-kill signal failures (#1715)

evan188199-tech6c5a6fa1c98e
diff --git a/deeptutor/runtime/launcher.py b/deeptutor/runtime/launcher.py
index 76cb6564c..968d8eab7 100644
--- a/deeptutor/runtime/launcher.py
+++ b/deeptutor/runtime/launcher.py
@@ -6,6 +6,7 @@ import atexit
from dataclasses import dataclass
import hashlib
import json
+import logging
import os
from pathlib import Path
import secrets
@@ -35,6 +36,8 @@ from deeptutor.services.app_update import LAUNCHER_PID_ENV
BACKEND_READY_TIMEOUT_ENV = "DEEPTUTOR_BACKEND_READY_TIMEOUT"
FRONTEND_READY_TIMEOUT_ENV = "DEEPTUTOR_FRONTEND_READY_TIMEOUT"
+logger = logging.getLogger(__name__)
+
def _ready_timeout(env_name: str, default: int) -> int:
"""Seconds to wait for a child to answer, overridable per deployment.
@@ -237,13 +240,17 @@ def _send_tree_signal(pid: int | None, pgid: int | None, sig: signal.Signals | i
cmd = ["taskkill", "/PID", str(pid), "/T"]
if sig == KILL_SIGNAL:
cmd.append("/F")
- subprocess.run(
+ completed = subprocess.run(
cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
**_no_window_kwargs(),
)
+ # Exit code 128 is taskkill's "process not found": the tree is
+ # already gone, which is what the caller wanted anyway.
+ if completed.returncode not in (0, 128):
+ raise RuntimeError(f"taskkill {pid} failed with exit code {completed.returncode}")
return
if os.name != "nt" and pgid is not None:
os.killpg(pgid, sig)
@@ -251,21 +258,50 @@ def _send_tree_signal(pid: int | None, pgid: int | None, sig: signal.Signals | i
os.kill(pid, sig)
+def _signal_target(
+ pid: int | None,
+ pgid: int | None,
+ sig: signal.Signals | int,
+ *,
+ target: str,
+) -> None:
+ """Deliver ``sig`` to a process tree, logging failures instead of
+ swallowing them.
+
+ A target that is already gone (ESRCH on POSIX, taskkill's "process not
+ found" on Windows) has reached the intended end state, so it is not a
+ delivery failure. Any other failure is logged with its cause; the
+ caller's escalation ladder is unchanged.
+ """
+ try:
+ _send_tree_signal(pid, pgid, sig)
+ except ProcessLookupError:
+ return
+ except Exception:
+ logger.warning(
+ "failed to send %s to %s",
+ getattr(sig, "name", sig),
+ target,
+ exc_info=True,
+ )
+
+
def _terminate(proc: ManagedProcess | None) -> None:
+ """Stop ``proc`` with the SIGTERM -> SIGKILL ladder.
+
+ A signal that could not be delivered is logged rather than silently
+ dropped, so a surviving child can be traced back to its failed
+ termination.
+ """
if proc is None or proc.process.poll() is not None:
return
_log(_t("start.stopping", name=proc.name, pid=proc.process.pid))
- try:
- _send_tree_signal(proc.process.pid, proc.pgid, signal.SIGTERM)
- except Exception:
- pass
+ target = f"{proc.name} (pid={proc.process.pid})"
+ _signal_target(proc.process.pid, proc.pgid, signal.SIGTERM, target=target)
try:
proc.process.wait(timeout=8)
except subprocess.TimeoutExpired:
- try:
- _send_tree_signal(proc.process.pid, proc.pgid, KILL_SIGNAL)
- except Exception:
- pass
+ _signal_target(proc.process.pid, proc.pgid, KILL_SIGNAL, target=target)
def _relax_console_encoding(streams: tuple[object, ...] | None = None) -> None:
@@ -497,22 +533,24 @@ def _prompt_new_ports(
def _kill_port_listeners(listeners: dict[int, list[tuple[int, str]]]) -> None:
+ """Signal every port listener with the SIGTERM -> SIGKILL ladder.
+
+ One unreachable listener does not stop the others from being signalled,
+ and each failed delivery leaves a warning in the log instead of a silent
+ zombie.
+ """
for port, entries in listeners.items():
for pid, command in entries:
_log(_t("start.port_killing", pid=pid, command=command))
- try:
- _send_tree_signal(pid, None, signal.SIGTERM)
- except Exception:
- pass
+ target = f"port {port} listener (pid={pid})"
+ _signal_target(pid, None, signal.SIGTERM, target=target)
deadline = time.monotonic() + 5
while time.monotonic() < deadline and _port_accepts_connection(port):
time.sleep(0.2)
if _port_accepts_connection(port):
for pid, _command in entries:
- try:
- _send_tree_signal(pid, None, KILL_SIGNAL)
- except Exception:
- pass
+ target = f"port {port} listener (pid={pid})"
+ _signal_target(pid, None, KILL_SIGNAL, target=target)
deadline = time.monotonic() + 3
while time.monotonic() < deadline and _port_accepts_connection(port):
time.sleep(0.2)
diff --git a/tests/runtime/test_launcher.py b/tests/runtime/test_launcher.py
index 23729cabc..f95eec796 100644
--- a/tests/runtime/test_launcher.py
+++ b/tests/runtime/test_launcher.py
@@ -2,7 +2,10 @@ from __future__ import annotations
import builtins
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import logging
from pathlib import Path
+import signal
+import subprocess
from threading import Thread
from types import SimpleNamespace
@@ -927,3 +930,181 @@ def test_port_listeners_tolerate_missing_stdout(monkeypatch, platform: str) -> N
launcher.subprocess, "run", lambda *args, **kwargs: SimpleNamespace(stdout=None)
)
assert launcher._port_listeners(3782) == []
+
+
+class _StubbornPopen:
+ """Popen stand-in that stays alive until a fake signal lets it exit."""
+
+ def __init__(self, *, pid: int = 4242) -> None:
+ self.pid = pid
+ self.exited = False
+
+ def poll(self) -> int | None:
+ return 0 if self.exited else None
+
+ def wait(self, timeout: float | None = None) -> int:
+ if not self.exited:
+ raise subprocess.TimeoutExpired(cmd="backend", timeout=timeout)
+ return 0
+
+
+def test_terminate_warns_when_both_signal_rounds_fail(
+ monkeypatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """A child no signal reaches must surface every failed round."""
+ child = _StubbornPopen()
+ proc = launcher.ManagedProcess(name="backend", process=child, pgid=4242)
+ sent: list[int] = []
+
+ def fake_send(pid, pgid, sig):
+ sent.append(sig)
+ # Neither round is delivered: SIGTERM is refused, SIGKILL vanishes.
+ raise PermissionError("operation not permitted")
+
+ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):
+ launcher._terminate(proc)
+
+ assert sent == [signal.SIGTERM, launcher.KILL_SIGNAL]
+ warnings = [record.getMessage() for record in caplog.records]
+ assert any("SIGTERM" in message for message in warnings)
+ assert any("SIGKILL" in message for message in warnings)
+
+
+def test_terminate_warns_on_sigterm_but_accepts_kill_escalation(
+ monkeypatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """The ladder keeps its shape: refused SIGTERM, working SIGKILL, done."""
+ child = _StubbornPopen()
+ proc = launcher.ManagedProcess(name="backend", process=child, pgid=4242)
+
+ def fake_send(pid, pgid, sig):
+ if sig == launcher.KILL_SIGNAL:
+ child.exited = True
+ return
+ raise PermissionError("operation not permitted")
+
+ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):
+ launcher._terminate(proc)
+
+ assert child.exited is True
+ warnings = [record.getMessage() for record in caplog.records]
+ assert any("SIGTERM" in message for message in warnings)
+ assert not any("SIGKILL" in message for message in warnings)
+
+
+def test_terminate_skips_absent_or_exited_child(
+ monkeypatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """Nothing to stop means no signal and no warning."""
+ sent: list[int] = []
+
+ def fake_send(pid, pgid, sig):
+ sent.append(sig)
+
+ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)
+ exited = _StubbornPopen()
+ exited.exited = True
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):
+ launcher._terminate(None)
+ launcher._terminate(launcher.ManagedProcess(name="web", process=exited, pgid=None))
+
+ assert sent == []
+ assert caplog.records == []
+
+
+def test_signal_target_treats_already_gone_target_as_terminated(
+ monkeypatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """ESRCH means the target exited on its own; that is not a failure."""
+
+ def fake_send(pid, pgid, sig):
+ raise ProcessLookupError()
+
+ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):
+ launcher._signal_target(123, None, signal.SIGTERM, target="port 8000 listener")
+
+ assert caplog.records == []
+
+
+def test_windows_taskkill_failures_surface_but_missing_processes_do_not(
+ monkeypatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """A failing taskkill must be logged; one that found no process must not."""
+ monkeypatch.setattr(launcher, "os", SimpleNamespace(name="nt"))
+ monkeypatch.setattr(launcher, "_no_window_kwargs", lambda: {})
+ codes: list[int] = []
+ monkeypatch.setattr(
+ launcher.subprocess,
+ "run",
+ lambda *args, **kwargs: SimpleNamespace(returncode=codes.pop(0)),
+ )
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):
+ for code in (1, 0, 128):
+ codes.append(code)
+ launcher._signal_target(7, None, signal.SIGTERM, target="backend (pid=7)")
+
+ warnings = [record.getMessage() for record in caplog.records]
+ assert len(warnings) == 1
+ assert "SIGTERM" in warnings[0]
+ assert "taskkill" in (caplog.records[0].exc_text or "")
+
+
+def test_kill_port_listeners_isolates_one_targets_sigterm_failure(
+ monkeypatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """One unreachable listener must not spare the others or hide itself."""
+ occupied = {8000}
+
+ def fake_send(pid, pgid, sig):
+ if pid == 123:
+ raise PermissionError("operation not permitted")
+ occupied.discard(8000)
+
+ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)
+ monkeypatch.setattr(launcher, "_port_accepts_connection", lambda port: port in occupied)
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):
+ launcher._kill_port_listeners({8000: [(123, "python uvicorn"), (456, "node")]})
+
+ assert occupied == set()
+ assert any("123" in record.getMessage() for record in caplog.records)
+
+
+def test_kill_port_listeners_reports_kill_round_failures_per_target(
+ monkeypatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """SIGTERM-ignoring listeners escalate; KILL failures stay per-target."""
+ occupied = {8000}
+ sent: list[tuple[int, int]] = []
+ clock = {"now": 0.0}
+
+ def fake_send(pid, pgid, sig):
+ sent.append((pid, sig))
+ if sig == launcher.KILL_SIGNAL and pid == 123:
+ raise OSError("no such process")
+ # pid 456 accepts both signals; zombie 123 keeps holding the port.
+
+ def fake_sleep(seconds: float) -> None:
+ clock["now"] += seconds
+
+ monkeypatch.setattr(launcher, "_send_tree_signal", fake_send)
+ monkeypatch.setattr(launcher, "_port_accepts_connection", lambda port: port in occupied)
+ monkeypatch.setattr(launcher.time, "monotonic", lambda: clock["now"])
+ monkeypatch.setattr(launcher.time, "sleep", fake_sleep)
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.runtime.launcher"):
+ launcher._kill_port_listeners({8000: [(123, "python uvicorn"), (456, "node")]})
+
+ assert (123, signal.SIGTERM) in sent
+ assert (456, signal.SIGTERM) in sent
+ assert (123, launcher.KILL_SIGNAL) in sent
+ assert (456, launcher.KILL_SIGNAL) in sent
+ assert any("SIGKILL" in record.getMessage() for record in caplog.records)