Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix: contain Windows Bird/X subprocess trees in job objects (#1245)

Ilia Alshanetsky4e36bf765db1
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
index d3d175f8..0da5c13a 100644
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@@ -28,6 +28,27 @@ jobs:
- name: Run test suite
run: uv run pytest --cov --cov-report=term-missing
+ windows-process-tree:
+ runs-on: windows-latest
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+
+ - name: Set up Python
+ run: uv python install 3.12
+
+ - name: Run Windows subprocess tests
+ env:
+ PYTHONPATH: ${{ github.workspace }}/skills/last30days/scripts
+ run: uv run python -m unittest tests.test_subproc -v
+
eval:
runs-on: ubuntu-latest
permissions:
diff --git a/changelog.d/823.fixed.md b/changelog.d/823.fixed.md
new file mode 100644
index 00000000..ee24e47a
--- /dev/null
+++ b/changelog.d/823.fixed.md
@@ -0,0 +1 @@
+Fixed Windows Bird/X timeouts that left Node shim descendants running after the shim or intermediate processes exited.
diff --git a/skills/last30days/scripts/lib/subproc.py b/skills/last30days/scripts/lib/subproc.py
index a2d693e1..0480de74 100644
--- a/skills/last30days/scripts/lib/subproc.py
+++ b/skills/last30days/scripts/lib/subproc.py
@@ -1,8 +1,7 @@
-"""Subprocess helpers: safe timeout + process-group cleanup.
+"""Subprocess helpers: safe timeout + process-tree cleanup.
-Used by bird_x.py (Node.js Bird search) and youtube_yt.py (yt-dlp search
-and transcript download). Both need the same os.setsid/killpg cleanup
-dance on timeout to avoid orphaning child processes.
+Used by Bird/X search and other external-source clients. POSIX children use
+their own process group; Windows children run inside a kill-on-close job.
"""
from __future__ import annotations
@@ -37,10 +36,12 @@ class SubprocTimeout(Exception):
# lock in register_child_pid.
_child_pids: set[int] = set()
_child_pids_lock = threading.RLock()
+_child_jobs: dict[int, object] = {}
# Set once cleanup_children starts. Worker threads keep running while the
# handler sleeps through the grace, so a source can spawn a child after the
# snapshot; register_child_pid kills such late children itself.
_shutting_down = False
+_WINDOWS = os.name == "nt"
def register_child_pid(pid: int) -> None:
@@ -54,6 +55,9 @@ def register_child_pid(pid: int) -> None:
def unregister_child_pid(pid: int) -> None:
with _child_pids_lock:
_child_pids.discard(pid)
+ job = _child_jobs.pop(pid, None)
+ if job is not None:
+ job.close()
# Upper bound on how long cleanup_children waits for SIGTERMed groups before
@@ -74,10 +78,40 @@ def _signal_group(pgid: int, sig: int) -> bool:
return True
+def _taskkill_pid(pid: int, timeout: float = 2) -> bool:
+ try:
+ result = subprocess.run(
+ ["taskkill", "/F", "/T", "/PID", str(pid)],
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ timeout=timeout,
+ check=False,
+ creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
+ )
+ except (OSError, subprocess.TimeoutExpired):
+ return False
+ return result.returncode == 0
+
+
+def _kill_windows_tree(pid: int) -> bool:
+ return _taskkill_pid(pid)
+
+
+def _kill_windows_managed_tree(pid: int) -> bool:
+ with _child_pids_lock:
+ job = _child_jobs.get(pid)
+ if job is not None and job.terminate():
+ return True
+ return _kill_windows_tree(pid)
+
+
def _kill_child_group(pid: int, sig: int) -> None:
if hasattr(os, "setsid") and hasattr(os, "killpg"):
_signal_group(pid, sig)
return
+ if _WINDOWS and _kill_windows_managed_tree(pid):
+ return
try:
os.kill(pid, sig)
except (ProcessLookupError, PermissionError, OSError):
@@ -87,10 +121,11 @@ def _kill_child_group(pid: int, sig: int) -> None:
def cleanup_children(grace: float = CLEANUP_TERM_GRACE_SECONDS) -> None:
"""Terminate the process group of every registered child.
- SIGTERM every group, wait up to ``grace`` seconds for the groups to
- empty, then SIGKILL the survivors. run_with_timeout starts each child
- with os.setsid, so the child's pid is its pgid; signalling the pgid
- directly still reaches grandchildren after the leader has been reaped,
+ Windows jobs terminate immediately. On POSIX, SIGTERM every group, wait
+ up to ``grace`` seconds for the groups to empty, then SIGKILL survivors.
+ run_with_timeout starts each POSIX child with os.setsid, so the child's
+ pid is its pgid; signalling the pgid directly still reaches grandchildren
+ after the leader has been reaped,
and the kernel does not reuse a pid while a group of that id has
members. A group whose only member is an unreaped zombie still reads as
live, which at worst costs the full grace and a harmless SIGKILL.
@@ -105,7 +140,9 @@ def cleanup_children(grace: float = CLEANUP_TERM_GRACE_SECONDS) -> None:
return
if not (hasattr(os, "setsid") and hasattr(os, "killpg")):
for pid in pids:
- _kill_child_group(pid, signal.SIGTERM)
+ with _child_pids_lock:
+ if pid in _child_pids:
+ _kill_child_group(pid, signal.SIGTERM)
return
live = [pgid for pgid in pids if _signal_group(pgid, signal.SIGTERM)]
deadline = time.monotonic() + grace
@@ -140,10 +177,11 @@ def run_with_timeout(
"""Run a subprocess with process-group cleanup on timeout.
Spawns ``cmd`` inside its own process group via ``start_new_session`` where
- available. If ``communicate(timeout=...)`` raises ``TimeoutExpired``,
- signals ``SIGTERM`` to the entire group, falls back to ``proc.kill()``
- if the signal fails, then waits up to ``cleanup_grace`` seconds before
- escalating to SIGKILL and waiting once more to reap the child.
+ available. On Windows, the child enters a kill-on-close job before it runs,
+ so cleanup reaches descendants that inherit the job after launchers exit.
+ Closing the job on normal return also stops detached descendants. Job setup
+ failure aborts the suspended child. POSIX signals the group with SIGTERM,
+ then escalates to SIGKILL if needed.
Args:
cmd: Command and arguments to spawn.
@@ -175,10 +213,16 @@ def run_with_timeout(
if capture_limit_bytes is not None and capture_limit_bytes < 0:
raise ValueError("capture limit must be nonnegative")
own_group = hasattr(os, "setsid") and hasattr(os, "killpg")
+ native_windows = _WINDOWS
capture = ExitStack()
+ job = None
try:
stdout_file = capture.enter_context(tempfile.TemporaryFile()) if capture_limit_bytes is not None else None
stderr_file = capture.enter_context(tempfile.TemporaryFile()) if capture_limit_bytes is not None else None
+ if native_windows:
+ from .windows_job import CREATE_SUSPENDED, WindowsJob
+
+ job = WindowsJob()
proc = subprocess.Popen(
list(cmd),
stdout=stdout_file if stdout_file is not None else subprocess.PIPE,
@@ -189,14 +233,47 @@ def run_with_timeout(
errors="replace",
start_new_session=own_group,
env=env,
+ **({"creationflags": CREATE_SUSPENDED} if native_windows else {}),
)
except BaseException as exc:
if isinstance(exc, OSError):
exc._last30days_subproc_launch_failed = True
+ if job is not None:
+ job.close()
capture.close()
raise
try:
- register_child_pid(proc.pid)
+ if native_windows:
+ from .windows_job import resume_suspended_process
+
+ try:
+ job.assign(proc._handle)
+ with _child_pids_lock:
+ _child_pids.add(proc.pid)
+ _child_jobs[proc.pid] = job
+ late = _shutting_down
+ if late:
+ if not job.terminate():
+ proc.kill()
+ else:
+ resume_suspended_process(proc.pid)
+ if late:
+ raise SubprocTimeout("Command cancelled during shutdown")
+ except BaseException as exc:
+ if isinstance(exc, OSError):
+ exc._last30days_subproc_launch_failed = True
+ if proc.poll() is None:
+ try:
+ proc.kill()
+ except OSError:
+ pass
+ try:
+ proc.wait(timeout=min(cleanup_grace, 2))
+ except subprocess.TimeoutExpired:
+ pass
+ raise
+ else:
+ register_child_pid(proc.pid)
if on_pid is not None:
try:
on_pid(proc.pid)
@@ -231,7 +308,7 @@ def run_with_timeout(
if own_group:
pgid = proc.pid
os.killpg(pgid, signal.SIGTERM)
- else:
+ elif not (_WINDOWS and _kill_windows_managed_tree(proc.pid)):
proc.kill()
except (ProcessLookupError, PermissionError, OSError, AttributeError):
pgid = None
@@ -243,7 +320,7 @@ def run_with_timeout(
try:
if pgid is not None:
os.killpg(pgid, signal.SIGKILL)
- else:
+ elif not (_WINDOWS and _kill_windows_managed_tree(proc.pid)):
proc.kill()
except (ProcessLookupError, PermissionError, OSError, AttributeError):
proc.kill()
@@ -279,6 +356,8 @@ def run_with_timeout(
raise
finally:
unregister_child_pid(proc.pid)
+ if job is not None:
+ job.close()
for pipe in (proc.stdin, proc.stdout, proc.stderr):
if pipe is not None:
pipe.close()
diff --git a/skills/last30days/scripts/lib/windows_job.py b/skills/last30days/scripts/lib/windows_job.py
new file mode 100644
index 00000000..68d58b80
--- /dev/null
+++ b/skills/last30days/scripts/lib/windows_job.py
@@ -0,0 +1,138 @@
+"""Windows process jobs used to retain descendants after launcher processes exit."""
+
+from __future__ import annotations
+
+import ctypes
+from ctypes import wintypes
+
+CREATE_SUSPENDED = 0x4
+TH32CS_SNAPTHREAD = 0x4
+THREAD_SUSPEND_RESUME = 0x2
+JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
+JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
+
+
+class _BasicLimit(ctypes.Structure):
+ _fields_ = [
+ ("per_process_time", ctypes.c_int64),
+ ("per_job_time", ctypes.c_int64),
+ ("flags", wintypes.DWORD),
+ ("min_working_set", ctypes.c_size_t),
+ ("max_working_set", ctypes.c_size_t),
+ ("active_processes", wintypes.DWORD),
+ ("affinity", ctypes.c_size_t),
+ ("priority", wintypes.DWORD),
+ ("scheduling", wintypes.DWORD),
+ ]
+
+
+class _IoCounters(ctypes.Structure):
+ _fields_ = [(name, ctypes.c_uint64) for name in (
+ "read_operations", "write_operations", "other_operations",
+ "read_bytes", "write_bytes", "other_bytes",
+ )]
+
+
+class _ExtendedLimit(ctypes.Structure):
+ _fields_ = [
+ ("basic", _BasicLimit),
+ ("io", _IoCounters),
+ ("process_memory", ctypes.c_size_t),
+ ("job_memory", ctypes.c_size_t),
+ ("peak_process_memory", ctypes.c_size_t),
+ ("peak_job_memory", ctypes.c_size_t),
+ ]
+
+
+class _ThreadEntry(ctypes.Structure):
+ _fields_ = [
+ ("size", wintypes.DWORD),
+ ("usage", wintypes.DWORD),
+ ("thread_id", wintypes.DWORD),
+ ("owner_pid", wintypes.DWORD),
+ ("base_priority", wintypes.LONG),
+ ("delta_priority", wintypes.LONG),
+ ("flags", wintypes.DWORD),
+ ]
+
+
+def _kernel32():
+ kernel = ctypes.WinDLL("kernel32", use_last_error=True)
+ kernel.CreateJobObjectW.argtypes = (ctypes.c_void_p, wintypes.LPCWSTR)
+ kernel.CreateJobObjectW.restype = wintypes.HANDLE
+ kernel.SetInformationJobObject.argtypes = (wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD)
+ kernel.SetInformationJobObject.restype = wintypes.BOOL
+ kernel.AssignProcessToJobObject.argtypes = (wintypes.HANDLE, wintypes.HANDLE)
+ kernel.AssignProcessToJobObject.restype = wintypes.BOOL
+ kernel.TerminateJobObject.argtypes = (wintypes.HANDLE, wintypes.UINT)
+ kernel.TerminateJobObject.restype = wintypes.BOOL
+ kernel.CreateToolhelp32Snapshot.argtypes = (wintypes.DWORD, wintypes.DWORD)
+ kernel.CreateToolhelp32Snapshot.restype = wintypes.HANDLE
+ kernel.Thread32First.argtypes = (wintypes.HANDLE, ctypes.POINTER(_ThreadEntry))
+ kernel.Thread32First.restype = wintypes.BOOL
+ kernel.Thread32Next.argtypes = (wintypes.HANDLE, ctypes.POINTER(_ThreadEntry))
+ kernel.Thread32Next.restype = wintypes.BOOL
+ kernel.OpenThread.argtypes = (wintypes.DWORD, wintypes.BOOL, wintypes.DWORD)
+ kernel.OpenThread.restype = wintypes.HANDLE
+ kernel.ResumeThread.argtypes = (wintypes.HANDLE,)
+ kernel.ResumeThread.restype = wintypes.DWORD
+ kernel.CloseHandle.argtypes = (wintypes.HANDLE,)
+ kernel.CloseHandle.restype = wintypes.BOOL
+ return kernel
+
+
+class WindowsJob:
+ def __init__(self):
+ self.kernel = _kernel32()
+ self.handle = self.kernel.CreateJobObjectW(None, None)
+ if not self.handle:
+ raise ctypes.WinError(ctypes.get_last_error())
+ limits = _ExtendedLimit()
+ limits.basic.flags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
+ if not self.kernel.SetInformationJobObject(
+ self.handle, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(limits), ctypes.sizeof(limits)
+ ):
+ error = ctypes.get_last_error()
+ self.close()
+ raise ctypes.WinError(error)
+
+ def assign(self, process_handle: int) -> None:
+ if not self.kernel.AssignProcessToJobObject(self.handle, process_handle):
+ raise ctypes.WinError(ctypes.get_last_error())
+
+ def terminate(self) -> bool:
+ return bool(self.handle and self.kernel.TerminateJobObject(self.handle, 1))
+
+ def close(self) -> None:
+ if self.handle:
+ self.kernel.CloseHandle(self.handle)
+ self.handle = None
+
+
+def resume_suspended_process(pid: int) -> None:
+ kernel = _kernel32()
+ snapshot = kernel.CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0)
+ if not snapshot or snapshot == ctypes.c_void_p(-1).value:
+ raise ctypes.WinError(ctypes.get_last_error())
+ try:
+ entry = _ThreadEntry()
+ entry.size = ctypes.sizeof(entry)
+ found = kernel.Thread32First(snapshot, ctypes.byref(entry))
+ while found:
+ if entry.owner_pid == pid:
+ thread = kernel.OpenThread(THREAD_SUSPEND_RESUME, False, entry.thread_id)
+ if not thread:
+ raise ctypes.WinError(ctypes.get_last_error())
+ try:
+ previous = kernel.ResumeThread(thread)
+ if previous == 0xFFFFFFFF:
+ raise ctypes.WinError(ctypes.get_last_error())
+ if previous != 1:
+ raise OSError(f"Unexpected suspend count {previous} for process {pid}")
+ return
+ finally:
+ kernel.CloseHandle(thread)
+ found = kernel.Thread32Next(snapshot, ctypes.byref(entry))
+ raise OSError(f"No primary thread found for suspended process {pid}")
+ finally:
+ kernel.CloseHandle(snapshot)
diff --git a/tests/test_subproc.py b/tests/test_subproc.py
index f2106c23..b5d69bde 100644
--- a/tests/test_subproc.py
+++ b/tests/test_subproc.py
@@ -8,8 +8,9 @@ import builtins
import errno
import os as real_os
import platform
+import sys
import unittest
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
from lib import subproc
@@ -18,11 +19,11 @@ IS_WINDOWS = platform.system() == "Windows"
def get_shell_cmd(cmd_str: str) -> list[str]:
if IS_WINDOWS:
if cmd_str == "echo hello":
- return ["cmd", "/c", "echo hello"]
+ return [sys.executable, "-c", "print('hello')"]
elif cmd_str == "exit 3":
return ["cmd", "/c", "exit 3"]
elif cmd_str == "echo err >&2":
- return ["cmd", "/c", "echo err 1>&2"]
+ return [sys.executable, "-c", "import sys;print('err', file=sys.stderr)"]
elif cmd_str in ("sleep 10", "sleep 10 & wait"):
return ["powershell", "-Command", "Start-Sleep 10"]
elif cmd_str == "echo $LAST30DAYS_TEST_VAR":
@@ -205,20 +206,30 @@ time.sleep(15)
def test_child_pid_registered_during_run_and_cleared_after(self):
"""Every run_with_timeout child is in the cleanup registry while alive."""
seen = []
+ observed_registered = []
+ observed_job = []
+
+ def observe(pid):
+ seen.append(pid)
+ observed_registered.append(pid in subproc._child_pids)
+ if IS_WINDOWS:
+ observed_job.append(pid in subproc._child_jobs)
+
with patch.object(
- subproc, "register_child_pid", wraps=subproc.register_child_pid
- ) as reg, patch.object(
subproc, "unregister_child_pid", wraps=subproc.unregister_child_pid
) as unreg:
result = subproc.run_with_timeout(
get_shell_cmd("echo ok"),
timeout=5,
- on_pid=seen.append,
+ on_pid=observe,
)
self.assertEqual(result.stdout.strip(), "ok")
- reg.assert_called_once_with(seen[0])
+ self.assertEqual(observed_registered, [True])
+ if IS_WINDOWS:
+ self.assertEqual(observed_job, [True])
unreg.assert_called_once_with(seen[0])
self.assertEqual(subproc._child_pids, set())
+ self.assertEqual(subproc._child_jobs, {})
def test_timed_out_child_is_unregistered(self):
with self.assertRaises(subproc.SubprocTimeout):
@@ -366,8 +377,9 @@ time.sleep(15)
]
self.assertEqual(offenders, [])
+ @unittest.skipIf(IS_WINDOWS, "POSIX fallback simulation uses sh")
def test_timeout_falls_back_to_kill_when_killpg_unavailable(self):
- """Simulate Windows (no killpg/getpgid) — should fall back to proc.kill()."""
+ """Without POSIX groups or Windows tree cleanup, kill the direct child."""
real_hasattr = builtins.hasattr
def selective_hasattr(obj, name):
@@ -382,6 +394,387 @@ time.sleep(15)
timeout=1,
)
+ def test_windows_timeout_terminates_owned_process_tree(self):
+ timeout_error = subproc.subprocess.TimeoutExpired("node", 1)
+
+ class FakeProc:
+ pid = 4321
+ _handle = 9876
+ stdin = stdout = stderr = None
+ returncode = None
+
+ def __init__(self):
+ self.killed = False
+
+ def communicate(self, **kwargs):
+ raise timeout_error
+
+ def wait(self, timeout=None):
+ self.returncode = 1
+ return 1
+
+ def poll(self):
+ return self.returncode
+
+ def kill(self):
+ self.killed = True
+
+ fake = FakeProc()
+ real_hasattr = builtins.hasattr
+
+ def windows_hasattr(obj, name):
+ if obj is real_os and name in ("setsid", "killpg"):
+ return False
+ return real_hasattr(obj, name)
+
+ job = MagicMock()
+ job.terminate.return_value = True
+ with patch.object(builtins, "hasattr", side_effect=windows_hasattr), \
+ patch.object(subproc, "_WINDOWS", True), \
+ patch("lib.windows_job.WindowsJob", return_value=job), \
+ patch("lib.windows_job.resume_suspended_process") as resume, \
+ patch.object(subproc, "_kill_windows_tree") as kill_tree, \
+ patch.object(subproc.subprocess, "Popen", return_value=fake):
+ with self.assertRaises(subproc.SubprocTimeout):
+ subproc.run_with_timeout(["node", "bird-search.mjs"], timeout=1)
+
+ job.assign.assert_called_once_with(fake._handle)
+ resume.assert_called_once_with(fake.pid)
+ job.terminate.assert_called_once_with()
+ kill_tree.assert_not_called()
+ self.assertFalse(fake.killed)
+
+ def test_windows_tree_kill_uses_pid_scoped_taskkill_and_handles_failure(self):
+ import subprocess
+
+ with patch.object(subproc.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run:
+ self.assertTrue(subproc._kill_windows_tree(4321))
+ run.assert_called_once_with(
+ ["taskkill", "/F", "/T", "/PID", "4321"],
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ timeout=2,
+ check=False,
+ creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
+ )
+
+ with patch.object(subproc.subprocess, "run", side_effect=subprocess.TimeoutExpired("taskkill", 2)):
+ self.assertFalse(subproc._kill_windows_tree(4321))
+
+ with patch.object(subproc.subprocess, "run", return_value=subprocess.CompletedProcess([], 1)):
+ self.assertFalse(subproc._kill_windows_tree(4321))
+
+ def test_windows_shutdown_terminates_owned_process_tree(self):
+ real_hasattr = builtins.hasattr
+
+ def windows_hasattr(obj, name):
+ if obj is real_os and name in ("setsid", "killpg"):
+ return False
+ return real_hasattr(obj, name)
+
+ with patch.object(builtins, "hasattr", side_effect=windows_hasattr), \
+ patch.object(subproc, "_WINDOWS", True), \
+ patch.object(subproc, "_child_pids", {4321}), \
+ patch.object(subproc, "_shutting_down", False), \
+ patch.object(subproc, "_kill_windows_tree", return_value=True) as kill_tree:
+ subproc.cleanup_children()
+
+ kill_tree.assert_called_once_with(4321)
+
+ def test_windows_tree_kill_failure_falls_back_to_direct_child(self):
+ timeout_error = subproc.subprocess.TimeoutExpired("node", 1)
+
+ class FakeProc:
+ pid = 4321
+ _handle = 9876
+ stdin = stdout = stderr = None
+ returncode = None
+
+ def __init__(self):
+ self.killed = False
+
+ def communicate(self, **kwargs):
+ raise timeout_error
+
+ def wait(self, timeout=None):
+ self.returncode = 1
+ return 1
+
+ def poll(self):
+ return self.returncode
+
+ def kill(self):
+ self.killed = True
+
+ fake = FakeProc()
+ real_hasattr = builtins.hasattr
+
+ def windows_hasattr(obj, name):
+ if obj is real_os and name in ("setsid", "killpg"):
+ return False
+ return real_hasattr(obj, name)
+
+ job = MagicMock()
+ job.terminate.return_value = False
+ with patch.object(builtins, "hasattr", side_effect=windows_hasattr), \
+ patch.object(subproc, "_WINDOWS", True), \
+ patch("lib.windows_job.WindowsJob", return_value=job), \
+ patch("lib.windows_job.resume_suspended_process") as resume, \
+ patch.object(subproc, "_kill_windows_tree", return_value=False) as kill_tree, \
+ patch.object(subproc.subprocess, "Popen", return_value=fake):
+ with self.assertRaises(subproc.SubprocTimeout):
+ subproc.run_with_timeout(["node", "bird-search.mjs"], timeout=1)
+
+ job.assign.assert_called_once_with(fake._handle)
+ resume.assert_called_once_with(fake.pid)
+ job.terminate.assert_called_once_with()
+ kill_tree.assert_called_once_with(fake.pid)
+ self.assertTrue(fake.killed)
+
+ @unittest.skipUnless(IS_WINDOWS, "native Windows process-tree validation")
+ def test_windows_timeout_kills_shim_grandchild(self):
+ self._assert_windows_timeout_kills_grandchild(shim_exits=False)
+
+ @unittest.skipUnless(IS_WINDOWS, "native Windows process-tree validation")
+ def test_windows_timeout_kills_grandchild_after_shim_exits(self):
+ self._assert_windows_timeout_kills_grandchild(shim_exits=True)
+
+ @unittest.skipUnless(IS_WINDOWS, "native Windows process-tree validation")
+ def test_windows_timeout_kills_grandchild_after_two_ancestors_exit(self):
+ self._assert_windows_timeout_kills_grandchild(shim_exits=True, intermediate_exits=True)
+
+ @unittest.skipUnless(IS_WINDOWS, "native Windows process-tree validation")
+ def test_windows_success_reaps_detached_grandchild(self):
+ self._assert_windows_timeout_kills_grandchild(shim_exits=True, normal_exit=True)
+
+ @unittest.skipUnless(IS_WINDOWS, "native Windows process-tree validation")
+ def test_windows_job_assignment_failure_never_starts_child(self):
+ import pathlib
+ import sys
+ import tempfile
+
+ from lib.windows_job import WindowsJob
+
+ spawned = []
+ real_popen = subproc.subprocess.Popen
+
+ def record_spawn(*args, **kwargs):
+ proc = real_popen(*args, **kwargs)
+ spawned.append(proc)
+ return proc
+
+ with tempfile.TemporaryDirectory() as tmp:
+ marker = pathlib.Path(tmp, "started")
+ try:
+ with patch.object(WindowsJob, "assign", side_effect=OSError("job assignment refused")), \
+ patch.object(subproc.subprocess, "Popen", side_effect=record_spawn):
+ with self.assertRaisesRegex(OSError, "job assignment refused"):
+ subproc.run_with_timeout(
+ [sys.executable, "-c", "import pathlib,sys;pathlib.Path(sys.argv[1]).touch()", str(marker)],
+ timeout=2,
+ )
+ self.assertEqual(len(spawned), 1)
+ self.assertIsNotNone(spawned[0].poll(), "suspended child survived assignment failure")
+ self.assertFalse(marker.exists(), "child ran without a process job")
+ self.assertNotIn(spawned[0].pid, subproc._child_pids)
+ self.assertNotIn(spawned[0].pid, subproc._child_jobs)
+ finally:
+ for proc in spawned:
+ if proc.poll() is None:
+ proc.kill()
+ proc.wait(timeout=2)
+
+ @unittest.skipUnless(IS_WINDOWS, "native Windows process-tree validation")
+ def test_windows_shutdown_during_registration_never_starts_child(self):
+ import pathlib
+ import sys
+ import tempfile
+
+ from lib.windows_job import WindowsJob
+
+ spawned = []
+ real_popen = subproc.subprocess.Popen
+ real_assign = WindowsJob.assign
+
+ def record_spawn(*args, **kwargs):
+ proc = real_popen(*args, **kwargs)
+ spawned.append(proc)
+ return proc
+
+ def shut_down_before_registration(job, handle):
+ real_assign(job, handle)
+ subproc._shutting_down = True
+
+ with tempfile.TemporaryDirectory() as tmp:
+ marker = pathlib.Path(tmp, "started")
+ try:
+ with patch.object(WindowsJob, "assign", shut_down_before_registration), \
+ patch.object(subproc.subprocess, "Popen", side_effect=record_spawn):
+ with self.assertRaises(subproc.SubprocTimeout):
+ subproc.run_with_timeout(
+ [sys.executable, "-c", "import pathlib,sys;pathlib.Path(sys.argv[1]).touch()", str(marker)],
+ timeout=2,
+ )
+ self.assertEqual(len(spawned), 1)
+ self.assertIsNotNone(spawned[0].poll(), "suspended child survived shutdown")
+ self.assertFalse(marker.exists(), "child ran after shutdown began")
+ self.assertNotIn(spawned[0].pid, subproc._child_pids)
+ self.assertNotIn(spawned[0].pid, subproc._child_jobs)
+ finally:
+ for proc in spawned:
+ if proc.poll() is None:
+ proc.kill()
+ proc.wait(timeout=2)
+
+ def _assert_windows_timeout_kills_grandchild(self, *, shim_exits, intermediate_exits=False, normal_exit=False):
+ import ctypes
+ import pathlib
+ import signal
+ import sys
+ import tempfile
+ import threading
+ import time
+
+ kernel = ctypes.WinDLL("kernel32", use_last_error=True)
+ kernel.OpenProcess.argtypes = (ctypes.c_ulong, ctypes.c_int, ctypes.c_ulong)
+ kernel.OpenProcess.restype = ctypes.c_void_p
+ kernel.GetExitCodeProcess.argtypes = (ctypes.c_void_p, ctypes.POINTER(ctypes.c_ulong))
+ kernel.GetExitCodeProcess.restype = ctypes.c_int
+ kernel.CloseHandle.argtypes = (ctypes.c_void_p,)
+
+ def running(pid):
+ handle = kernel.OpenProcess(0x1000, False, pid)
+ if not handle:
+ return False
+ try:
+ exit_code = ctypes.c_ulong()
+ self.assertTrue(kernel.GetExitCodeProcess(handle, ctypes.byref(exit_code)))
+ return exit_code.value == 259
+ finally:
+ kernel.CloseHandle(handle)
+
+ grandchild = (
+ "import os,pathlib,sys,time;"
+ "pathlib.Path(sys.argv[1]).write_text(str(os.getpid()));"
+ "time.sleep(12)"
+ )
+ shim = (
+ "import subprocess,sys,time;"
+ "subprocess.Popen([sys.executable,'-c',sys.argv[2],sys.argv[1]]"
+ + (",stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL" if normal_exit else "")
+ + ");"
+ + ("" if shim_exits else "time.sleep(30)")
+ )
+ with tempfile.TemporaryDirectory() as tmp:
+ pidfile = pathlib.Path(tmp, "grandchild.pid")
+ intermediate_pidfile = pathlib.Path(tmp, "intermediate.pid")
+ if intermediate_exits:
+ intermediate = (
+ "import os,pathlib,subprocess,sys;"
+ "pathlib.Path(sys.argv[1]).write_text(str(os.getpid()));"
+ "subprocess.Popen([sys.executable,'-c',sys.argv[3],sys.argv[2]])"
+ )
+ shim = (
+ "import subprocess,sys;"
+ "subprocess.Popen([sys.executable,'-c',sys.argv[3],sys.argv[1],sys.argv[2],sys.argv[4]])"
+ )
+ cmd = [
+ sys.executable, "-c", shim, str(intermediate_pidfile),
+ str(pidfile), intermediate, grandchild,
+ ]
+ else:
+ cmd = [sys.executable, "-c", shim, str(pidfile), grandchild]
+ shim_pids = []
+ observed_live = []
+ observed_root_exit = []
+ observed_intermediate_exit = []
+ bystander = (
+ subproc.subprocess.Popen(
+ [sys.executable, "-c", "import time;time.sleep(12)"],
+ stdout=subproc.subprocess.DEVNULL,
+ stderr=subproc.subprocess.DEVNULL,
+ )
+ if shim_exits else None
+ )
+
+ def wait_for_grandchild(pid):
+ shim_pids.append(pid)
+ deadline = time.monotonic() + 5
+ while time.monotonic() < deadline:
+ if pidfile.exists() and pidfile.stat().st_size:
+ observed_live.append(running(int(pidfile.read_text())))
+ if shim_exits:
+ while running(pid) and time.monotonic() < deadline:
+ time.sleep(0.01)
+ observed_root_exit.append(not running(pid))
+ if intermediate_exits:
+ while not intermediate_pidfile.exists() and time.monotonic() < deadline:
+ time.sleep(0.01)
+ if intermediate_pidfile.exists():
+ intermediate_pid = int(intermediate_pidfile.read_text())
+ while running(intermediate_pid) and time.monotonic() < deadline:
+ time.sleep(0.01)
+ observed_intermediate_exit.append(not running(intermediate_pid))
+ return
+ time.sleep(0.01)
+ self.fail("grandchild did not write its PID")
+
+ def kill_owned_processes():
+ grandchild_pid = pidfile.read_text().strip() if pidfile.exists() else ""
+ targets = shim_pids + ([int(grandchild_pid)] if grandchild_pid.isdigit() else [])
+ if intermediate_pidfile.exists():
+ targets.append(int(intermediate_pidfile.read_text()))
+ for pid in targets:
+ if running(pid):
+ try:
+ subproc.subprocess.run(
+ ["taskkill", "/F", "/T", "/PID", str(pid)],
+ stdout=subproc.subprocess.DEVNULL,
+ stderr=subproc.subprocess.DEVNULL,
+ timeout=2,
+ check=False,
+ )
+ except (OSError, subproc.subprocess.TimeoutExpired):
+ pass
+ if running(pid):
+ try:
+ real_os.kill(pid, signal.SIGTERM)
+ except OSError:
+ pass
+
+ watchdog = threading.Timer(10, kill_owned_processes)
+ watchdog.daemon = True
+ watchdog.start()
+ try:
+ started = time.monotonic()
+ if normal_exit:
+ result = subproc.run_with_timeout(cmd, timeout=5, on_pid=wait_for_grandchild)
+ self.assertEqual(result.returncode, 0)
+ else:
+ with self.assertRaises(subproc.SubprocTimeout):
+ subproc.run_with_timeout(cmd, timeout=0.2, on_pid=wait_for_grandchild)
+ self.assertLess(time.monotonic() - started, 8, "cleanup waited for the grandchild to exit")
+ self.assertTrue(shim_pids)
+ self.assertEqual(observed_live, [True], "grandchild was not alive before timeout")
+ if shim_exits:
+ self.assertEqual(observed_root_exit, [True], "shim did not exit before timeout")
+ if intermediate_exits:
+ self.assertEqual(observed_intermediate_exit, [True], "intermediate did not exit before timeout")
+ self.assertTrue(pidfile.exists())
+ shim_pid = shim_pids[0]
+ grandchild_pid = int(pidfile.read_text())
+ self.assertFalse(running(shim_pid), "shim survived its timeout")
+ self.assertFalse(running(grandchild_pid), "grandchild survived its timeout")
+ if bystander is not None:
+ self.assertTrue(running(bystander.pid), "unrelated process was terminated")
+ finally:
+ watchdog.cancel()
+ kill_owned_processes()
+ if bystander is not None and running(bystander.pid):
+ bystander.kill()
+ bystander.wait(timeout=2)
+
def test_on_pid_callback_exceptions_are_suppressed(self):
"""If the PID callback raises, the subprocess should still run to completion."""
def raising_callback(pid):
@@ -396,6 +789,7 @@ time.sleep(15)
self.assertEqual(result.returncode, 0)
self.assertEqual(result.stdout.strip(), "ok")
+ @unittest.skipIf(IS_WINDOWS, "POSIX SIGTERM escalation uses sh")
def test_sigterm_ignoring_child_is_sigkill_escalated(self):
"""A child that ignores SIGTERM must be escalated to SIGKILL.
@@ -409,6 +803,7 @@ time.sleep(15)
timeout=1,
)
+ @unittest.skipIf(IS_WINDOWS, "POSIX killpg simulation requires os.getpgid")
def test_escalation_path_guards_killpg_attributeerror(self):
"""The SIGKILL escalation must not crash if killpg is unavailable (Windows).