Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

test(remote-access): end every process a remote-access test starts, however the test ends (#2489)

Call Whiteda004adb110b
diff --git a/tests/conftest.py b/tests/conftest.py
index a6ea44fb6..6b9a40d8e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -221,6 +221,35 @@ def pytest_runtest_teardown(item):
_reaped_processes.extend(f"{item.nodeid}: {each}" for each in processes.reap())
+def reap_helper_spawns(request, tmp_path):
+ """Reap what the test's helper interpreters spawned, by identity, and fail the test for each one left.
+
+ A helper started with ``process_reaper.record_spawns`` appends each spawn to
+ ``helper_spawn_record(tmp_path)``; once the helper exits those spawns
+ reparent to init, where no record of this pytest process reaches them.
+ Torn down after the test's own fixtures, so their cleanup runs first. A run
+ killed before this teardown leaves them running. An orphan that names
+ ``tmp_path`` but is in no record also fails the test, and is never
+ signalled. POSIX only, like the reaper.
+ """
+
+ yield
+ if not _SIGNAL_GUARD_SUPPORTED:
+ return
+ __tracebackhide__ = True
+ reaped, orphans = process_reaper.end_helper_spawns(
+ process_reaper.helper_spawn_record(tmp_path), tmp_path, request.node.stash.get(_TEST_PROCESSES, None)
+ )
+ problems = []
+ if reaped:
+ problems.append("spawns of this test's helper processes were still running and were killed: " + ", ".join(reaped))
+ if orphans:
+ problems.append("orphans naming this test's tmp_path are in no spawn record and were left running: " + ", ".join(orphans))
+ if problems:
+ pytest.fail("\n".join(problems))
+
+
def _module_uses_default_sqlite_state(request: pytest.FixtureRequest) -> bool:
"""Avoid creating a migration template for tests that never use SQLite state."""
@@ -601,7 +630,7 @@ class _ForeignSignalGuard:
def __init__(self, test_temp: Path) -> None:
self.me = _REAL_OS_GETPID()
# The whole final component: `test_x1` must not claim `test_x10`.
- self.names_test_temp = re.compile(re.escape(str(test_temp)) + r"(?![\w.-])").search
+ self.names_test_temp = process_reaper.names_path(test_temp)
self.owned: set[int] = set()
# Every pid psutil listed while this test ran: a scan finds real processes by design.
self.listed: set[int] = set()
diff --git a/tests/process_reaper.py b/tests/process_reaper.py
index 3b1b0030f..55bf434f3 100644
--- a/tests/process_reaper.py
+++ b/tests/process_reaper.py
@@ -18,8 +18,10 @@ Hub engines, outlived their deleted worktrees by days. Two owners close that.
Opt-in, for the suites whose children are real and long-lived: the job wrapper
tests (``tests/agent_core/tools/test_bash.py``, ``test_jobs.py``,
-``tests/test_watch_job_target.py``) and the mock-engine tests
-(``tests/test_model_hub_runtime.py``, ``test_model_hub_local_error_details.py``).
+``tests/test_watch_job_target.py``), the mock-engine tests
+(``tests/test_model_hub_runtime.py``, ``test_model_hub_local_error_details.py``)
+and the remote-access process tests (``tests/test_tunnel_recovery.py``,
+``tests/test_cloudflared_runtime.py``).
Each starts its children through ``subprocess.Popen`` in a session of their own,
which is what the record covers. A marked test starts no ``multiprocessing``
helper, which serves a whole run and would be reaped with the test.
@@ -36,6 +38,15 @@ its body. Every signal is proven to reach what the test started:
it. A pid reused by any live process makes the record stale. The warden
forgets each test's records once its reap has confirmed them gone.
+A helper interpreter a test starts (a cross-process child) spawns processes
+this pytest process never sees: once the helper exits they reparent to init.
+The helper calls ``record_spawns`` first, so each of its spawns is appended to
+the test's spawn record, and the ``reap_helper_spawns`` fixture reaps them by
+identity after the test's own cleanup. A run killed before that teardown
+leaves them running: the warden reads only this pytest process's own records.
+Nothing is ever killed by name: an orphan that is in no record but names the
+test's ``tmp_path`` is only reported, and fails the test.
+
Reports name a process by pid and executable name only: an argv can carry a
credential.
@@ -45,10 +56,13 @@ POSIX only, like the foreign-signal guard whose ``Popen`` hook records spawns.
from __future__ import annotations
import os
+import re
import signal
import subprocess
import sys
import time
+from collections.abc import Callable
+from pathlib import Path
from typing import NamedTuple
import psutil
@@ -125,6 +139,236 @@ def _describe(pid: int) -> str:
return str(pid)
+def _add_tree(targets: dict[int, float], process: psutil.Process) -> None:
+ try:
+ for each in (process, *process.children(recursive=True)):
+ targets.setdefault(each.pid, each.create_time())
+ except psutil.Error:
+ pass
+
+
+def _kill_and_wait(targets: dict[int, float], groups: set[int]) -> None:
+ """SIGKILL every group, then every target still the same process, and wait until all are gone."""
+ __tracebackhide__ = True
+ for pgid in groups:
+ try:
+ _KILLPG(pgid, signal.SIGKILL)
+ except OSError:
+ pass
+ for pid, created in targets.items():
+ if _same(pid, created) is not None:
+ try:
+ _KILL(pid, signal.SIGKILL)
+ except OSError:
+ pass
+ give_up = time.monotonic() + hang_guard.HANG_GUARD_SECONDS
+ while running := [pid for pid, created in targets.items() if _same(pid, created) is not None]:
+ if time.monotonic() >= give_up:
+ pytest.fail(
+ f"hung for {hang_guard.HANG_GUARD_SECONDS:.0f}s waiting for SIGKILLed processes to end: "
+ + ", ".join(map(_describe, running))
+ )
+ time.sleep(0.01)
+
+
+# A log sink ends once the connector whose output it drains has: what the
+# test's own cleanup stopped gets this long to go before it counts as left behind.
+_SPAWN_EXIT_SECONDS = 5.0
+# SIGSTOP lands asynchronously; a process that has not stopped by then is killed anyway.
+_STOP_SECONDS = 2.0
+
+
+def _stop_all(targets: dict[int, float]) -> dict[int, float]:
+ """SIGSTOP each target that is still the same process, then wait until each has stopped; return those signalled."""
+ stopped: dict[int, float] = {}
+ for pid, created in targets.items():
+ if _same(pid, created) is not None:
+ try:
+ _KILL(pid, signal.SIGSTOP)
+ stopped[pid] = created
+ except OSError:
+ pass
+
+ def settled(pid: int, created: float) -> bool:
+ try:
+ process = psutil.Process(pid)
+ return process.create_time() != created or process.status() in {psutil.STATUS_STOPPED, psutil.STATUS_ZOMBIE}
+ except psutil.Error:
+ return True
+
+ give_up = time.monotonic() + _STOP_SECONDS
+ while not all(settled(pid, created) for pid, created in stopped.items()) and time.monotonic() < give_up:
+ time.sleep(0.005)
+ return stopped
+
+
+def names_path(path: Path) -> Callable[[str], re.Match | None]:
+ """Search for ``path`` itself or a path inside it: the whole final component, so ``test_x1`` never names ``test_x10``."""
+ return re.compile(re.escape(str(path)) + r"(?![\w.-])").search
+
+
+def helper_spawn_record(test_temp: Path) -> Path:
+ """Where a test's helper interpreters record their spawns."""
+ return test_temp / "helper-spawns"
+
+
+def _append_spawn(record: Path, pid: int) -> None:
+ created = _created(pid)
+ if created is None:
+ return
+ try:
+ founded_group = _GETPGID(pid) == pid != _GETPGRP()
+ except (OSError, TypeError):
+ founded_group = False
+ with open(record, "a", encoding="utf-8") as file:
+ file.write(f"{pid} {created!r} {int(founded_group)}\n")
+ file.flush()
+ os.fsync(file.fileno())
+
+
+def record_spawns(record: Path) -> None:
+ """Called first in a helper interpreter a test starts: record it, then each process it spawns, in ``record``.
+
+ One line per process, ``<pid> <creation time> <1 if it founded its group>``,
+ flushed and synced before ``Popen`` returns, so a helper killed right after
+ a spawn has still recorded it. The helper's own line lets teardown end a
+ helper that is still running before it reads the record for the last time.
+ """
+ _append_spawn(record, os.getpid())
+ initialize = subprocess.Popen.__init__
+
+ def recording_init(self, *args, **kwargs):
+ initialize(self, *args, **kwargs)
+ try:
+ _append_spawn(record, self.pid)
+ except BaseException:
+ # The caller never gets this child and no record names it: end it here,
+ # with what it already started. Until wait() reaps it, its pid cannot be
+ # reused, even as a zombie, so the pid is its identity; it founded a group
+ # exactly when these arguments asked for one.
+ targets: dict[int, float] = {}
+ try:
+ _add_tree(targets, psutil.Process(self.pid))
+ except psutil.Error:
+ pass
+ groups = {self.pid} if kwargs.get("start_new_session") or kwargs.get("process_group") == 0 else set()
+ for member in _group_members(groups) if groups else ():
+ _add_tree(targets, member)
+ _kill_and_wait(targets, groups)
+ self.wait()
+ raise
+
+ subprocess.Popen.__init__ = recording_init
+
+
+def _recorded_spawns(record: Path) -> dict[int, _Spawn]:
+ spawns: dict[int, _Spawn] = {}
+ try:
+ lines = record.read_text(encoding="utf-8").splitlines()
+ except FileNotFoundError:
+ return spawns
+ for line in lines:
+ try:
+ pid, created, founded_group = line.split()
+ spawns[int(pid)] = _Spawn(float(created), founded_group == "1")
+ except ValueError:
+ # A line cut short by a helper killed mid-write.
+ continue
+ return spawns
+
+
+def _lineage() -> set[int]:
+ me = psutil.Process()
+ return {me.pid, *(ancestor.pid for ancestor in me.parents())}
+
+
+def orphans_naming(test_temp: Path) -> list[str]:
+ """Describe each orphan (ppid 1) whose executable or argv names ``test_temp``. Never signals one.
+
+ A process with a live parent, such as a developer's ``tail -f``, is not an orphan.
+ """
+ names = names_path(test_temp)
+ lineage = _lineage()
+ found = []
+ for process in psutil.process_iter():
+ if process.pid in lineage:
+ continue
+ try:
+ if process.ppid() != 1 or process.status() == psutil.STATUS_ZOMBIE:
+ continue
+ words = [process.exe(), *process.cmdline()]
+ except psutil.Error:
+ continue
+ if any(word and names(word) for word in words):
+ found.append(_describe(process.pid))
+ return found
+
+
+def end_helper_spawns(
+ record: Path, test_temp: Path, started: TestProcesses | None = None
+) -> tuple[list[str], list[str]]:
+ """After the test's own cleanup, reap what ``record`` names by identity; then list unrecorded orphans.
+
+ ``started`` is the test's own ``reap_processes`` record. Its children
+ include the helpers, recorded from the moment they exist, so a helper that
+ has not yet recorded itself still holds the grace open and is ended before
+ the last read.
+
+ Returns every process that was still running and was killed (a helper, a
+ spawn, a descendant, or a member a recorded group leader left behind) and
+ the orphans still naming ``test_temp``, which get no signal.
+ """
+ __tracebackhide__ = True
+ lineage = _lineage()
+ spawns: dict[int, _Spawn] = dict(started.spawned) if started is not None else {}
+
+ def leftovers() -> tuple[dict[int, float], set[int]]:
+ # A helper still running may append at any time, so every pass reads the record again.
+ spawns.update(_recorded_spawns(record))
+ targets: dict[int, float] = {}
+ for pid, spawn in spawns.items():
+ if (process := _same(pid, spawn.created)) is not None:
+ _add_tree(targets, process)
+ # A group outlives its leader: what a spawned shell put in the background stays in it.
+ groups = {pid for pid, spawn in spawns.items() if spawn.founded_group and _group_is_ours(pid, spawn.created)}
+ for member in _group_members(groups) if groups else ():
+ _add_tree(targets, member)
+ for pid in lineage:
+ targets.pop(pid, None)
+ return targets, groups
+
+ give_up = time.monotonic() + _SPAWN_EXIT_SECONDS
+ while (leftovers()[0] or orphans_naming(test_temp)) and time.monotonic() < give_up:
+ time.sleep(0.05)
+ reaped: list[str] = []
+ while (alive := leftovers()[0]):
+ # Freeze, then reap: a stopped process forks nothing and appends nothing, and
+ # a fork already in flight has completed, so its child is in the next snapshot.
+ # Stopping repeats until the snapshot holds nothing that is not stopped.
+ stopped: dict[int, float] = {}
+ signalled: set[int] = set()
+ try:
+ for _ in range(20):
+ fresh = {pid: created for pid, created in alive.items() if pid not in signalled}
+ if not fresh:
+ break
+ signalled.update(fresh)
+ stopped.update(_stop_all(fresh))
+ alive = leftovers()[0]
+ targets, groups = leftovers()
+ reaped.extend(_describe(pid) for pid, created in targets.items() if _same(pid, created) is not None)
+ _kill_and_wait(targets, groups)
+ finally:
+ # Whatever this pass stopped it also kills, however the pass ended.
+ for pid, created in stopped.items():
+ if _same(pid, created) is not None:
+ try:
+ _KILL(pid, signal.SIGKILL)
+ except OSError:
+ pass
+ return reaped, orphans_naming(test_temp)
+
+
class TestProcesses:
"""One test's spawns, and the descendants this pytest process had before it began."""
@@ -135,6 +379,10 @@ class TestProcesses:
self._before = _descendants()
self._spawned: dict[int, _Spawn] = {}
+ @property
+ def spawned(self) -> dict[int, _Spawn]:
+ return dict(self._spawned)
+
def record(self, pid: int) -> None:
"""Called for every ``subprocess.Popen`` child the test starts, once the child exists."""
created = _created(pid)
@@ -152,47 +400,21 @@ class TestProcesses:
"""Kill what the test left running and wait until it is gone; describe each process killed."""
__tracebackhide__ = True
targets: dict[int, float] = {}
-
- def add(process: psutil.Process) -> None:
- try:
- for each in (process, *process.children(recursive=True)):
- targets.setdefault(each.pid, each.create_time())
- except psutil.Error:
- pass
-
for pid, created in _descendants().items():
if self._before.get(pid) != created and (process := _same(pid, created)) is not None:
- add(process)
+ _add_tree(targets, process)
for pid, spawn in self._spawned.items():
if (process := _same(pid, spawn.created)) is not None:
- add(process)
+ _add_tree(targets, process)
# A group outlives its leader: what a spawned shell put in the background stays in it.
groups = {
pid for pid, spawn in self._spawned.items() if spawn.founded_group and _group_is_ours(pid, spawn.created)
}
for member in _group_members(groups) if groups else ():
- add(member)
+ _add_tree(targets, member)
targets.pop(os.getpid(), None)
reaped = [_describe(pid) for pid, created in targets.items() if _same(pid, created) is not None]
- for pgid in groups:
- try:
- _KILLPG(pgid, signal.SIGKILL)
- except OSError:
- pass
- for pid, created in targets.items():
- if _same(pid, created) is not None:
- try:
- _KILL(pid, signal.SIGKILL)
- except OSError:
- pass
- give_up = time.monotonic() + hang_guard.HANG_GUARD_SECONDS
- while running := [pid for pid, created in targets.items() if _same(pid, created) is not None]:
- if time.monotonic() >= give_up:
- pytest.fail(
- f"hung for {hang_guard.HANG_GUARD_SECONDS:.0f}s waiting for SIGKILLed processes to end: "
- + ", ".join(map(_describe, running))
- )
- time.sleep(0.01)
+ _kill_and_wait(targets, groups)
if self._warden is not None:
self._warden.settle()
return reaped
diff --git a/tests/test_cloudflared_runtime.py b/tests/test_cloudflared_runtime.py
index 41a21949d..92355d288 100644
--- a/tests/test_cloudflared_runtime.py
+++ b/tests/test_cloudflared_runtime.py
@@ -13,6 +13,9 @@ from core.cloudflared_runtime import CloudflaredRuntimeManager
from tests.test_remote_access_vibe_cloud import _config
from vibe import remote_access, runtime
+# Real fake-cloudflared connectors and their log sinks.
+pytestmark = [pytest.mark.reap_processes, pytest.mark.usefixtures("reap_helper_spawns")]
+
UPSTREAM = "https://github.com/cloudflare/cloudflared/releases/download/2026.10.0"
diff --git a/tests/test_process_reaper.py b/tests/test_process_reaper.py
index 283fed386..370873b9a 100644
--- a/tests/test_process_reaper.py
+++ b/tests/test_process_reaper.py
@@ -7,8 +7,10 @@ reaper and the warden act exactly as they do for every marked test file.
from __future__ import annotations
import os
+import signal
import subprocess
import sys
+import time
from pathlib import Path
import psutil
@@ -83,6 +85,72 @@ def test_an_unmarked_run_starts_no_warden():
'''
+# A helper interpreter, as a cross-process child is: it spawns a detached
+# connector and exits, so the connector reparents to init. Unless the mode is
+# `skip` it records itself and the spawn first, as the remote-access helpers do;
+# `late` spawns a second after the test has returned, and `group` spawns a shell
+# that leaves a background member in its group when it exits.
+DETACHING = '''
+import os
+import signal
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from tests import process_reaper
+
+pytestmark = [pytest.mark.reap_processes, pytest.mark.usefixtures("reap_helper_spawns")]
+
+HELPER = (
+ "import os, subprocess, sys, time\\n"
+ "from pathlib import Path\\n"
+ "import psutil\\n"
+ "mode, connector, record = sys.argv[1:4]\\n"
+ "if mode != 'skip':\\n"
+ " from tests import process_reaper\\n"
+ " process_reaper.record_spawns(Path(record))\\n"
+ "if mode == 'late':\\n"
+ " time.sleep(1)\\n"
+ "args = ['/bin/sh', '-c', 'sleep 600 & sleep 0.5'] if mode == 'group' else [connector]\\n"
+ "spawn = subprocess.Popen(args, start_new_session=True, stdin=subprocess.DEVNULL,"
+ " stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\\n"
+ "created = psutil.Process(spawn.pid).create_time()\\n"
+ "Path(os.environ['REAPER_PROBE_RECORD']).write_text(f'{spawn.pid} {created!r}\\\\n')\\n"
+)
+
+
+def _run_helper(tmp_path, mode):
+ connector = tmp_path / "bin" / "cloudflared"
+ connector.parent.mkdir()
+ connector.write_text("#!/bin/sh\\nwhile :; do sleep 0.2; done\\n")
+ connector.chmod(0o755)
+ record = process_reaper.helper_spawn_record(tmp_path)
+ helper = subprocess.Popen([sys.executable, "-c", HELPER, mode, str(connector), str(record)])
+ if mode != "late":
+ assert helper.wait() == 0
+
+
+def test_a_recorded_spawn(tmp_path):
+ _run_helper(tmp_path, "record")
+
+
+def test_an_unrecorded_orphan(tmp_path):
+ _run_helper(tmp_path, "skip")
+
+
+def test_a_spawn_recorded_after_the_test_returned(tmp_path):
+ _run_helper(tmp_path, "late")
+
+
+def test_a_group_member_left_by_a_recorded_leader(tmp_path):
+ _run_helper(tmp_path, "group")
+
+
+'''
+
+
def _run_inner(tmp_path: Path, selection: str, source: str = INNER) -> tuple[subprocess.CompletedProcess, Path]:
inner = tmp_path / "test_inner.py"
inner.write_text(source)
@@ -134,6 +202,211 @@ def test_a_run_without_a_marked_test_has_no_warden(tmp_path):
assert result.returncode == 0, result.stdout + result.stderr
+def _group_of(pid: int) -> int | None:
+ try:
+ return os.getpgid(pid)
+ except OSError:
+ return None
+
+
+def _reported(output: str, after: str) -> str:
+ return output.split(after)[1].splitlines()[0]
+
+
+def test_a_recorded_spawn_is_killed_by_identity_and_fails_its_test(tmp_path):
+ result, record = _run_inner(tmp_path, "test_a_recorded_spawn", DETACHING)
+
+ output = result.stdout + result.stderr
+ assert result.returncode == 1, output
+ # The body passed; the teardown reaped the spawn and failed the test.
+ assert "1 passed" in output and "1 error" in output, output
+ pid = record.read_text().split()[0]
+ reported = _reported(output, "were still running and were killed: ")
+ # A report names the process, never its arguments, which can carry a credential.
+ assert reported.startswith(f"{pid} (") and "/" not in reported
+ assert _running(record) == []
+
+
+def test_a_spawn_a_helper_records_during_teardown_is_reaped(tmp_path):
+ result, record = _run_inner(tmp_path, "recorded_after_the_test_returned", DETACHING)
+
+ output = result.stdout + result.stderr
+ assert result.returncode == 1, output
+ pid = record.read_text().split()[0]
+ assert f"were still running and were killed: {pid} (" in output, output
+ assert _running(record) == []
+
+
+def test_a_group_member_a_recorded_leader_left_is_killed_and_fails_its_test(tmp_path):
+ result, record = _run_inner(tmp_path, "group_member_left", DETACHING)
+
+ output = result.stdout + result.stderr
+ assert result.returncode == 1, output
+ assert "(sleep)" in _reported(output, "were still running and were killed: "), output
+ leader = int(record.read_text().split()[0])
+ assert [process.pid for process in psutil.process_iter() if _group_of(process.pid) == leader] == []
+
+
[email protected]_foreign_signals(reason="ends the orphan the inner run deliberately left unrecorded")
+def test_an_unrecorded_orphan_is_reported_and_never_signalled(tmp_path):
+ try:
+ result, record = _run_inner(tmp_path, "unrecorded_orphan", DETACHING)
+ pid = int(record.read_text().split()[0])
+ output = result.stdout + result.stderr
+ assert result.returncode == 1, output
+ reported = _reported(output, "are in no spawn record and were left running: ")
+ assert reported.startswith(f"{pid} (") and "/" not in reported
+ assert _running(record) == [pid]
+ finally:
+ # No reaper knows this orphan, so the test ends it by the identity the
+ # helper wrote, however the inner run ended. It leads its own group:
+ # its shell's sleeps end with it.
+ written = tmp_path / "spawned"
+ for running in _running(written) if written.exists() else []:
+ os.killpg(running, signal.SIGKILL)
+
+
+def test_a_process_with_a_live_parent_naming_tmp_path_is_neither_killed_nor_reported(tmp_path):
+ # A developer's `tail -f` on a test log: it names tmp_path, but its parent lives.
+ watcher = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)", str(tmp_path / "connector.log")])
+ try:
+ assert process_reaper.end_helper_spawns(process_reaper.helper_spawn_record(tmp_path), tmp_path) == ([], [])
+ assert watcher.poll() is None
+ finally:
+ watcher.kill()
+ watcher.wait()
+
+
+# A helper whose record write fails after `Popen` has started the child: the
+# disk filling up between the spawn and its fsync. The child, a shell leading a
+# session of its own, has put a command in the background by then and, with
+# `exit`, already exited.
+FAILING_RECORD = """
+import os, signal, subprocess, sys, time
+from pathlib import Path
+
+import psutil
+
+from tests import process_reaper
+
+process_reaper.record_spawns(Path(sys.argv[1]))
+spawned = []
+
+
+def in_group(pgid):
+ found = []
+ for process in psutil.process_iter():
+ try:
+ if os.getpgid(process.pid) == pgid and process.status() != psutil.STATUS_ZOMBIE:
+ found.append(process)
+ except (OSError, psutil.Error):
+ pass
+ return found
+
+
+def fail(fd):
+ shell = psutil.Process().children()[0].pid
+ spawned.append(shell)
+ deadline = time.monotonic() + 10
+ while time.monotonic() < deadline:
+ try:
+ exited = psutil.Process(shell).status() == psutil.STATUS_ZOMBIE
+ except psutil.NoSuchProcess:
+ exited = True
+ if exited and in_group(shell):
+ break
+ time.sleep(0.01)
+ raise OSError(28, "No space left on device")
+
+
+os.fsync = fail
+try:
+ subprocess.Popen(["/bin/sh", "-c", "sleep 600 & exit 0"], start_new_session=True)
+except OSError as exc:
+ print("raised", exc.errno)
+left = [process for pgid in spawned for process in in_group(pgid)]
+print("left", len(left))
+for pgid in spawned:
+ try:
+ os.killpg(pgid, signal.SIGKILL)
+ except OSError:
+ pass
+"""
+
+
+def test_a_spawn_that_cannot_be_recorded_is_ended_before_the_failure_propagates(tmp_path):
+ result = subprocess.run(
+ [sys.executable, "-c", FAILING_RECORD, str(process_reaper.helper_spawn_record(tmp_path))],
+ cwd=REPO_ROOT,
+ capture_output=True,
+ text=True,
+ timeout=hang_guard.HANG_GUARD_SECONDS,
+ )
+
+ assert result.stdout.splitlines() == ["raised 28", "left 0"], result.stdout + result.stderr
+
+
+# A recorded helper that waits on a control pipe and, when told, spawns a
+# detached child its record never names (`posix_spawn` bypasses the `Popen`
+# wrapper), as a helper killed before its append would leave one.
+CONTROLLED_HELPER = """
+import os, sys
+from pathlib import Path
+
+import psutil
+
+from tests import process_reaper
+
+process_reaper.record_spawns(Path(sys.argv[1]))
+written = Path(sys.argv[2])
+for line in sys.stdin:
+ pid = os.posix_spawn(sys.executable, [sys.executable, "-c", "import time; time.sleep(600)"], os.environ, setsid=True)
+ written.write_text(f"{pid} {psutil.Process(pid).create_time()!r}\\n")
+"""
+
+
[email protected]_foreign_signals(reason="ends the child an unfrozen reap would leave, by the pid the helper wrote")
+def test_a_helper_is_frozen_before_the_last_snapshot_so_it_spawns_nothing_before_its_kill(tmp_path, monkeypatch):
+ record = process_reaper.helper_spawn_record(tmp_path)
+ written = tmp_path / "late-child"
+ helper = subprocess.Popen(
+ [sys.executable, "-c", CONTROLLED_HELPER, str(record), str(written)],
+ cwd=REPO_ROOT,
+ stdin=subprocess.PIPE,
+ text=True,
+ )
+ kill_and_wait = process_reaper._kill_and_wait
+
+ def ask_the_helper_to_spawn_then_kill(targets, groups):
+ # Between the last snapshot and the kill: a running helper spawns at once.
+ helper.stdin.write("spawn\n")
+ helper.stdin.flush()
+ give_up = time.monotonic() + 1.0
+ while not written.exists() and time.monotonic() < give_up:
+ time.sleep(0.01)
+ kill_and_wait(targets, groups)
+
+ try:
+ give_up = time.monotonic() + hang_guard.HANG_GUARD_SECONDS
+ while not record.exists() and time.monotonic() < give_up:
+ time.sleep(0.01)
+ monkeypatch.setattr(process_reaper, "_SPAWN_EXIT_SECONDS", 0.2)
+ monkeypatch.setattr(process_reaper, "_kill_and_wait", ask_the_helper_to_spawn_then_kill)
+
+ reaped, orphans = process_reaper.end_helper_spawns(record, tmp_path)
+
+ # Stopped, the helper never read the request: no child exists to outlive it.
+ assert not written.exists()
+ assert len(reaped) == 1 and reaped[0].startswith(f"{helper.pid} (") and orphans == []
+ finally:
+ if written.exists():
+ for pid in _running(written):
+ os.killpg(pid, signal.SIGKILL)
+ if helper.poll() is None:
+ helper.kill()
+ helper.wait()
+
+
@pytest.fixture
def group_leader():
"""A live process leading its own group, as a stranger that reused a recorded pid would."""
diff --git a/tests/test_tunnel_recovery.py b/tests/test_tunnel_recovery.py
index 9df27ac5c..736a0f8f2 100644
--- a/tests/test_tunnel_recovery.py
+++ b/tests/test_tunnel_recovery.py
@@ -12,9 +12,13 @@ import psutil
import pytest
from config.v2_config import V2Config
+from tests import process_reaper
from tests.test_remote_access_vibe_cloud import _config
from vibe import remote_access, runtime
+# Real fake-cloudflared connectors and log sinks, some started by child Pythons.
+pytestmark = [pytest.mark.reap_processes, pytest.mark.usefixtures("reap_helper_spawns")]
+
def _quality(median: float) -> dict:
return {
@@ -1742,6 +1746,10 @@ _LIFECYCLE_CHILD = """
import json, sys
from pathlib import Path
+from tests import process_reaper
+
+process_reaper.record_spawns(Path(sys.argv[3]))
+
import psutil
from vibe import remote_access, runtime
@@ -1770,7 +1778,7 @@ report.write_text(json.dumps({"ok": result.get("ok"), "error": result.get("error
def _lifecycle_child(tmp_path: Path, action: str, name: str) -> tuple[subprocess.Popen, Path]:
report = tmp_path / f"{name}.json"
process = subprocess.Popen(
- [sys.executable, "-c", _LIFECYCLE_CHILD, action, str(report)],
+ [sys.executable, "-c", _LIFECYCLE_CHILD, action, str(report), str(process_reaper.helper_spawn_record(tmp_path))],
cwd=str(Path(remote_access.__file__).resolve().parents[1]),
env={**os.environ, "AVIBE_HOME": str(tmp_path)},
stdin=subprocess.PIPE,