Skip to content

Official Codebase of 3DCodeVerse

Python

proc: one ManagedProcess owns every child's lifecycle

gaoypengea379293a365
diff --git a/harness/codeverse/agents/watchdog.py b/harness/codeverse/agents/watchdog.py
index 8b685f0f..6a42131d 100644
--- a/harness/codeverse/agents/watchdog.py
+++ b/harness/codeverse/agents/watchdog.py
@@ -10,7 +10,9 @@ Three clocks govern a run:
* ``hard_timeout_s`` — absolute ceiling; kill regardless (``"hard_timeout"``).
The child is started in its own session so ``os.killpg`` takes the whole
-tree (node → chrome, blender, mcp servers ...).
+tree (node → chrome, blender, mcp servers ...). The process lifecycle itself
+(pipes, pump threads, stdin writer, kill-on-exception, drain, reap) lives in
+:class:`codeverse.proc.ManagedProcess`; this module owns only the clocks.
"""
from __future__ import annotations
@@ -25,12 +27,20 @@ from collections.abc import Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
+from codeverse.proc import ManagedProcess
+
_SKIP_DIRS = {"node_modules", ".git", "__pycache__", ".gemini", ".gemini_home"}
@dataclass
class CompletedProc:
- """Outcome of :func:`run_with_watchdog`."""
+ """Outcome of :func:`run_with_watchdog`.
+
+ ``stdout``/``stderr`` are bounded to ``codeverse.proc.STREAM_BUDGET_BYTES`` per
+ stream (head + tail kept, truncation marker in between); the ``*_lines`` lists
+ are derived from those bounded texts — streaming consumers that must see every
+ line unconditionally use ``on_line``.
+ """
rc: int
stdout: str
@@ -73,19 +83,12 @@ def _latest_mtime(dirs: Iterable[Path]) -> float:
return latest
-def _pump(stream, sink: list[str], tracker: ActivityTracker, on_line: Callable[[str, str], None] | None, tag: str) -> None:
- for raw in iter(stream.readline, b""):
- line = raw.decode("utf-8", errors="replace").rstrip("\n")
- sink.append(line)
- tracker.touch()
- if on_line is not None:
- with contextlib.suppress(Exception): # observer bugs must not kill the pump
- on_line(tag, line)
- stream.close()
-
-
def kill_process_group(proc: subprocess.Popen, grace_s: float = 5.0) -> None:
- """SIGTERM the child's process group, then SIGKILL after ``grace_s``."""
+ """SIGTERM the child's process group, then SIGKILL after ``grace_s``.
+
+ Kept as the public utility for raw ``Popen`` holders; ``ManagedProcess.terminate``
+ implements the same contract for the managed path (``proc.py`` is stdlib-only and
+ cannot import this module)."""
if proc.poll() is not None:
return
try:
@@ -119,6 +122,15 @@ def run_with_watchdog(
``on_line(stream, line)`` is called for every line (stream is ``"stdout"``
or ``"stderr"``). ``activity_dirs`` (default ``cwd/src``) are polled for
mtime changes (every ``scan_s``) that also count as activity.
+
+ Since 2026-08-27 the process lifecycle lives in
+ :class:`codeverse.proc.ManagedProcess`; this function keeps only the clocks.
+ Fixed by that move: a KeyboardInterrupt anywhere in the poll loop kills the
+ group before propagating (it used to orphan the whole node → chrome tree —
+ SIGINT never reaches a ``start_new_session`` child); stdin is written from a
+ helper thread, so a child that never reads a >64 KiB prompt (codex pipes
+ prompts >100 kB via stdin) can no longer wedge the main thread before the
+ clocks start; captured output is bounded per stream (head+tail + marker).
"""
if hard_timeout_s is None:
hard_timeout_s = max(soft_timeout_s * 1.5, soft_timeout_s + 600.0)
@@ -127,52 +139,39 @@ def run_with_watchdog(
tracker = ActivityTracker()
t0 = time.monotonic()
_reject_control_chars(cmd, cwd)
- proc = subprocess.Popen(
- list(cmd), cwd=str(cwd), env=dict(env) if env is not None else None,
- stdin=subprocess.PIPE if stdin is not None else subprocess.DEVNULL,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True,
- )
- out: list[str] = []
- err: list[str] = []
- threads = [
- threading.Thread(target=_pump, args=(proc.stdout, out, tracker, on_line, "stdout"), daemon=True),
- threading.Thread(target=_pump, args=(proc.stderr, err, tracker, on_line, "stderr"), daemon=True),
- ]
- for t in threads:
- t.start()
- if stdin is not None:
- try:
- proc.stdin.write(stdin.encode("utf-8")) # type: ignore[union-attr]
- proc.stdin.close() # type: ignore[union-attr]
- except BrokenPipeError:
- pass
+
+ def observe(stream: str, line: str) -> None:
+ tracker.touch()
+ if on_line is not None:
+ on_line(stream, line) # ManagedProcess suppresses observer exceptions
killed = ""
- last_mtime = _latest_mtime(dirs)
- next_scan = t0 + scan_s
- while proc.poll() is None:
- now = time.monotonic()
- elapsed = now - t0
- if now >= next_scan:
- m = _latest_mtime(dirs)
- if m > last_mtime:
- last_mtime = m
- tracker.touch()
- next_scan = now + scan_s
- if elapsed >= hard_timeout_s:
- killed = "hard_timeout"
- elif elapsed >= soft_timeout_s and tracker.idle_for() >= idle_grace_s:
- killed = "idle"
- if killed:
- kill_process_group(proc)
- break
- time.sleep(poll_s)
- for t in threads:
- t.join(timeout=10)
- rc = proc.returncode if proc.returncode is not None else -9
+ with ManagedProcess(cmd, cwd=cwd, env=env, stdin_text=stdin, on_line=observe) as mp:
+ last_mtime = _latest_mtime(dirs)
+ next_scan = t0 + scan_s
+ while mp.poll() is None:
+ now = time.monotonic()
+ elapsed = now - t0
+ if now >= next_scan:
+ m = _latest_mtime(dirs)
+ if m > last_mtime:
+ last_mtime = m
+ tracker.touch()
+ next_scan = now + scan_s
+ if elapsed >= hard_timeout_s:
+ killed = "hard_timeout"
+ elif elapsed >= soft_timeout_s and tracker.idle_for() >= idle_grace_s:
+ killed = "idle"
+ if killed:
+ mp.terminate() # TERM → 5 s grace → KILL: the kill_process_group contract
+ break
+ time.sleep(poll_s)
+ rc = mp.returncode if mp.returncode is not None else -9
+ out_text, err_text = mp.stdout_text, mp.stderr_text
return CompletedProc(
- rc=rc, stdout="\n".join(out), stderr="\n".join(err), duration_s=time.monotonic() - t0,
- timed_out=bool(killed), killed_reason=killed, stdout_lines=out, stderr_lines=err,
+ rc=rc, stdout=out_text, stderr=err_text, duration_s=time.monotonic() - t0,
+ timed_out=bool(killed), killed_reason=killed,
+ stdout_lines=out_text.splitlines(), stderr_lines=err_text.splitlines(),
)
diff --git a/harness/codeverse/proc.py b/harness/codeverse/proc.py
index 90478610..dce76957 100644
--- a/harness/codeverse/proc.py
+++ b/harness/codeverse/proc.py
@@ -1,7 +1,8 @@
"""Subprocess + atomic-JSON primitives shared across the harness.
One home for the run-a-child-process pattern (own process group, wall-clock
-timeout, group kill, captured output), the tmp+rename JSON write, and the
+timeout, group kill, bounded captured output — :class:`ManagedProcess` and its
+thin wrapper :func:`run_subprocess`), the tmp+rename JSON write, and the
tolerant JSON / JSONL readers every "best effort" side-car consumer needs. Peer
of ``workspace.py``; stdlib-only — imports nothing from ``codeverse`` so wrappers,
spatial helpers and agents can all use it without layering back-edges.
@@ -20,10 +21,11 @@ import signal
import subprocess
import threading
import time
-from collections.abc import Callable, Iterator, Mapping
+from collections import deque
+from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
-from typing import Any
+from typing import IO, Any
@dataclass(frozen=True)
@@ -50,63 +52,277 @@ def run_subprocess(
Never raises on non-zero exit — callers inspect :attr:`ProcResult.returncode`.
``preexec_fn`` runs in the child before exec (e.g. rlimits); the child is
- always started in a new session so :func:`kill_group` reaps grandchildren.
+ always started in a new session so the group kill reaps grandchildren.
+
+ Thin wrapper over :class:`ManagedProcess` since 2026-08-27, which changed three
+ behaviours on purpose:
+
+ * output decodes as utf-8 ``errors="replace"`` — one bad byte used to raise
+ UnicodeDecodeError out of ``communicate()`` and lose the whole result;
+ * each stream is capped at :data:`STREAM_BUDGET_BYTES` (head + tail kept, a
+ truncation marker in between) instead of growing unbounded in RAM;
+ * KeyboardInterrupt — or any exception — mid-run kills the process group before
+ propagating. ``start_new_session`` children sit outside the terminal's
+ foreground group, so Ctrl-C's SIGINT never reaches them: without this, every
+ interrupt during a blender/node/render run orphaned the whole tree.
"""
t0 = time.monotonic()
- proc = subprocess.Popen(
- cmd,
- cwd=str(cwd),
- env=dict(env) if env is not None else None,
- stdin=subprocess.PIPE if stdin_text is not None else subprocess.DEVNULL,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- start_new_session=True,
- preexec_fn=preexec_fn,
- )
timed_out = False
- try:
- out, err = proc.communicate(input=stdin_text, timeout=timeout_s)
- except subprocess.TimeoutExpired:
- timed_out = True
- kill_group(proc)
- out, err = _drain_after_kill(proc)
+ with ManagedProcess(cmd, cwd=cwd, env=env, stdin_text=stdin_text, preexec_fn=preexec_fn) as mp:
+ try:
+ mp.wait(timeout=timeout_s)
+ except subprocess.TimeoutExpired:
+ timed_out = True
+ mp.kill() # straight SIGKILL on the group, as this path has always behaved
return ProcResult(
- returncode=proc.returncode if proc.returncode is not None else -1,
- stdout=out or "",
- stderr=err or "",
+ returncode=mp.returncode if mp.returncode is not None else -1,
+ stdout=mp.stdout_text,
+ stderr=mp.stderr_text,
timed_out=timed_out,
duration_ms=int((time.monotonic() - t0) * 1000),
)
-#: how long to keep draining the pipes after the group kill. Matches
-#: ``agents/watchdog.py``'s ``thread.join(timeout=10)``.
+#: how long to keep draining the pipes after the child is dead. Kept from the old
+#: ``_drain_after_kill`` (and ``agents/watchdog.py``'s ``thread.join(timeout=10)``);
+#: a descendant that setsid'd or daemonised while inheriting stdout/stderr keeps the
+#: pipes open with no upper bound, and waiting on it silently voided ``timeout_s``
+#: for every build/render path that funnels through here.
DRAIN_TIMEOUT_S = 10.0
-_ABANDONED = "(output abandoned: a detached descendant still holds the pipe)"
+_ABANDONED = "(output truncated: a detached descendant still held the pipe past the drain window)"
+#: per-stream cap on captured output: the first ``budget // 2`` bytes and the last
+#: ``budget - budget // 2`` bytes are kept, with a truncation marker in between.
+#: 8 MiB per stream — roomy for real build/render logs, while a runaway print loop
+#: can no longer grow the harness RSS (or the trajectory files, which record full
+#: stdout — only the model-facing SHELL_OUT_CAP existed before) without bound.
+STREAM_BUDGET_BYTES = 8 * 1024 * 1024
-def _drain_after_kill(proc: subprocess.Popen[str]) -> tuple[str, str]:
- """Collect whatever the killed child wrote, WITHOUT waiting forever.
+#: longest chunk one pump read returns. A child that emits gigabytes with no
+#: newline would otherwise accumulate the whole "line" in RAM before the stream
+#: budget could apply; a line longer than this reaches ``on_line`` as several
+#: chunk-sized calls.
+_LINE_CAP_BYTES = 1024 * 1024
- ``kill_group`` killpg's only the child's own session, so a descendant that
- setsid'd or daemonised while inheriting stdout/stderr keeps those pipes open —
- and an unbounded ``communicate()`` then blocks until IT exits, which has no upper
- bound. That silently voided ``timeout_s`` for every build/render path (blender,
- node, cadquery, urdf, gl_render, joints_export, doctor) that funnels through here.
+
+class _BoundedSink:
+ """Head+tail byte buffer for ONE stream (first half of the budget from the head,
+ last half from the tail, everything between counted and replaced by a marker).
+
+ The lock only pairs one pump thread's ``feed`` with a late ``text`` read — it is
+ never contended in the normal path (``text`` runs after the pump is joined).
"""
- try:
- return proc.communicate(timeout=DRAIN_TIMEOUT_S)
- except subprocess.TimeoutExpired:
- # The child itself is already SIGKILLed; close our ends of the pipes so the
- # escaped descendant cannot hold us, then reap the child (it is a zombie now).
- for pipe in (proc.stdin, proc.stdout, proc.stderr):
- if pipe is not None:
- with contextlib.suppress(OSError):
- pipe.close()
+
+ def __init__(self, budget: int) -> None:
+ self._lock = threading.Lock()
+ self._head_max = budget // 2
+ self._tail_max = budget - self._head_max
+ self._head = bytearray()
+ self._tail: deque[bytes] = deque()
+ self._tail_len = 0
+ self._dropped = 0
+
+ def feed(self, chunk: bytes) -> None:
+ with self._lock:
+ room = self._head_max - len(self._head)
+ if room > 0:
+ self._head += chunk[:room]
+ chunk = chunk[room:]
+ if not chunk:
+ return
+ self._tail.append(chunk)
+ self._tail_len += len(chunk)
+ while self._tail_len > self._tail_max and len(self._tail) > 1:
+ gone = self._tail.popleft()
+ self._tail_len -= len(gone)
+ self._dropped += len(gone)
+
+ def text(self) -> str:
+ """Decoded output — ``errors="replace"``, so a bad byte costs one U+FFFD, not the run."""
+ with self._lock:
+ head = self._head.decode("utf-8", errors="replace")
+ tail_part = b"".join(self._tail).decode("utf-8", errors="replace")
+ dropped = self._dropped
+ if not dropped:
+ return head + tail_part
+ marker = (f"\n[... {dropped} bytes dropped: stream exceeded the "
+ f"{self._head_max + self._tail_max}-byte capture budget (head+tail kept) ...]\n")
+ return head + marker + tail_part
+
+
+class ManagedProcess:
+ """A ``Popen`` in its own session whose process group cannot outlive the ``with`` block.
+
+ The shared lifecycle core under :func:`run_subprocess` and
+ ``agents.watchdog.run_with_watchdog`` (which keep their public signatures):
+
+ * binary pipes + two pump threads feeding a :class:`_BoundedSink` per stream
+ (:data:`STREAM_BUDGET_BYTES` each, decoded utf-8 ``errors="replace"``);
+ * an optional ``on_line(stream, line)`` observer called from the pump threads for
+ every line (``stream`` is ``"stdout"``/``"stderr"``, the line is decoded with
+ its trailing newline stripped; a line longer than :data:`_LINE_CAP_BYTES`
+ arrives as several chunk-sized calls; observer exceptions are suppressed);
+ * a stdin WRITER THREAD — a child that never reads a >64 KiB pipe-buffer of
+ stdin can no longer wedge the caller's main thread in ``stdin.write`` (codex
+ really does deliver prompts via stdin once they exceed 100 kB);
+ * ``__exit__``: on ANY exception — KeyboardInterrupt above all, because
+ ``start_new_session`` children never receive the terminal's Ctrl-C SIGINT —
+ or with the child still running at scope exit, the whole group gets
+ SIGTERM → ``grace_s`` → SIGKILL, the pumps are drained for at most
+ :data:`DRAIN_TIMEOUT_S`, and the child is reaped with ``wait()`` so no zombie
+ is left. A descendant that setsid'd away and still holds a pipe cannot stall
+ the drain: closing the stream from this thread would block on the buffer lock
+ the parked reader holds (measured), so the stuck pump is flagged to stop, drops
+ its in-flight chunk, and self-cleans whenever its read finally returns —
+ ``stderr_text`` then carries a truncation note.
+ """
+
+ def __init__(
+ self,
+ cmd: Sequence[str],
+ *,
+ cwd: Path | str,
+ env: Mapping[str, str] | None = None,
+ stdin_text: str | None = None,
+ preexec_fn: Callable[[], None] | None = None,
+ on_line: Callable[[str, str], None] | None = None,
+ stream_budget: int = STREAM_BUDGET_BYTES,
+ grace_s: float = 5.0,
+ ) -> None:
+ self._on_line = on_line
+ self._grace_s = grace_s
+ self._stop = threading.Event() # tells an abandoned pump to quit feeding
+ self._abandoned = False
+ self._sinks = {"stdout": _BoundedSink(stream_budget), "stderr": _BoundedSink(stream_budget)}
+ self.proc: subprocess.Popen[bytes] = subprocess.Popen(
+ list(cmd),
+ cwd=str(cwd),
+ env=dict(env) if env is not None else None,
+ stdin=subprocess.PIPE if stdin_text is not None else subprocess.DEVNULL,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ start_new_session=True,
+ preexec_fn=preexec_fn,
+ )
+ assert self.proc.stdout is not None and self.proc.stderr is not None
+ self._pumps = [
+ threading.Thread(target=self._pump, args=(self.proc.stdout, "stdout"), daemon=True),
+ threading.Thread(target=self._pump, args=(self.proc.stderr, "stderr"), daemon=True),
+ ]
+ for t in self._pumps:
+ t.start()
+ self._writer: threading.Thread | None = None
+ if stdin_text is not None:
+ self._writer = threading.Thread(
+ target=self._write_stdin, args=(stdin_text.encode("utf-8"),), daemon=True
+ )
+ self._writer.start()
+
+ # ------------------------------------------------------------- pump / writer threads
+ def _pump(self, stream: IO[bytes], tag: str) -> None:
+ sink = self._sinks[tag]
+ with contextlib.suppress(OSError, ValueError): # ValueError: stream closed under us
+ while True:
+ chunk = stream.readline(_LINE_CAP_BYTES)
+ if not chunk or self._stop.is_set():
+ break
+ sink.feed(chunk)
+ if self._on_line is not None:
+ with contextlib.suppress(Exception): # observer bugs must not kill the pump
+ self._on_line(tag, chunk.decode("utf-8", errors="replace").rstrip("\n"))
+ with contextlib.suppress(OSError):
+ stream.close()
+
+ def _write_stdin(self, data: bytes) -> None:
+ stdin = self.proc.stdin
+ if stdin is None: # pragma: no cover — the constructor only starts us with a pipe
+ return
+ with contextlib.suppress(OSError, ValueError): # BrokenPipe (child never read) is OSError
+ stdin.write(data)
+ with contextlib.suppress(OSError, ValueError):
+ stdin.close()
+
+ # ------------------------------------------------------------- what the wrappers need
+ @property
+ def pid(self) -> int:
+ """The child's pid — also its process-GROUP id (``start_new_session``)."""
+ return self.proc.pid
+
+ @property
+ def returncode(self) -> int | None:
+ return self.proc.returncode
+
+ def poll(self) -> int | None:
+ return self.proc.poll()
+
+ def wait(self, timeout: float | None = None) -> int:
+ """``Popen.wait`` — raises ``subprocess.TimeoutExpired`` exactly like it."""
+ return self.proc.wait(timeout=timeout)
+
+ @property
+ def stdout_text(self) -> str:
+ """Decoded, bounded stdout. Complete only once the ``with`` block has exited."""
+ return self._sinks["stdout"].text()
+
+ @property
+ def stderr_text(self) -> str:
+ """Decoded, bounded stderr (+ a note when an escaped descendant held the pipe)."""
+ text = self._sinks["stderr"].text()
+ if self._abandoned:
+ return f"{text}\n{_ABANDONED}" if text else _ABANDONED
+ return text
+
+ # ------------------------------------------------------------- kills and teardown
+ def kill(self) -> None:
+ """SIGKILL the whole group right now (fall back to just the child)."""
+ try:
+ os.killpg(self.proc.pid, signal.SIGKILL)
+ except (ProcessLookupError, PermissionError):
+ with contextlib.suppress(ProcessLookupError):
+ self.proc.kill()
+
+ def terminate(self, grace_s: float | None = None) -> None:
+ """SIGTERM the group, escalate to SIGKILL after ``grace_s`` (default: the
+ constructor's ``grace_s`` — the old ``watchdog.kill_process_group`` contract)."""
+ grace = self._grace_s if grace_s is None else grace_s
+ if self.proc.poll() is not None:
+ return
+ try:
+ os.killpg(self.proc.pid, signal.SIGTERM)
+ except ProcessLookupError:
+ return
+ except PermissionError: # cannot signal the group — go for the child itself
+ self.proc.terminate()
+ try:
+ self.proc.wait(timeout=grace)
+ except subprocess.TimeoutExpired:
+ self.kill()
+ with contextlib.suppress(subprocess.TimeoutExpired):
+ self.proc.wait(timeout=self._grace_s)
+
+ def __enter__(self) -> ManagedProcess:
+ return self
+
+ def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: object) -> None:
+ # Any in-flight exception (KeyboardInterrupt included) or a child still
+ # running at scope exit: the group dies HERE. Ctrl-C's SIGINT never reaches
+ # a start_new_session child, so this is the only thing standing between an
+ # interrupt and an orphaned blender/node/chrome tree. Never swallows exc.
+ if self.proc.poll() is None:
+ self.terminate()
+ self._drain_and_reap()
+
+ def _drain_and_reap(self) -> None:
+ deadline = time.monotonic() + DRAIN_TIMEOUT_S # module global on purpose: tests dial it down
+ if self._writer is not None:
+ self._writer.join(timeout=max(0.0, deadline - time.monotonic()))
+ for t in self._pumps:
+ t.join(timeout=max(0.0, deadline - time.monotonic()))
+ if any(t.is_alive() for t in self._pumps):
+ self._stop.set() # the parked daemon pump self-cleans when its read returns
+ self._abandoned = True
with contextlib.suppress(subprocess.TimeoutExpired):
- proc.wait(timeout=DRAIN_TIMEOUT_S)
- return "", _ABANDONED
+ self.proc.wait(timeout=DRAIN_TIMEOUT_S) # reap — a zombie survives killpg
def kill_group(proc: subprocess.Popen[str]) -> None:
diff --git a/harness/tests/agents/test_watchdog.py b/harness/tests/agents/test_watchdog.py
index fd2074f8..fcb79fb7 100644
--- a/harness/tests/agents/test_watchdog.py
+++ b/harness/tests/agents/test_watchdog.py
@@ -3,10 +3,13 @@
from __future__ import annotations
import os
+import signal
import sys
import time
from pathlib import Path
+import pytest
+
from codeverse.agents.watchdog import run_with_watchdog
PY = sys.executable
@@ -92,3 +95,54 @@ def test_file_mtime_counts_as_activity(tmp_path: Path):
res = run_with_watchdog([PY, "-c", code], cwd=tmp_path, env=None, soft_timeout_s=1, idle_grace_s=1.5,
hard_timeout_s=30, poll_s=0.2, scan_s=0.3, activity_dirs=[src])
assert res.rc == 0 and not res.timed_out
+
+
+def _raise_ki(signum: int, frame: object) -> None:
+ raise KeyboardInterrupt
+
+
[email protected](60, method="thread") # thread method: the test drives SIGALRM itself
+def test_keyboard_interrupt_kills_the_group(tmp_path: Path):
+ """A KI in the poll loop used to leak the whole start_new_session tree (no
+ try/finally anywhere) — and Ctrl-C's SIGINT never reaches that tree on its own.
+ The group must die and the KI must still propagate."""
+ pid_file = tmp_path / "pid"
+ code = (
+ "import os, pathlib, sys, time\n"
+ "pathlib.Path(sys.argv[1]).write_text(str(os.getpid()))\n"
+ "time.sleep(300)\n"
+ )
+ old = signal.signal(signal.SIGALRM, _raise_ki)
+ try:
+ signal.setitimer(signal.ITIMER_REAL, 0.8) # lands in time.sleep(poll_s)
+ with pytest.raises(KeyboardInterrupt):
+ run_with_watchdog([PY, "-c", code, str(pid_file)], cwd=tmp_path, env=None,
+ soft_timeout_s=120, idle_grace_s=120, hard_timeout_s=300, poll_s=0.1)
+ finally:
+ signal.setitimer(signal.ITIMER_REAL, 0.0)
+ signal.signal(signal.SIGALRM, old)
+ pgid = int(pid_file.read_text())
+ deadline = time.monotonic() + 10
+ while time.monotonic() < deadline:
+ try:
+ os.killpg(pgid, 0)
+ except ProcessLookupError:
+ return # gone and reaped
+ time.sleep(0.05)
+ raise AssertionError(f"process group {pgid} survived the KeyboardInterrupt")
+
+
[email protected](60, method="thread")
+def test_child_that_never_reads_a_big_stdin_still_times_out(tmp_path: Path):
+ """The prompt used to be written SYNCHRONOUSLY in the main thread before the
+ poll loop started: a child that doesn't read >64 KiB of stdin blocked
+ proc.stdin.write() forever and BOTH timeouts were dead. Reachable for real:
+ codex delivers prompts via stdin exactly when they exceed STDIN_PROMPT_BYTES
+ (100 kB). 1 MB of stdin into a child that never reads it must still die on
+ the clocks."""
+ t0 = time.monotonic()
+ res = run_with_watchdog([PY, "-c", "import time; time.sleep(300)"], cwd=tmp_path, env=None,
+ soft_timeout_s=0.5, idle_grace_s=0.5, hard_timeout_s=8, poll_s=0.2,
+ stdin="x" * 1_000_000)
+ assert res.timed_out and res.killed_reason in ("idle", "hard_timeout")
+ assert time.monotonic() - t0 < 30
diff --git a/harness/tests/core/test_proc.py b/harness/tests/core/test_proc.py
index 465e31f4..1b4f888a 100644
--- a/harness/tests/core/test_proc.py
+++ b/harness/tests/core/test_proc.py
@@ -5,12 +5,17 @@ from __future__ import annotations
import json
import os
import resource
+import signal
import sys
import threading
import time
from pathlib import Path
+import pytest
+
from codeverse.proc import (
+ STREAM_BUDGET_BYTES,
+ ManagedProcess,
ProcResult,
append_jsonl_line,
iter_jsonl_lines,
@@ -298,3 +303,90 @@ def test_scrub_secrets_stays_in_sync_with_agents_cli_common():
+ ["PATH", "HOME", "NODE_PATH", "TOKENIZERS_PARALLELISM"])
for name in probes:
assert is_secret_env(name) == (name not in scrub_secrets({name: "v"})), name
+
+
+# --------------------------------------------------------------------------- ManagedProcess lifecycle
+def _raise_ki(signum: int, frame: object) -> None:
+ raise KeyboardInterrupt
+
+
+def _assert_group_gone(pgid: int, timeout_s: float = 10.0) -> None:
+ deadline = time.monotonic() + timeout_s
+ while time.monotonic() < deadline:
+ try:
+ os.killpg(pgid, 0)
+ except ProcessLookupError:
+ return # every member is dead AND reaped
+ time.sleep(0.05)
+ raise AssertionError(f"process group {pgid} is still alive")
+
+
+#: writes its own pid to argv[1], then hangs — a child that ignores nothing.
+_PID_THEN_SLEEP = (
+ "import os, pathlib, sys, time\n"
+ "pathlib.Path(sys.argv[1]).write_text(str(os.getpid()))\n"
+ "time.sleep(300)\n"
+)
+
+
[email protected](60, method="thread") # thread method: the test drives SIGALRM itself
+def test_keyboard_interrupt_kills_the_group(tmp_path: Path):
+ """``start_new_session`` puts the child outside the terminal's foreground group,
+ so Ctrl-C's SIGINT NEVER reaches it — the parent's KeyboardInterrupt must kill
+ the group itself. Pre-fix there was no try/finally around communicate(): every
+ interrupt during a blender/node run orphaned the whole tree (the wedged-browser
+ incident). The KI must still propagate to the caller."""
+ pid_file = tmp_path / "pid"
+ old = signal.signal(signal.SIGALRM, _raise_ki)
+ try:
+ signal.setitimer(signal.ITIMER_REAL, 0.8) # fires while run_subprocess blocks in wait()
+ with pytest.raises(KeyboardInterrupt):
+ run_subprocess([sys.executable, "-c", _PID_THEN_SLEEP, str(pid_file)],
+ cwd=tmp_path, timeout_s=120)
+ finally:
+ signal.setitimer(signal.ITIMER_REAL, 0.0)
+ signal.signal(signal.SIGALRM, old)
+ _assert_group_gone(int(pid_file.read_text()))
+
+
+def test_invalid_utf8_replaces_instead_of_raising(tmp_path: Path):
+ """text=True with a strict decode raised UnicodeDecodeError out of communicate()
+ — one bad byte lost the whole result. Now: U+FFFD, like the JSONL readers here."""
+ r = run_subprocess(
+ [sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'ok \\xff\\xfe end')"],
+ cwd=tmp_path, timeout_s=30,
+ )
+ assert r.returncode == 0 and not r.timed_out
+ assert r.stdout == "ok �� end"
+
+
+def test_huge_output_is_bounded_head_and_tail(tmp_path: Path):
+ """Output used to be collected unbounded in RAM (and written whole to the
+ trajectory). ~48 MB in → at most the budget (+ marker) out, keeping both ends."""
+ code = (
+ "import sys\n"
+ "w = sys.stdout.buffer.write\n"
+ "w(b'HEADSTART\\n')\n"
+ "chunk = b'x' * (1 << 20)\n"
+ "for _ in range(48):\n"
+ " w(chunk)\n"
+ "w(b'\\nTAILEND\\n')\n"
+ )
+ r = run_subprocess([sys.executable, "-c", code], cwd=tmp_path, timeout_s=110)
+ assert r.returncode == 0 and not r.timed_out
+ assert len(r.stdout) <= STREAM_BUDGET_BYTES + 200 # the budget plus the truncation marker
+ assert r.stdout.startswith("HEADSTART") # head survived
+ assert r.stdout.rstrip("\n").endswith("TAILEND") # tail survived
+ assert "bytes dropped" in r.stdout # marker sits between them
+ cut = r.stdout.index("bytes dropped")
+ assert "x" * 1000 in r.stdout[:cut] and "x" * 1000 in r.stdout[cut:]
+
+
+def test_managed_process_reaps_on_exit_no_zombie(tmp_path: Path):
+ with ManagedProcess([sys.executable, "-c", "print('hi')"], cwd=tmp_path) as mp:
+ mp.wait(timeout=30)
+ pid = mp.pid
+ assert mp.stdout_text.strip() == "hi"
+ with pytest.raises(ChildProcessError):
+ os.waitpid(pid, os.WNOHANG) # __exit__ already reaped it — no zombie left