Skip to content

The agent that grows with you

test(e2e): harden the classic-CLI E2Es: fresh-output steps, witness attestation, timeout-proof cleanup, Linux gate

Teknium48fb79f50b0c
diff --git a/tests/e2e/core/cli_tmux/_classic.py b/tests/e2e/core/cli_tmux/_classic.py
index 817073d01b7..86c6920d3d6 100644
--- a/tests/e2e/core/cli_tmux/_classic.py
+++ b/tests/e2e/core/cli_tmux/_classic.py
@@ -10,6 +10,12 @@ Driver rule (the CLI reads an Enter processed within 50 ms of the last buffer ch
newline): type with ``send-keys -l``, wait until the composer shows the text, pause, then send
``Enter`` separately. Each step waits for its needle, then until the screen (digits masked) has
been static for ``quiet`` seconds.
+
+Stale output never counts: every ``submit``/``keys``/``launch`` snapshots the scrollback first, and
+``wait`` accepts a needle only when it occurs more often than in that snapshot. An unchanged
+transcript (a no-op command) therefore fails the step instead of matching old output.
+
+Linux only: process discovery and cleanup read ``/proc/<pid>/environ``.
"""
from __future__ import annotations
@@ -25,6 +31,7 @@ import time
import uuid
from dataclasses import dataclass, field
from pathlib import Path
+from typing import Callable
from tests.fakes.fake_llm_provider import MODEL_ID, OFFER_VERSION
@@ -35,6 +42,51 @@ STATUS_BAR = f"\u2624 {MODEL_ID}"
_DIGITS = re.compile(r"\d")
_TRACEBACK = "Traceback (most recent call last)"
+
+def new_needle(needles: list[str], before: str, after: str) -> str | None:
+ """The first needle that occurs more often in *after* than in *before* (output produced since)."""
+ for needle in needles:
+ if after.count(needle) > before.count(needle):
+ return needle
+ return None
+
+
+def home_pids(hermes_home: Path) -> set[int]:
+ """PIDs (never this one) whose environ carries exactly ``HERMES_HOME=<hermes_home>``."""
+ needle = f"HERMES_HOME={hermes_home}".encode()
+ found: set[int] = set()
+ for entry in os.listdir("/proc"):
+ if not entry.isdigit() or int(entry) == os.getpid():
+ continue
+ try:
+ with open(f"/proc/{entry}/environ", "rb") as fh:
+ if needle in fh.read().split(b"\0"):
+ found.add(int(entry))
+ except OSError:
+ continue
+ return found
+
+
+def kill_home_processes(hermes_home: Path, stop: Callable[[], object] | None = None) -> None:
+ """Graceful stop, then SIGTERM/SIGKILL whatever still runs against *hermes_home*.
+
+ The signal sweep runs in ``finally``: a ``stop`` that times out or raises still leaves no
+ CLI or gateway behind.
+ """
+ try:
+ if stop is not None:
+ stop()
+ finally:
+ for sig, grace in ((signal.SIGTERM, 8.0), (signal.SIGKILL, 2.0)):
+ for pid in home_pids(hermes_home):
+ try:
+ os.kill(pid, sig)
+ except OSError:
+ pass
+ end = time.monotonic() + grace
+ while home_pids(hermes_home) and time.monotonic() < end:
+ time.sleep(0.2)
+
BASE_CONFIG = f"""model:
provider: custom
base_url: {{base_url}}
@@ -68,6 +120,8 @@ auxiliary:
class ClassicHome:
root: Path
extra_env: dict[str, str] = field(default_factory=dict)
+ # (pid, argv) of every hermes process the harness started: tmux pane CLIs and piped runs.
+ spawned: list[tuple[int, str]] = field(default_factory=list)
@property
def home(self) -> Path:
@@ -111,35 +165,38 @@ class ClassicHome:
def run(self, *args: str, timeout: float = 180.0) -> subprocess.CompletedProcess:
"""A non-TTY run (stdout piped): ``chat -q`` one-shots, ``gateway stop``."""
- return subprocess.run(self.argv(*args), cwd=self.work, env=self.env(), capture_output=True, text=True,
- timeout=timeout, stdin=subprocess.DEVNULL, check=False)
+ argv = self.argv(*args)
+ with subprocess.Popen(argv, cwd=self.work, env=self.env(), stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ stdin=subprocess.DEVNULL, text=True) as proc:
+ self.spawned.append((proc.pid, " ".join(args)))
+ try:
+ out, err = proc.communicate(timeout=timeout)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ out, err = proc.communicate()
+ raise subprocess.TimeoutExpired(argv, timeout, out, err) from None
+ return subprocess.CompletedProcess(argv, proc.returncode, out, err)
def pids(self) -> set[int]:
- needle = f"HERMES_HOME={self.hermes_home}".encode()
- found: set[int] = set()
- for entry in os.listdir("/proc"):
- if not entry.isdigit() or int(entry) == os.getpid():
- continue
+ return home_pids(self.hermes_home)
+
+ def diagnostics(self, limit: int = 3000) -> str:
+ """The CLI's stderr and the gateway log tails: why a pane died, for a failure message."""
+ parts = []
+ for label, path in (("cli stderr", self.stderr_log), ("gateway log", self.hermes_home / "logs" / "gateway.log"),
+ ("agent log", self.hermes_home / "logs" / "agent.log")):
try:
- with open(f"/proc/{entry}/environ", "rb") as fh:
- if needle in fh.read().split(b"\0"):
- found.add(int(entry))
+ text = path.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
- return found
+ if text.strip():
+ parts.append(f"--- {label} ({path.name}) ---\n{text[-limit:]}")
+ return "\n".join(parts)
def reap(self) -> None:
- """Operator stop for the profile's gateway, then SIGTERM/SIGKILL whatever this home still runs."""
- self.run("gateway", "stop", timeout=60)
- for sig, grace in ((signal.SIGTERM, 8.0), (signal.SIGKILL, 2.0)):
- for pid in self.pids():
- try:
- os.kill(pid, sig)
- except OSError:
- pass
- end = time.monotonic() + grace
- while self.pids() and time.monotonic() < end:
- time.sleep(0.2)
+ """Operator stop for the profile's gateway, then SIGTERM/SIGKILL whatever this home still runs
+ (even when the stop times out)."""
+ kill_home_processes(self.hermes_home, lambda: self.run("gateway", "stop", timeout=60))
class Pane:
@@ -148,6 +205,8 @@ class Pane:
def __init__(self, ch: ClassicHome) -> None:
self.ch = ch
self.sock = f"hermes-e2e-classic-{uuid.uuid4().hex[:8]}"
+ # Scrollback snapshot taken just before the latest input: what `wait` must see grow.
+ self.baseline = ""
def tmux(self, *args: str) -> str:
env = {k: v for k, v in os.environ.items() if not k.startswith("TMUX")}
@@ -165,6 +224,11 @@ class Pane:
env={k: v for k, v in os.environ.items() if not k.startswith("TMUX")})
self.tmux("set", "-g", "remain-on-exit", "on")
self.tmux("set", "-g", "history-limit", "50000")
+ # `sh -c "exec env -i ... python"` keeps the pane's pid: this IS the CLI process.
+ pid = self.tmux("display-message", "-p", "-t", "p", "#{pane_pid}").strip()
+ if pid.isdigit():
+ self.ch.spawned.append((int(pid), " ".join(("chat", *args)) if args else "chat"))
+ self.baseline = "" # a fresh tmux server: everything on it is new
def screen(self) -> str:
return self.tmux("capture-pane", "-p", "-t", "p")
@@ -176,16 +240,17 @@ class Pane:
return self.tmux("display-message", "-p", "-t", "p", "#{pane_dead}").strip() == "1"
def keys(self, *k: str) -> None:
+ self.baseline = self.scrollback()
self.tmux("send-keys", "-t", "p", *k)
- def wait(self, needles: list[str], timeout: float = 60.0, *, full: bool = False) -> str | None:
- """The first needle seen (screen, or the whole scrollback with ``full``), None on timeout/exit."""
+ def wait(self, needles: list[str], timeout: float = 60.0, *, since: str | None = None) -> str | None:
+ """The first needle that appeared after the latest input (or after *since*), None on timeout/exit."""
+ before = self.baseline if since is None else since
end = time.monotonic() + timeout
while time.monotonic() < end:
- text = self.scrollback() if full else self.screen()
- for needle in needles:
- if needle in text:
- return needle
+ hit = new_needle(needles, before, self.scrollback())
+ if hit is not None:
+ return hit
if self.dead():
return None
time.sleep(0.25)
@@ -208,10 +273,11 @@ class Pane:
time.sleep(1.2) # health: allow HX013 -- the gap IS the behaviour: the CLI's paste detection is time-based
def submit(self, text: str) -> None:
+ self.baseline = self.scrollback()
self.tmux("send-keys", "-t", "p", "-l", text)
- self.wait([f"{PROMPT} {text}"[-40:]], 10)
+ self.wait([f"{PROMPT} {text}"[-40:]], 10, since=self.baseline)
self.paste_gap()
- self.keys("Enter")
+ self.tmux("send-keys", "-t", "p", "Enter")
def wait_dead(self, timeout: float = 30.0) -> bool:
end = time.monotonic() + timeout
diff --git a/tests/e2e/core/cli_tmux/test_classic_cli_live_session.py b/tests/e2e/core/cli_tmux/test_classic_cli_live_session.py
index 003f92ff03d..f2c1af4bfd5 100644
--- a/tests/e2e/core/cli_tmux/test_classic_cli_live_session.py
+++ b/tests/e2e/core/cli_tmux/test_classic_cli_live_session.py
@@ -32,7 +32,8 @@ import pytest
from tests.e2e.core.cli_tmux._classic import ClassicHome, Pane, last_user, text_of
from tests.fakes.fake_llm_provider import FakeLLMServer, Text, ToolCall
-pytestmark = pytest.mark.skipif(shutil.which("tmux") is None, reason="needs tmux")
+# Process discovery, cleanup and the witness's pid accounting read /proc.
+pytestmark = pytest.mark.platforms("linux")
WITNESS_DIR = Path(__file__).resolve().parent / "witness"
TITLE = "Live session title"
@@ -115,11 +116,13 @@ class Session:
self.steps = 0
def step(self, name: str, action: Callable[[], Any], needles: list[str], *, timeout: float = 60.0,
- quiet: float = 2.0, full: bool = False) -> str:
+ quiet: float = 2.0) -> str:
+ """Run *action*, then require one of *needles* to appear in output produced after the step's
+ latest input (``Pane.wait``): text already in the scrollback never satisfies a step."""
self.steps += 1
started = time.monotonic()
action()
- hit = self.pane.wait(needles, timeout, full=full) if needles else True
+ hit = self.pane.wait(needles, timeout) if needles else True
self.pane.settle(quiet=quiet, timeout=30)
text = self.pane.scrollback()
self.timings.append((name, round(time.monotonic() - started, 1)))
@@ -136,6 +139,35 @@ def _witness(path: Path) -> list[dict[str, Any]]:
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+def check_witness_coverage(events: list[dict[str, Any]], spawned: list[tuple[int, str]]) -> None:
+ """Every process the harness spawned, and the gateway daemon, attested at startup.
+
+ The negative checks below read "no event" as "did not happen"; that holds only for a process
+ whose witness was loaded. A spawned CLI with no ``startup`` line (``PYTHONPATH`` lost, witness
+ not imported) fails here instead of passing those checks vacuously.
+ """
+ attested = {e["pid"] for e in events if e["kind"] == "startup"}
+ assert spawned, "the harness recorded no spawned CLI process"
+ missing = [(pid, argv) for pid, argv in spawned if pid not in attested]
+ assert not missing, f"spawned CLI processes with no witness startup attestation: {missing}"
+ assert any(e["kind"] == "startup" and e["role"] == "gateway" for e in events), \
+ "no gateway daemon attested: its tool/agent/state.db events cannot be trusted"
+
+
+def test_witness_coverage_rejects_an_uninstrumented_cli() -> None:
+ """The negative ownership checks are not vacuous: a silent, unattested CLI fails coverage."""
+ gateway = {"pid": 2, "ppid": 1, "role": "gateway", "kind": "startup", "detail": "-m gateway run"}
+ cli = {"pid": 10, "ppid": 1, "role": "cli", "kind": "startup", "detail": "-m chat"}
+ check_witness_coverage([gateway, cli], [(10, "chat")])
+ with pytest.raises(AssertionError, match="no witness startup attestation"):
+ check_witness_coverage([gateway, cli], [(10, "chat"), (11, "chat -c")])
+ with pytest.raises(AssertionError, match="no gateway daemon attested"):
+ check_witness_coverage([cli], [(10, "chat")])
+ with pytest.raises(AssertionError, match="recorded no spawned"):
+ check_witness_coverage([gateway], [])
+
+
[email protected](shutil.which("tmux") is None, reason="needs tmux")
def test_classic_cli_live_session_runs_every_turn_in_the_gateway(tmp_path: Path) -> None:
llm = FakeLLMServer(_main, aux=_aux)
witness = tmp_path / "witness.jsonl"
@@ -159,36 +191,35 @@ def test_classic_cli_live_session_runs_every_turn_in_the_gateway(tmp_path: Path)
s.step("clarify_modal", lambda: pane.submit("LIVE08 ask me"), ["LIVE08 which colour?"], quiet=1.5)
s.step("clarify_answer", lambda: (pane.keys("Enter"), pane.paste_gap(), pane.keys("Enter")), ["LIVE08 done"])
s.step("delegate", lambda: pane.submit("LIVE09 delegate"), ["LIVE09 done"], timeout=90)
- s.step("delegate_result", lambda: None, ["LIVECHILD result"], timeout=90, full=True, quiet=3.0)
+ s.step("delegate_result", lambda: None, ["LIVECHILD result"], timeout=90, quiet=3.0)
s.step("bg_process", lambda: pane.submit("LIVE10 background"), ["LIVE10 done"])
- s.step("bg_process_notice", lambda: None, ["live10-bg-finished"], timeout=60, full=True, quiet=3.0)
+ s.step("bg_process_notice", lambda: None, ["live10-bg-finished"], timeout=60, quiet=3.0)
for command, needle in SLASH_SWEEP:
- s.step(command, lambda c=command: pane.submit(c), [needle], timeout=30, full=True)
+ s.step(command, lambda c=command: pane.submit(c), [needle], timeout=30)
s.step("/model", lambda: pane.submit("/model"), ["Model Picker"], timeout=30, quiet=1.5)
s.step("/model close", lambda: pane.keys("Escape"), [], quiet=1.0)
s.step("/bg", lambda: pane.submit("/bg LIVE14 compute something"), ["LIVE14 background answer"],
- timeout=90, full=True)
+ timeout=90)
s.step("/btw", lambda: pane.submit("/btw LIVE15 what did I ask about?"), ["LIVE15 side answer"],
- timeout=90, full=True)
+ timeout=90)
s.step("/branch", lambda: pane.submit("/branch"), ["Branched session"], timeout=30, quiet=3.0)
# The sweep's /reasoning retired the CLI's agent (main's _retire_agent, rebuilt on the next
# turn) and /compress refuses without one, as on main: run a turn first.
s.step("turn on the branch", lambda: pane.submit("LIVE31 on the branch"), ["LIVE31 ok."])
s.step("/compress", lambda: pane.submit("/compress"), ["Compressed:", "No changes from compression"],
- timeout=90, quiet=3.0, full=True)
+ timeout=90, quiet=3.0)
s.step("interrupt", lambda: (pane.submit("LIVE20 long story"), pane.wait(["w030"], 60), pane.keys("C-c")),
- ["Interrupted"], timeout=45, full=True, quiet=3.0)
+ ["Interrupted"], timeout=45, quiet=3.0)
if "THE-END" in pane.scrollback():
s.failures.append("interrupt: the turn streamed to THE-END")
s.step("/new", lambda: pane.submit("/new"), ["destroys conversation state"], timeout=30, quiet=1.5)
- s.step("/new confirm", lambda: pane.keys("Enter"), ["New session started"], timeout=30, full=True)
+ s.step("/new confirm", lambda: pane.keys("Enter"), ["New session started"], timeout=30)
s.step("plain turn after /new", lambda: pane.submit("LIVE30 hello again"), ["LIVE30 ok."])
- s.step("/quit", lambda: pane.submit("/quit"), ["Resume this session with:"], timeout=30, full=True)
+ s.step("/quit", lambda: pane.submit("/quit"), ["Resume this session with:"], timeout=30)
if not pane.wait_dead(30):
s.failures.append("/quit: the CLI was still running 30 s later")
- s.step("resume -c", lambda: pane.launch("-c"), ["Resumed session"], timeout=120, quiet=3.0, full=True)
- s.step("/quit after resume", lambda: pane.submit("/quit"), ["Resume this session with:"], timeout=30,
- full=True)
+ s.step("resume -c", lambda: pane.launch("-c"), ["Resumed session"], timeout=120, quiet=3.0)
+ s.step("/quit after resume", lambda: pane.submit("/quit"), ["Resume this session with:"], timeout=30)
pane.wait_dead(30)
s.steps += 1
piped = ch.run("chat", "-q", "LIVE01 one shot")
@@ -199,13 +230,16 @@ def test_classic_cli_live_session_runs_every_turn_in_the_gateway(tmp_path: Path)
ch.reap()
print("step timings (s):", s.timings)
- assert not s.failures, f"{len(s.failures)}/{s.steps} steps failed:\n\n" + "\n\n".join(s.failures)
+ assert not s.failures, (f"{len(s.failures)}/{s.steps} steps failed:\n\n" + "\n\n".join(s.failures[:5])
+ + f"\n\n{ch.diagnostics()}")
events = _witness(witness)
assert not [e for e in events if e["kind"] == "witness_error"], events
+ check_witness_coverage(events, ch.spawned)
by_role: dict[str, Counter] = {"cli": Counter(), "gateway": Counter()}
for e in events:
- by_role[e["role"]][f"{e['kind']} {e['detail']}"] += 1
+ if e["kind"] != "startup":
+ by_role[e["role"]][f"{e['kind']} {e['detail']}"] += 1
cli, gateway = by_role["cli"], by_role["gateway"]
# The one-gateway contract: the CLI only reads and submits.
assert not [k for k in cli if k.startswith(("agent_init", "tool "))], f"the CLI process ran agent work: {dict(cli)}"
diff --git a/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.py b/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.py
index 01a6710bfac..f43316e18c5 100644
--- a/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.py
+++ b/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.py
@@ -28,7 +28,10 @@ import pytest
from tests.e2e.core.cli_tmux._classic import PROMPT, ClassicHome, Pane, assert_no_traceback, last_user
from tests.fakes.fake_llm_provider import FakeLLMServer, Text, ToolCall
-pytestmark = pytest.mark.skipif(shutil.which("tmux") is None, reason="needs tmux")
+pytestmark = [
+ pytest.mark.platforms("linux"), # the harness finds and reaps processes through /proc
+ pytest.mark.skipif(shutil.which("tmux") is None, reason="needs tmux"),
+]
MARKDOWN_REPLY = ("# Heading Two\n\nSome intro text for the markdown check.\n\n- alpha item\n- beta item\n"
"- gamma item\n\n```python\ndef hello():\n return 'hi'\n```\n\nEnd of STEP2 reply.")
@@ -99,7 +102,8 @@ class Steps:
def expect(self, name: str, needles: list[str], text: str) -> None:
missing = [n for n in needles if n not in text]
- assert not missing, f"step {name}: main showed {missing!r}; screen now:\n{text[-4000:]}"
+ assert not missing, (f"step {name}: main showed {missing!r}; screen now:\n{text[-4000:]}\n"
+ f"{self.pane.ch.diagnostics()}")
def test_classic_cli_matches_main_baseline_across_fifteen_steps(tmp_path: Path) -> None:
@@ -163,7 +167,7 @@ def test_classic_cli_matches_main_baseline_across_fifteen_steps(tmp_path: Path)
assert pane.wait(["w040"], 60), pane.screen()
pane.keys("C-c")
# The interrupt round-trips through the gateway; the notice lands when the turn ends.
- interrupted = pane.wait(["Interrupted"], 45, full=True)
+ interrupted = pane.wait(["Interrupted"], 45)
pane.settle(quiet=2.0, timeout=30)
text = s.capture("08_interrupt", scrollback=True)
assert not pane.dead(), f"08_interrupt: Ctrl+C killed the CLI (main stays at the prompt):\n{text[-3000:]}"
diff --git a/tests/e2e/core/cli_tmux/witness/sitecustomize.py b/tests/e2e/core/cli_tmux/witness/sitecustomize.py
index c2d4f5d947c..8a3ab1a2763 100644
--- a/tests/e2e/core/cli_tmux/witness/sitecustomize.py
+++ b/tests/e2e/core/cli_tmux/witness/sitecustomize.py
@@ -4,13 +4,17 @@ Loaded as ``sitecustomize`` by every Python process the CLI test spawns (the ``P
for this directory is set only in that test's child env). Appends one JSON line per event to
``$HERMES_E2E_WITNESS``; never changes behaviour. Events:
+* ``startup <argv>`` emitted once, at interpreter start, by every instrumented process: the
+ attestation that this pid is witnessed. A spawned CLI with no
+ ``startup`` line was not instrumented, and its silence proves nothing.
* ``agent_init <class>`` an ``AIAgent`` was constructed in this process
* ``tool <name>`` ``tools.registry`` dispatched a tool in this process
* ``sqlite_write <file>:<table>`` an INSERT/UPDATE/DELETE on a ``*.db`` file opened by this process
-``role`` is ``gateway`` when the process argv carries ``gateway``, else ``cli``. The test asserts the
-contract of the one-gateway design: the CLI process builds no agent, runs no tool and writes no
-``state.db`` row; the gateway does all three.
+``role`` is ``gateway`` for ``gateway run`` (the daemon), else ``cli``; ``ppid`` lets the test tie
+a line to the process it spawned. The test asserts the contract of the one-gateway design: every
+CLI process it started attests, builds no agent, runs no tool and writes no ``state.db`` row; the
+gateway does all three.
"""
import importlib.abc
@@ -24,10 +28,18 @@ _PATH = os.environ.get("HERMES_E2E_WITNESS")
_lock = threading.Lock()
+def _role():
+ argv = list(getattr(sys, "argv", None) or [])
+ for i, word in enumerate(argv[:-1]):
+ if word == "gateway" and argv[i + 1] == "run":
+ return "gateway"
+ return "cli"
+
+
def _emit(kind, detail=""):
try:
- role = "gateway" if "gateway" in sys.argv else "cli"
- line = json.dumps({"pid": os.getpid(), "role": role, "kind": kind, "detail": str(detail)[:200]})
+ line = json.dumps({"pid": os.getpid(), "ppid": os.getppid(), "role": _role(), "kind": kind,
+ "detail": str(detail)[:200]})
with _lock, open(_PATH, "a", encoding="utf-8") as fh:
fh.write(line + "\n")
except Exception: # health: allow BLE001 -- a witness must never break the process it watches
@@ -35,6 +47,10 @@ def _emit(kind, detail=""):
if _PATH:
+ # The attestation. At sitecustomize time ``sys.argv`` is already the command's arguments
+ # (``-m`` stands in for the module path until runpy fills it in).
+ _emit("startup", " ".join(getattr(sys, "argv", None) or []))
+
import sqlite3
_connect = sqlite3.connect
diff --git a/tests/e2e/core/providers/test_native_vertex_cli_background.py b/tests/e2e/core/providers/test_native_vertex_cli_background.py
index 58725ee1817..96acaa97eb4 100644
--- a/tests/e2e/core/providers/test_native_vertex_cli_background.py
+++ b/tests/e2e/core/providers/test_native_vertex_cli_background.py
@@ -17,7 +17,6 @@ from __future__ import annotations
import os
import shutil
-import signal
import subprocess
import sys
import time
@@ -29,10 +28,14 @@ import pytest
pytest.importorskip("google.auth", reason="Vertex minting needs google-auth (CI installs it)")
+from tests.e2e.core.cli_tmux._classic import kill_home_processes
from tests.e2e.core.providers._native_helpers import make_home, messages
from tests.fakes.providers.vertex import PROJECT, REGION, SA_EMBEDDED_PROJECT, FakeVertex, Say, hermes_setup
-pytestmark = pytest.mark.skipif(shutil.which("tmux") is None, reason="needs tmux")
+pytestmark = [
+ pytest.mark.platforms("linux"), # cleanup finds this home's processes through /proc
+ pytest.mark.skipif(shutil.which("tmux") is None, reason="needs tmux"),
+]
BG_MARK = "VERTEXBG"
BG_ANSWER = "VERTEXBG-ANSWER from the background agent"
@@ -50,24 +53,7 @@ def _respond(record: dict[str, Any]) -> Say:
return Say(BG_ANSWER if BG_MARK in _user_text(record["body"]) else "foreground ok")
-def _home_pids(hermes_home: Path) -> set[int]:
- needle = f"HERMES_HOME={hermes_home}".encode()
- found = set()
- for entry in os.listdir("/proc"):
- if entry.isdigit() and int(entry) != os.getpid():
- try:
- with open(f"/proc/{entry}/environ", "rb") as fh:
- if needle in fh.read().split(b"\0"):
- found.add(int(entry))
- except OSError:
- continue
- return found
-
-
-def test_vertex_config_label_and_background_task_through_the_gateway(tmp_path: Path) -> None:
- fake = FakeVertex(tmp_path / "fake", project=PROJECT, region=REGION, sa_project=SA_EMBEDDED_PROJECT,
- script=_respond, aux=lambda _r: Say("Vertex title"))
- fake.start()
+def _drive(fake: FakeVertex, tmp_path: Path):
nh = make_home(tmp_path / "h", **hermes_setup(fake, extra_config={"display": {"interface": "cli"}}))
sock = f"hermes-e2e-vertex-{uuid.uuid4().hex[:8]}"
env = nh.env({**fake.child_env(), "TERM": "xterm-256color"})
@@ -98,9 +84,9 @@ def test_vertex_config_label_and_background_task_through_the_gateway(tmp_path: P
# `-t file` like the sibling Vertex suites: the default toolset trips #109115 (terminal.notify's
# anyOf schema), tracked in test_native_vertex_tools.py, which is not what this test is about.
argv = [sys.executable, "-m", "hermes_cli.main", "chat", "--cli", "--yolo", "-t", "file"]
- subprocess.run(["tmux", "-L", sock, "-f", os.devnull, "new-session", "-d", "-s", "p", "-x", "160", "-y", "48",
- "-c", str(nh.project), *argv], env=env, check=True, timeout=30)
try:
+ subprocess.run(["tmux", "-L", sock, "-f", os.devnull, "new-session", "-d", "-s", "p", "-x", "160", "-y", "48",
+ "-c", str(nh.project), *argv], env=env, check=True, timeout=30)
tmux("set", "-g", "remain-on-exit", "on")
wait_for("\u2624 ", timeout=120) # the status bar: the app owns the terminal (raw mode)
@@ -117,15 +103,25 @@ def test_vertex_config_label_and_background_task_through_the_gateway(tmp_path: P
submit(f"/bg {BG_MARK} compute something")
wait_for(BG_ANSWER, timeout=120)
finally:
- tmux("kill-server")
- subprocess.run([sys.executable, "-m", "hermes_cli.main", "gateway", "stop"], cwd=nh.project, env=env,
- capture_output=True, text=True, timeout=60, check=False)
- for pid in _home_pids(nh.hermes_home):
- try:
- os.kill(pid, signal.SIGKILL)
- except OSError:
- pass
- fake.stop()
+ # Independent steps: a tmux failure or a graceful stop that times out still kills every
+ # process on this home (the fake server is stopped by the caller's own finally).
+ try:
+ tmux("kill-server")
+ finally:
+ kill_home_processes(nh.hermes_home, lambda: subprocess.run(
+ [sys.executable, "-m", "hermes_cli.main", "gateway", "stop"], cwd=nh.project, env=env,
+ capture_output=True, text=True, timeout=60, check=False))
+ return nh
+
+
+def test_vertex_config_label_and_background_task_through_the_gateway(tmp_path: Path) -> None:
+ fake = FakeVertex(tmp_path / "fake", project=PROJECT, region=REGION, sa_project=SA_EMBEDDED_PROJECT,
+ script=_respond, aux=lambda _r: Say("Vertex title"))
+ fake.start()
+ try:
+ nh = _drive(fake, tmp_path)
+ finally:
+ fake.stop() # independent of the CLI cleanup inside _drive, which may itself raise
bg = [r for r in fake.requests if BG_MARK in _user_text(r.get("body") or {})]
assert bg and all(r.get("status") == 200 and not r.get("rejected") for r in bg), bg