The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
test(e2e): fold the 15-step classic-CLI main comparison into the E2E suite
Tekniumcfae1548a9eb
diff --git a/tests/e2e/core/cli_tmux/_classic.py b/tests/e2e/core/cli_tmux/_classic.pynew file mode 100644index 00000000000..a499eb8a2db--- /dev/null+++ b/tests/e2e/core/cli_tmux/_classic.py@@ -0,0 +1,244 @@+"""Shared harness for the classic-CLI tmux suites (``display.interface: cli``).++A real ``python -m hermes_cli.main`` runs in a private tmux server (own ``-L`` socket) with a+hermetic fake HOME / HERMES_HOME and a config that routes the model to a loopback+``FakeLLMServer``. The CLI is a thin client of the profile's gateway daemon, which it spawns on+first connect; cleanup stops that daemon and then kills anything left whose environ carries this+run's exact ``HERMES_HOME`` (never a pattern kill).++Driver rule (the CLI reads an Enter processed within 50 ms of the last buffer change as a pasted+newline): type with ``send-keys -l``, wait until the composer shows the text, pause, then send+``Enter`` separately. Each step waits for its needle, then until the screen (digits masked) has+been static for ``quiet`` seconds.+"""++from __future__ import annotations++import json+import os+import re+import shlex+import signal+import subprocess+import sys+import time+import uuid+from dataclasses import dataclass, field+from pathlib import Path++from tests.fakes.fake_llm_provider import MODEL_ID, OFFER_VERSION++REPO_ROOT = Path(__file__).resolve().parents[4]+COLS, ROWS = 160, 50+PROMPT = "\u276f"+STATUS_BAR = f"\u2624 {MODEL_ID}"+_DIGITS = re.compile(r"\d")+_TRACEBACK = "Traceback (most recent call last)"++BASE_CONFIG = f"""model:+ provider: custom+ base_url: {{base_url}}+ default: {MODEL_ID}+ context_length: 128000+agent:+ api_max_retries: 1+telemetry:+ shared_metrics:+ enabled: false+ send: false+ offer_version: {OFFER_VERSION}+display:+ interface: cli+approvals:+ mode: manual+ timeout: 60+skills:+ creation_nudge_interval: 0+updates:+ check: false+models_dev:+ url: http://127.0.0.1:9/api.json+auxiliary:+ title_generation:+ enabled: false+"""+++@dataclass+class ClassicHome:+ root: Path+ extra_env: dict[str, str] = field(default_factory=dict)++ @property+ def home(self) -> Path:+ return self.root / "home"++ @property+ def hermes_home(self) -> Path:+ return self.home / ".hermes"++ @property+ def work(self) -> Path:+ return self.root / "work"++ @property+ def stderr_log(self) -> Path:+ return self.root / "stderr.log"++ def write(self, base_url: str, extra_config: str = "") -> ClassicHome:+ self.hermes_home.mkdir(parents=True, exist_ok=True)+ self.work.mkdir(parents=True, exist_ok=True)+ (self.hermes_home / "config.yaml").write_text(BASE_CONFIG.format(base_url=base_url) + extra_config,+ encoding="utf-8")+ (self.hermes_home / ".env").write_text("OPENAI_API_KEY=sk-fake-e2e\n", encoding="utf-8")+ return self++ def env(self) -> dict[str, str]:+ """Allowlisted child env: no inherited HERMES_*, provider keys, display or TMUX variables."""+ keep = ("PATH", "LANG", "USER", "LOGNAME", "SHELL", "TMPDIR")+ env = {k: os.environ[k] for k in keep if k in os.environ}+ env.update({+ "LC_ALL": "C.UTF-8", "LANG": "C.UTF-8", "TERM": "xterm-256color", "TZ": "UTC",+ "HOME": str(self.home), "HERMES_HOME": str(self.hermes_home), "PYTHONPATH": str(REPO_ROOT),+ # The child's state.db IS this tmp home's db; the live-DB guard would refuse it under pytest.+ "HERMES_STATE_DB_GUARD_BYPASS": "1", "PYTHONFAULTHANDLER": "1", "NO_GCE_CHECK": "True",+ })+ env.update(self.extra_env)+ return env++ def argv(self, *args: str) -> list[str]:+ return [sys.executable, "-m", "hermes_cli.main", *args]++ def run(self, *args: str, timeout: float = 180.0) -> subprocess.CompletedProcess:+ """A non-TTY run (stdout piped): ``chat -q`` one-shots, ``gateway stop``."""+ return subprocess.run(self.argv(*args), cwd=self.work, env=self.env(), capture_output=True, text=True,+ timeout=timeout, stdin=subprocess.DEVNULL, check=False)++ def pids(self) -> set[int]:+ needle = f"HERMES_HOME={self.hermes_home}".encode()+ found: set[int] = set()+ for entry in os.listdir("/proc"):+ if not entry.isdigit() or int(entry) == os.getpid():+ continue+ try:+ with open(f"/proc/{entry}/environ", "rb") as fh:+ if needle in fh.read().split(b"\0"):+ found.add(int(entry))+ except OSError:+ continue+ return found++ def reap(self) -> None:+ """Operator stop for the profile's gateway, then SIGTERM/SIGKILL whatever this home still runs."""+ self.run("gateway", "stop", timeout=60)+ for sig, grace in ((signal.SIGTERM, 8.0), (signal.SIGKILL, 2.0)):+ for pid in self.pids():+ try:+ os.kill(pid, sig)+ except OSError:+ pass+ end = time.monotonic() + grace+ while self.pids() and time.monotonic() < end:+ time.sleep(0.2)+++class Pane:+ """One private tmux server holding the CLI pane ``p``."""++ def __init__(self, ch: ClassicHome) -> None:+ self.ch = ch+ self.sock = f"hermes-e2e-classic-{uuid.uuid4().hex[:8]}"++ def tmux(self, *args: str) -> str:+ env = {k: v for k, v in os.environ.items() if not k.startswith("TMUX")}+ return subprocess.run(["tmux", "-L", self.sock, *args], capture_output=True, text=True, encoding="utf-8",+ errors="replace", timeout=30, check=False, env=env).stdout++ def launch(self, *args: str) -> None:+ self.tmux("kill-server")+ env = self.ch.env()+ envs = " ".join(f"{k}={shlex.quote(v)}" for k, v in env.items())+ cmd = (f"exec env -i {envs} {' '.join(shlex.quote(a) for a in self.ch.argv(*args))} "+ f"2>>{shlex.quote(str(self.ch.stderr_log))}")+ subprocess.run(["tmux", "-L", self.sock, "-f", os.devnull, "new-session", "-d", "-s", "p", "-x", str(COLS),+ "-y", str(ROWS), "-c", str(self.ch.work), "sh", "-c", cmd], check=True, timeout=30,+ env={k: v for k, v in os.environ.items() if not k.startswith("TMUX")})+ self.tmux("set", "-g", "remain-on-exit", "on")+ self.tmux("set", "-g", "history-limit", "50000")++ def screen(self) -> str:+ return self.tmux("capture-pane", "-p", "-t", "p")++ def scrollback(self) -> str:+ return self.tmux("capture-pane", "-p", "-J", "-t", "p", "-S", "-", "-E", "-")++ def dead(self) -> bool:+ return self.tmux("display-message", "-p", "-t", "p", "#{pane_dead}").strip() == "1"++ def keys(self, *k: str) -> None:+ self.tmux("send-keys", "-t", "p", *k)++ def wait(self, needles: list[str], timeout: float = 60.0, *, full: bool = False) -> str | None:+ """The first needle seen (screen, or the whole scrollback with ``full``), None on timeout/exit."""+ end = time.monotonic() + timeout+ while time.monotonic() < end:+ text = self.scrollback() if full else self.screen()+ for needle in needles:+ if needle in text:+ return needle+ if self.dead():+ return None+ time.sleep(0.25)+ return None++ def settle(self, quiet: float = 2.0, timeout: float = 20.0) -> None:+ end = time.monotonic() + timeout+ last, since = None, time.monotonic()+ while time.monotonic() < end:+ text = _DIGITS.sub("#", self.screen())+ if text != last:+ last, since = text, time.monotonic()+ elif time.monotonic() - since >= quiet:+ return+ time.sleep(0.2)++ def submit(self, text: str) -> None:+ self.tmux("send-keys", "-t", "p", "-l", text)+ self.wait([f"{PROMPT} {text}"[-40:]], 10)+ time.sleep(1.2)+ self.keys("Enter")++ def wait_dead(self, timeout: float = 30.0) -> bool:+ end = time.monotonic() + timeout+ while time.monotonic() < end and not self.dead():+ time.sleep(0.3)+ return self.dead()++ def ready(self, timeout: float = 120.0) -> str | None:+ """Wait for the running app's status bar (painted in raw mode), then let startup settle."""+ hit = self.wait([STATUS_BAR], timeout)+ self.settle(quiet=3.0, timeout=30)+ time.sleep(1.5)+ return hit++ def kill(self) -> None:+ self.tmux("kill-server")+++def assert_no_traceback(text: str, where: str) -> None:+ assert _TRACEBACK not in text, f"{where}: a traceback reached the terminal:\n{text[-4000:]}"+++def text_of(content) -> str:+ if isinstance(content, str):+ return content+ if isinstance(content, list):+ return " ".join(str(p.get("text", "")) if isinstance(p, dict) else str(p) for p in content)+ return json.dumps(content)+++def last_user(body: dict) -> str:+ for message in reversed(body.get("messages") or []):+ if message.get("role") == "user":+ return text_of(message.get("content"))+ return ""diff --git a/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.py b/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.pynew file mode 100644index 00000000000..6aec8968a77--- /dev/null+++ b/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.py@@ -0,0 +1,214 @@+"""The classic CLI keeps main's behaviour under the one gateway: the 15-step main-vs-branch comparison.++Ported from the local A/B harness used on #106742 (#134889 T24). That harness ran one scripted+session on a ``main`` tree and on the rewire branch in tmux and diffed the normalized screens;+its MAIN-vs-MAIN self-check was 15/15 identical. With the rewire merged there is no second tree to+diff, so each step here asserts what main's capture showed (the pre-rewire baseline) against the+current checkout, where the gateway daemon runs the agent, tools and session writes.++One real ``hermes`` (``display.interface: cli``) session in a private tmux server, loopback+``FakeLLMServer`` keyed on the ``STEPn`` marker in the latest user message (zero spend):++01 startup · 02 markdown · 03a approval modal · 03b approve once → tool reply · 04 read_file ·+05 /help · 06a /model picker · 06b /title · 07 /usage · 08 Ctrl+C mid-stream · 09 /compress ·+10a /quit exit summary · 10b ``-c`` resume recap · 11a ``chat -q`` on a TTY · 11b ``chat -q`` piped.+"""++from __future__ import annotations++import json+import shutil+import sqlite3+import time+from pathlib import Path+from typing import Any++import pytest++from tests.e2e.core.cli_tmux._classic import PROMPT, ClassicHome, Pane, assert_no_traceback, last_user+from tests.fakes.fake_llm_provider import FakeLLMServer, Text, ToolCall++pytestmark = pytest.mark.skipif(shutil.which("tmux") is None, reason="needs tmux")++MARKDOWN_REPLY = ("# Heading Two\n\nSome intro text for the markdown check.\n\n- alpha item\n- beta item\n"+ "- gamma item\n\n```python\ndef hello():\n return 'hi'\n```\n\nEnd of STEP2 reply.")+# Flagged "shell command via -c/-lc flag" -> approval modal. The seq makes an EARLY tool result bulky,+# so /compress has a middle worth summarizing.+STEP3_CMD = 'bash -c "echo hi && seq 1 2500"'+STEP3_DONE = "STEP3 done: the command printed hi."+STEP4_DONE = "STEP4 done: notes.txt holds the fixture lines."+LONG_REPLY = "STEP8 story: " + " ".join(f"w{i:03d}" for i in range(600)) + " THE-END"+HELLO_REPLY = "STEP11 hello back from the fake model."+TITLE = "Some Title"++EXTRA_CONFIG = """memory:+ memory_enabled: false+ user_profile_enabled: false+ nudge_interval: 0+compression:+ threshold_tokens: 30000+ protect_last_n: 2+ min_tail_user_messages: 1+"""+++def _main(record: dict[str, Any]):+ body = record.get("body") or {}+ msgs = body.get("messages") or []+ after_tool = bool(msgs) and msgs[-1].get("role") == "tool"+ user = last_user(body)+ if "STEP2" in user:+ return Text(MARKDOWN_REPLY)+ if "STEP3" in user:+ return Text(STEP3_DONE) if after_tool else ToolCall("terminal", {"command": STEP3_CMD})+ if "STEP4" in user:+ return Text(STEP4_DONE) if after_tool else ToolCall("read_file", {"path": "notes.txt"})+ if "STEP8" in user:+ return Text(LONG_REPLY, chunk_chars=6, delay_per_chunk=0.04)+ if "hello" in user.lower():+ return Text(HELLO_REPLY)+ return Text("ok")+++def _aux(record: dict[str, Any]):+ blob = json.dumps((record.get("body") or {}).get("messages") or [])+ return Text("Scripted session title" if "You name chat sessions" in blob else "Fake summary of the earlier conversation.")+++def _session_rows(ch: ClassicHome) -> list[dict[str, Any]]:+ conn = sqlite3.connect(f"file:{ch.hermes_home / 'state.db'}?mode=ro", uri=True, timeout=10)+ conn.row_factory = sqlite3.Row+ try:+ return [dict(r) for r in conn.execute("SELECT id, title, message_count FROM sessions ORDER BY started_at")]+ finally:+ conn.close()+++class Steps:+ """Runs the steps in order, keeping each step's screen; a failed step names itself."""++ def __init__(self, pane: Pane) -> None:+ self.pane = pane+ self.screens: dict[str, str] = {}++ def capture(self, name: str, *, scrollback: bool = False) -> str:+ text = self.pane.scrollback() if scrollback else self.pane.screen()+ self.screens[name] = text+ assert_no_traceback(text, name)+ return text++ def expect(self, name: str, needles: list[str], text: str) -> None:+ missing = [n for n in needles if n not in text]+ assert not missing, f"step {name}: main showed {missing!r}; screen now:\n{text[-4000:]}"+++def test_classic_cli_matches_main_baseline_across_fifteen_steps(tmp_path: Path) -> None:+ llm = FakeLLMServer(_main, aux=_aux)+ ch = ClassicHome(tmp_path / "run")+ pane = Pane(ch)+ s = Steps(pane)+ with llm:+ ch.write(llm.base_url, EXTRA_CONFIG)+ (ch.work / "notes.txt").write_text("fixture line one\nfixture line two\nfixture line three\n", encoding="utf-8")+ try:+ pane.launch()+ assert pane.ready(), f"01_startup: no status bar:\n{pane.screen()}"+ s.expect("01_startup", ["Welcome to Hermes Agent!", "Session:", PROMPT], s.capture("01_startup", scrollback=True))++ pane.submit("STEP2 please show me some markdown")+ assert pane.wait(["End of STEP2 reply."], 60), pane.screen()+ pane.settle()+ md = s.capture("02_markdown")+ # Rendered through Rich inside main's reply box, not raw markdown.+ s.expect("02_markdown", ["\u2624 Hermes", "Heading Two", "- alpha item", "def hello():"], md)+ assert "# Heading Two" not in md and "```" not in md, f"02_markdown: raw markdown:\n{md}"++ pane.submit("STEP3 please run echo hi in the terminal")+ assert pane.wait(["Dangerous Command"], 45), f"03a_approval_modal: no modal:\n{pane.screen()}"+ pane.settle(quiet=1.5, timeout=10)+ s.expect("03a_approval_modal", ["Dangerous Command", STEP3_CMD, "1. Allow once", "4. Deny"],+ s.capture("03a_approval_modal"))+ pane.keys("Enter") # main's preselected first choice: Allow once+ assert pane.wait([STEP3_DONE], 60), pane.screen()+ pane.settle()+ s.expect("03b_tool_reply", ["\u2192 allowed once", "\u2624 Hermes", STEP3_DONE], s.capture("03b_tool_reply"))++ pane.submit("STEP4 please read notes.txt")+ assert pane.wait([STEP4_DONE], 60), pane.screen()+ pane.settle()+ s.expect("04_read_file", ["read", "notes.txt", STEP4_DONE], s.capture("04_read_file"))++ pane.submit("/help")+ pane.settle(quiet=2.0, timeout=20)+ s.expect("05_help", ["/quit", "Exit the CLI", "/help skills"], s.capture("05_help"))++ pane.submit("/model")+ assert pane.wait(["Model Picker"], 30), f"06a_model: no picker:\n{pane.screen()}"+ pane.settle(quiet=1.5, timeout=10)+ s.expect("06a_model", ["Model Picker", "Current: fake-model"], s.capture("06a_model"))+ pane.keys("Escape")+ time.sleep(1.0)+ pane.settle(quiet=1.0, timeout=5)++ pane.submit(f"/title {TITLE}")+ assert pane.wait([f"Session title set: {TITLE}"], 30), pane.screen()+ s.capture("06b_title")++ pane.submit("/usage")+ assert pane.wait(["Session Token Usage"], 30), pane.screen()+ pane.settle()+ s.expect("07_usage", ["Session Token Usage", "Model:", "fake-model", "API calls:", "Messages:",+ "Compressions:"], s.capture("07_usage"))++ pane.submit("STEP8 tell me a very long story")+ assert pane.wait(["w040"], 60), pane.screen()+ pane.keys("C-c")+ # The interrupt round-trips through the gateway; the notice lands when the turn ends.+ interrupted = pane.wait(["Interrupted"], 45, full=True)+ pane.settle(quiet=2.0, timeout=30)+ text = s.capture("08_interrupt", scrollback=True)+ assert not pane.dead(), f"08_interrupt: Ctrl+C killed the CLI (main stays at the prompt):\n{text[-3000:]}"+ assert "THE-END" not in text, "08_interrupt: the turn was not interrupted"+ assert interrupted, f"08_interrupt: main showed an 'Interrupted' notice; screen now:\n{text[-4000:]}"++ pane.submit("/compress")+ assert pane.wait(["Compressed:", "No changes from compression"], 90), pane.screen()+ pane.settle(quiet=3.0, timeout=60)+ s.expect("09_compress", ["Compressing", "Compressed:"], s.capture("09_compress"))++ pane.submit("/quit")+ assert pane.wait_dead(30), f"10a_exit_summary: CLI alive 30 s after /quit:\n{pane.screen()}"+ summary = s.capture("10a_exit_summary", scrollback=True).split("/quit")[-1]+ s.expect("10a_exit_summary", ["Resume this session with:", "hermes --resume ", f'hermes -c "{TITLE}"',+ "Session:", f"Title: {TITLE}", "Duration:", "Messages:"], summary)++ pane.launch("-c")+ assert pane.ready(), f"10b_resume_recap: no prompt:\n{pane.scrollback()[-3000:]}"+ recap = s.capture("10b_resume_recap", scrollback=True)+ s.expect("10b_resume_recap", ["Resumed session", TITLE, "Previous Conversation",+ "STEP2 please show me some markdown"], recap)+ assert "END OF CONTEXT SUMMARY" not in recap, "10b: the raw compaction summary was dumped"+ pane.submit("/quit")+ pane.wait_dead(30)++ pane.launch("chat", "-q", "hello")+ assert pane.wait([HELLO_REPLY], 120), f"11a_q_tty:\n{pane.screen()}"+ pane.settle(quiet=3.0, timeout=30)+ s.expect("11a_q_tty", ["\u2624 Hermes", HELLO_REPLY, PROMPT], s.capture("11a_q_tty"))+ pane.kill()++ piped = ch.run("chat", "-q", "hello")+ assert_no_traceback(piped.stdout + piped.stderr, "11b_q_pipe")+ assert piped.returncode == 0, piped.stderr[-3000:]+ s.expect("11b_q_pipe", ["Query: hello", HELLO_REPLY, "Resume this session with:", "Session:",+ "Messages:"], piped.stdout)+ finally:+ pane.kill()+ ch.reap()++ # Zero spend and one writer: every model call went to the loopback fake, and the session row+ # the gateway wrote carries the title and the conversation the screens showed.+ assert llm.main_requests(), "no request reached the fake provider"+ rows = _session_rows(ch)+ titled = [r for r in rows if r["title"] == TITLE]+ assert titled and titled[0]["message_count"] >= 6, rows