Skip to content

The agent that grows with you

test(e2e): fail, not skip, when the CI e2e lane has no tmux

Tekniumce53cae6ca32
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 075a6c05405..9a126213c8b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -188,6 +188,15 @@ jobs:
rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
rg --version
+ - name: Install tmux
+ # tests/e2e/core/cli_tmux and the Vertex /bg CLI suite drive real CLIs in tmux. The runner
+ # image ships it today; install it if it ever stops, and require_tmux() fails (not skips)
+ # without it under HERMES_E2E_REQUIRE_TUI=1, so the lane can't go green over zero coverage.
+ run: |
+ set -euo pipefail
+ command -v tmux >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq tmux; }
+ tmux -V
+
- name: Require a WAL-capable SQLite
# The state.db suites skip on a WAL-reset-vulnerable SQLite; fail
# instead of reporting green over zero coverage.
diff --git a/tests/ci/test_e2e_tmux_required.py b/tests/ci/test_e2e_tmux_required.py
new file mode 100644
index 00000000000..badd3c4fcda
--- /dev/null
+++ b/tests/ci/test_e2e_tmux_required.py
@@ -0,0 +1,62 @@
+"""The release/dispatch E2E lane fails, not skips, when the classic-CLI tmux suites lose tmux.
+
+The tmux suites (``tests/e2e/core/cli_tmux`` and the Vertex ``/bg`` CLI test) skip without tmux,
+which is right on a developer box. In the CI ``e2e`` job a skip would turn the lane green over zero
+coverage if the runner image ever dropped tmux. That job sets ``HERMES_E2E_REQUIRE_TUI=1`` (and
+GitHub sets ``CI``); under both, a missing tmux is a failure.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+
+from tests.e2e.core.cli_tmux import _classic
+
+_REPO = Path(__file__).resolve().parents[2]
+_TMUX_SUITES = sorted(
+ [*(_REPO / "tests" / "e2e" / "core" / "cli_tmux").glob("test_*.py"),
+ _REPO / "tests" / "e2e" / "core" / "providers" / "test_native_vertex_cli_background.py"]
+)
+_BARE_SKIP = re.compile(r"skipif\(\s*shutil\.which\(\s*[\"']tmux[\"']\s*\)")
+
+
+def _no_tmux(monkeypatch, *, ci: str | None, require: str | None) -> None:
+ monkeypatch.setattr(_classic.shutil, "which", lambda name: None if name == "tmux" else "/bin/x")
+ for name, value in (("CI", ci), ("HERMES_E2E_REQUIRE_TUI", require)):
+ if value is None:
+ monkeypatch.delenv(name, raising=False)
+ else:
+ monkeypatch.setenv(name, value)
+
+
+def test_missing_tmux_fails_in_the_ci_e2e_lane(monkeypatch):
+ _no_tmux(monkeypatch, ci="true", require="1")
+ with pytest.raises(pytest.fail.Exception, match="tmux"):
+ _classic.require_tmux()
+
+
[email protected](("ci", "require"), [(None, None), ("true", None), (None, "1")])
+def test_missing_tmux_skips_outside_the_ci_e2e_lane(monkeypatch, ci, require):
+ _no_tmux(monkeypatch, ci=ci, require=require)
+ with pytest.raises(pytest.skip.Exception):
+ _classic.require_tmux()
+
+
+def test_every_tmux_suite_uses_the_gate_instead_of_a_bare_skip():
+ assert len(_TMUX_SUITES) >= 4, _TMUX_SUITES
+ bare = [str(p.relative_to(_REPO)) for p in _TMUX_SUITES if _BARE_SKIP.search(p.read_text(encoding="utf-8"))]
+ assert not bare, f"skip over zero coverage in the e2e lane: {bare}"
+ ungated = [str(p.relative_to(_REPO)) for p in _TMUX_SUITES if "require_tmux()" not in p.read_text(encoding="utf-8")]
+ assert not ungated, ungated
+
+
+def test_the_e2e_job_requires_tmux():
+ yaml = pytest.importorskip("hermes_yaml")
+ job = yaml.safe_load((_REPO / ".github" / "workflows" / "tests.yml").read_text(encoding="utf-8"))["jobs"]["e2e"]
+ run_step = next(s for s in job["steps"] if s.get("name") == "Run e2e tests")
+ assert run_step["env"]["HERMES_E2E_REQUIRE_TUI"] == "1"
+ assert any("tmux" in s.get("run", "") and "tmux -V" in s.get("run", "") for s in job["steps"]), \
+ "the e2e job must install/verify tmux before the tmux suites run"
diff --git a/tests/e2e/core/cli_tmux/_classic.py b/tests/e2e/core/cli_tmux/_classic.py
index a31cf86fd71..028e20e6d5e 100644
--- a/tests/e2e/core/cli_tmux/_classic.py
+++ b/tests/e2e/core/cli_tmux/_classic.py
@@ -30,6 +30,7 @@ import json
import os
import re
import shlex
+import shutil
import signal
import subprocess
import sys
@@ -39,8 +40,23 @@ from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable
+import pytest
+
from tests.fakes.fake_llm_provider import MODEL_ID, OFFER_VERSION
+
+def require_tmux() -> None:
+ """Skip without tmux, except in the CI E2E lane, where a skip is green over zero coverage.
+
+ The release/dispatch ``e2e`` job (``.github/workflows/tests.yml``) sets
+ ``HERMES_E2E_REQUIRE_TUI=1`` and GitHub sets ``CI``; under both, a missing tmux fails.
+ """
+ if shutil.which("tmux") is not None:
+ return
+ if os.environ.get("CI") and os.environ.get("HERMES_E2E_REQUIRE_TUI") == "1":
+ pytest.fail("tmux missing in the CI e2e lane (HERMES_E2E_REQUIRE_TUI=1): install it in the e2e job")
+ pytest.skip("needs tmux")
+
REPO_ROOT = Path(__file__).resolve().parents[4]
COLS, ROWS = 160, 50
PROMPT = "\u276f"
diff --git a/tests/e2e/core/cli_tmux/test_classic_cli_live_session.py b/tests/e2e/core/cli_tmux/test_classic_cli_live_session.py
index bc55ec7bef4..d358a738dc7 100644
--- a/tests/e2e/core/cli_tmux/test_classic_cli_live_session.py
+++ b/tests/e2e/core/cli_tmux/test_classic_cli_live_session.py
@@ -29,7 +29,7 @@ from typing import Any, Callable
import pytest
-from tests.e2e.core.cli_tmux._classic import ClassicHome, Pane, last_user, new_needle, text_of
+from tests.e2e.core.cli_tmux._classic import ClassicHome, Pane, last_user, new_needle, require_tmux, text_of
from tests.fakes.fake_llm_provider import FakeLLMServer, Text, ToolCall
# Process discovery, cleanup and the witness's pid accounting read /proc.
@@ -174,8 +174,8 @@ def test_witness_coverage_rejects_an_uninstrumented_cli() -> None:
check_witness_coverage([gateway], [])
[email protected](shutil.which("tmux") is None, reason="needs tmux")
def test_classic_cli_live_session_runs_every_turn_in_the_gateway(tmp_path: Path) -> None:
+ require_tmux()
llm = FakeLLMServer(_main, aux=_aux)
witness = tmp_path / "witness.jsonl"
ch = ClassicHome(tmp_path / "run")
diff --git a/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.py b/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.py
index f43316e18c5..a5a99c258ba 100644
--- a/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.py
+++ b/tests/e2e/core/cli_tmux/test_classic_cli_main_parity.py
@@ -25,12 +25,11 @@ from typing import Any
import pytest
-from tests.e2e.core.cli_tmux._classic import PROMPT, ClassicHome, Pane, assert_no_traceback, last_user
+from tests.e2e.core.cli_tmux._classic import PROMPT, ClassicHome, Pane, assert_no_traceback, last_user, require_tmux
from tests.fakes.fake_llm_provider import FakeLLMServer, Text, ToolCall
pytestmark = [
pytest.mark.platforms("linux"), # the harness finds and reaps processes through /proc
- pytest.mark.skipif(shutil.which("tmux") is None, reason="needs tmux"),
]
MARKDOWN_REPLY = ("# Heading Two\n\nSome intro text for the markdown check.\n\n- alpha item\n- beta item\n"
@@ -107,6 +106,7 @@ class Steps:
def test_classic_cli_matches_main_baseline_across_fifteen_steps(tmp_path: Path) -> None:
+ require_tmux()
llm = FakeLLMServer(_main, aux=_aux)
ch = ClassicHome(tmp_path / "run")
pane = Pane(ch)
diff --git a/tests/e2e/core/cli_tmux/test_resize_scrollback.py b/tests/e2e/core/cli_tmux/test_resize_scrollback.py
index 34af39567e9..6428542000b 100644
--- a/tests/e2e/core/cli_tmux/test_resize_scrollback.py
+++ b/tests/e2e/core/cli_tmux/test_resize_scrollback.py
@@ -25,10 +25,9 @@ from pathlib import Path
import pytest
+from tests.e2e.core.cli_tmux._classic import require_tmux
from tests.fakes.fake_llm_provider import FakeLLMServer, Text, write_hermes_home
-pytestmark = pytest.mark.skipif(shutil.which("tmux") is None, reason="needs tmux")
-
REPO_ROOT = Path(__file__).resolve().parents[4]
WORDS = {1: 60, 2: 260, 3: 130}
@@ -39,6 +38,7 @@ def _reply(turn: int) -> str:
def test_resizes_keep_each_transcript_line_once_in_tmux_scrollback(tmp_path: Path) -> None:
+ require_tmux()
sock = f"hermes-e2e-{uuid.uuid4().hex[:8]}"
home = tmp_path / "home"
(tmp_path / "work").mkdir()
diff --git a/tests/e2e/core/providers/test_native_vertex_cli_background.py b/tests/e2e/core/providers/test_native_vertex_cli_background.py
index 96acaa97eb4..c8f5cf925cd 100644
--- a/tests/e2e/core/providers/test_native_vertex_cli_background.py
+++ b/tests/e2e/core/providers/test_native_vertex_cli_background.py
@@ -28,13 +28,12 @@ import pytest
pytest.importorskip("google.auth", reason="Vertex minting needs google-auth (CI installs it)")
-from tests.e2e.core.cli_tmux._classic import kill_home_processes
+from tests.e2e.core.cli_tmux._classic import kill_home_processes, require_tmux
from tests.e2e.core.providers._native_helpers import make_home, messages
from tests.fakes.providers.vertex import PROJECT, REGION, SA_EMBEDDED_PROJECT, FakeVertex, Say, hermes_setup
pytestmark = [
pytest.mark.platforms("linux"), # cleanup finds this home's processes through /proc
- pytest.mark.skipif(shutil.which("tmux") is None, reason="needs tmux"),
]
BG_MARK = "VERTEXBG"
@@ -115,6 +114,7 @@ def _drive(fake: FakeVertex, tmp_path: Path):
def test_vertex_config_label_and_background_task_through_the_gateway(tmp_path: Path) -> None:
+ require_tmux()
fake = FakeVertex(tmp_path / "fake", project=PROJECT, region=REGION, sa_project=SA_EMBEDDED_PROJECT,
script=_respond, aux=lambda _r: Say("Vertex title"))
fake.start()