Skip to content

HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.

test(runner): the OSS runner shipped with no regression coverage of its own

richard-epsillafb2dbf0435be
diff --git a/runner/tests/test_claude_error_filter.py b/runner/tests/test_claude_error_filter.py
new file mode 100644
index 00000000..4d769dad
--- /dev/null
+++ b/runner/tests/test_claude_error_filter.py
@@ -0,0 +1,56 @@
+"""Claude Code injects transient provider errors into its stream AS assistant text
+("API Error: 400 ..."), then retries. That diagnostic is CLI UX, not model output — rendering it
+as the reply made a working opus-4.7/4.8 turn look failed. _claude_passthrough must drop those
+error-only assistant messages and strip the error block from mixed ones, without touching real
+replies, results, or other event types."""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import server as rn # noqa: E402
+
+
+def _asst(*texts):
+ return {"type": "assistant", "message": {"content": [{"type": "text", "text": t} for t in texts]}}
+
+
+def test_pure_api_error_message_is_dropped():
+ for code in ("400", "429", "500", "529"):
+ ev = _asst(f'API Error: {code} "..enabled" is not supported for this model.')
+ assert rn._claude_passthrough(ev, {}) == [], f"{code} error should render nothing"
+
+
+def test_error_stripped_but_real_text_kept():
+ ev = _asst("API Error: 429 rate limited", "Hello! How can I help?")
+ out = rn._claude_passthrough(ev, {})
+ assert len(out) == 1
+ kept = out[0]["message"]["content"]
+ assert kept == [{"type": "text", "text": "Hello! How can I help?"}]
+
+
+def test_normal_reply_untouched():
+ ev = _asst("Here is your answer.")
+ assert rn._claude_passthrough(ev, {}) == [ev]
+
+
+def test_tool_use_block_preserved_even_with_error_text():
+ ev = {"type": "assistant", "message": {"content": [
+ {"type": "text", "text": "API Error: 400 bad thinking param"},
+ {"type": "tool_use", "id": "t1", "name": "bash", "input": {"cmd": "ls"}}]}}
+ out = rn._claude_passthrough(ev, {})
+ assert out and any(c.get("type") == "tool_use" for c in out[0]["message"]["content"])
+ assert all(not (c.get("type") == "text") for c in out[0]["message"]["content"])
+
+
+def test_result_and_other_events_pass_through():
+ res = {"type": "result", "subtype": "success", "result": "done", "is_error": False}
+ assert rn._claude_passthrough(res, {}) == [res]
+ init = {"type": "system", "subtype": "init", "session_id": "s1"}
+ assert rn._claude_passthrough(init, {}) == [init]
+
+
+def test_non_error_text_that_merely_mentions_api_error_is_kept():
+ # The guard anchors on a leading "API Error: <3-digit code>" — prose that happens to discuss
+ # errors must not be stripped.
+ ev = _asst("To handle an API error, catch the 400 status and retry.")
+ assert rn._claude_passthrough(ev, {}) == [ev]
diff --git a/runner/tests/test_codex_reasoning_strip.py b/runner/tests/test_codex_reasoning_strip.py
new file mode 100644
index 00000000..3819f6c4
--- /dev/null
+++ b/runner/tests/test_codex_reasoning_strip.py
@@ -0,0 +1,82 @@
+"""Guard: codex resume must strip account-bound encrypted reasoning from the rollout.
+
+Regression this pins: a Responses `reasoning` item carries `encrypted_content` (the `gAAA…`
+chain-of-thought blob) that only the upstream account that minted it can decrypt. Codex replays the
+whole rollout as the next turn's input, so a follow-up served by a DIFFERENT account 400s with
+`invalid_encrypted_content` — and the orphaned item stays in the rollout, wedging every later resume.
+`_strip_codex_encrypted_reasoning` removes reasoning items before resume; messages, tool calls, and
+tool outputs must survive untouched so history is preserved.
+"""
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import server as rn # noqa: E402
+
+
+def _rollout(tmp_path) -> Path:
+ """A minimal rollout mirroring the real codex jsonl shape (see a live rollout-*.jsonl)."""
+ lines = [
+ {"type": "session_meta", "payload": {"id": "s1"}},
+ {"type": "response_item", "payload": {"type": "message", "role": "user",
+ "content": [{"type": "input_text", "text": "hi"}]}},
+ {"type": "response_item", "payload": {"type": "reasoning", "id": "rs_1", "summary": [],
+ "encrypted_content": "gAAAAAoneXXX"}},
+ {"type": "response_item", "payload": {"type": "function_call", "call_id": "c1",
+ "name": "shell", "arguments": "{}"}},
+ {"type": "response_item", "payload": {"type": "function_call_output", "call_id": "c1",
+ "output": "ok"}},
+ {"type": "event_msg", "payload": {"type": "agent_message", "message": "done"}},
+ {"type": "response_item", "payload": {"type": "reasoning", "id": "rs_2", "summary": [],
+ "encrypted_content": "gAAAAAtwoYYY"}},
+ # a failed-turn telemetry record whose ERROR STRING mentions encrypted content — must be kept
+ {"type": "event_msg", "payload": {"type": "task_complete",
+ "error": {"message": "The encrypted content gAAA...= could not be verified"}}},
+ ]
+ p = tmp_path / "rollout.jsonl"
+ p.write_text("\n".join(json.dumps(x) for x in lines) + "\n")
+ return p
+
+
+def test_strip_removes_reasoning_items_only(tmp_path):
+ p = _rollout(tmp_path)
+ n = rn._strip_codex_encrypted_reasoning([str(p)])
+ assert n == 2 # both reasoning items removed
+
+ kept = [json.loads(l) for l in p.read_text().splitlines() if l.strip()]
+ types = [(o.get("type"), (o.get("payload") or {}).get("type")) for o in kept]
+ assert ("response_item", "reasoning") not in types
+ # everything else survives, in order
+ assert types == [
+ ("session_meta", None),
+ ("response_item", "message"),
+ ("response_item", "function_call"),
+ ("response_item", "function_call_output"),
+ ("event_msg", "agent_message"),
+ ("event_msg", "task_complete"),
+ ]
+ # the only remaining "encrypted_content" substring is inside the kept error-message telemetry
+ remaining = [o for o in kept if "encrypted_content" in json.dumps(o)]
+ assert remaining == [] # error msg text uses "encrypted content" (space), not the JSON key
+
+
+def test_strip_is_noop_without_reasoning(tmp_path):
+ p = tmp_path / "r.jsonl"
+ p.write_text(json.dumps({"type": "response_item",
+ "payload": {"type": "message", "role": "user", "content": []}}) + "\n")
+ before = p.read_text()
+ assert rn._strip_codex_encrypted_reasoning([str(p)]) == 0
+ assert p.read_text() == before # untouched when there is nothing to strip
+
+
+def test_strip_tolerates_missing_and_malformed(tmp_path):
+ good = tmp_path / "good.jsonl"
+ good.write_text(json.dumps({"type": "response_item",
+ "payload": {"type": "reasoning", "id": "r", "encrypted_content": "gAAAz"}}) + "\n"
+ + "not json but mentions reasoning and encrypted_content\n")
+ missing = tmp_path / "nope.jsonl"
+ n = rn._strip_codex_encrypted_reasoning([str(good), str(missing)])
+ assert n == 1
+ # the malformed line is preserved verbatim (never dropped on a parse failure)
+ assert "not json but mentions" in good.read_text()
diff --git a/runner/tests/test_codex_sandbox.py b/runner/tests/test_codex_sandbox.py
new file mode 100644
index 00000000..8aa012b7
--- /dev/null
+++ b/runner/tests/test_codex_sandbox.py
@@ -0,0 +1,57 @@
+"""Guard: codex must run WRITE-ENABLED in both the exec and app-server paths.
+
+The regression this pins (QA CT: "codex app-server sandbox_mode 退回只读"): codex `app-server`
+has no --dangerously-bypass flag and reads its policy from config.toml + the thread/start
+`sandbox` field. If either drifts off `danger-full-access` (or the exec path loses its bypass
+flag), codex mounts the workspace READ-ONLY and — with approval_policy "never" — silently rejects
+every apply_patch/file write, so the agent produces no files and gives up ("workspace is
+read-only"). That failure is invisible until a user runs a real turn. These are pure/deterministic
+assertions on the rendered config, so they catch the drift at CI time without a live sandbox.
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import server as rn # noqa: E402
+
+WRITE_MODE = "danger-full-access"
+
+
+def _render_config(tmp_path) -> str:
+ """Render config.toml exactly as a real turn would, via the shared codex setup."""
+ home = tmp_path / "home"
+ home.mkdir()
+ env = {"HOME": str(home)}
+ auth = rn.Auth(api_key="sk-test", base_url="https://example.invalid/v1", wire_api="responses")
+ cfg_dir = rn._codex_prepare_env("azure", auth, "gpt-5.5", str(tmp_path), env)
+ return (cfg_dir / "config.toml").read_text()
+
+
+def test_exec_path_config_is_write_enabled(tmp_path):
+ cfg = _render_config(tmp_path)
+ # The exec path's sandbox policy lives in config.toml. Read-only here == silent write failures.
+ assert f'sandbox_mode = "{WRITE_MODE}"' in cfg, (
+ "codex config.toml no longer renders danger-full-access — the exec path would mount the "
+ "workspace read-only and silently reject every file write"
+ )
+ assert 'approval_policy = "never"' in cfg # never escalate; only safe BECAUSE sandbox is full-access
+
+
+def test_appserver_default_sandbox_is_write_enabled():
+ # app-server has no bypass flag; the sandbox is set purely from this value in thread/start.
+ assert rn._CODEX_SANDBOX == WRITE_MODE, (
+ f"CODEX_APPSERVER_SANDBOX default drifted to {rn._CODEX_SANDBOX!r} — app-server turns would "
+ "run read-only. Only override to a write-enabled mode (danger-full-access / workspace-write)."
+ )
+
+
+def test_exec_path_passes_bypass_flag(tmp_path):
+ # `codex exec` disables its own sandbox ONLY via this flag; losing it re-enables read-only.
+ home = tmp_path / "home"
+ home.mkdir()
+ argv = rn._build_codex(
+ "azure",
+ rn.Auth(api_key="sk-test", base_url="https://example.invalid/v1", wire_api="responses"),
+ "gpt-5.5", "hello", str(tmp_path), {"HOME": str(home)},
+ )
+ assert "--dangerously-bypass-approvals-and-sandbox" in argv
diff --git a/runner/tests/test_hermes_hang.py b/runner/tests/test_hermes_hang.py
new file mode 100644
index 00000000..9048ef26
--- /dev/null
+++ b/runner/tests/test_hermes_hang.py
@@ -0,0 +1,149 @@
+"""Hermes driver hang guards.
+
+hermes-agent emits NO stdout event stream — the driver's only progress signal is what lands in
+state.db. Three failure modes this pins:
+
+1. A fresh turn whose subprocess blocks (e.g. an unbounded provider auth/model-init call) showed
+ zero events, no error, indistinguishable from "still working," for up to the full per-turn cap
+ (hours). _HERMES_STARTUP_TIMEOUT_S bounds just that startup window, independent of and much
+ shorter than the overall turn timeout.
+2. The guard must key on THE MODEL PRODUCING OUTPUT, not on a session row existing. hermes writes
+ the session and echoes the user's prompt before it ever calls the provider, so a guard keyed on
+ the session row disarms itself a fraction of a second in — which is how a hung glm-5.2 turn ran
+ to the six-hour cap with the console showing "Working…" the whole time.
+3. A resume whose target session isn't found in this sandbox silently fell back to a fresh start
+ with only a server-log line — no signal in the turn's own event stream. A caller that believed
+ this was a continuation had no way to detect the dropped context.
+"""
+import os
+import sqlite3
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import server as rn # noqa: E402
+
+
+def _fake_popen_spawning(real_cmd: list[str]):
+ """Replace rn.subprocess.Popen with one that ignores the requested argv and runs `real_cmd`
+ instead, keeping only the kwargs _run_hermes_bg actually relies on (pipes, cwd, env, session)."""
+ real_popen = subprocess.Popen
+ keep = ("cwd", "env", "text", "bufsize", "stdout", "stderr", "start_new_session")
+
+ def fake(cmd, **kw):
+ return real_popen(real_cmd, **{k: v for k, v in kw.items() if k in keep})
+ return fake
+
+
+def _state_db(path: Path, started_at: float, rows: list[tuple[str, str]]) -> str:
+ """A state.db shaped the way hermes writes one: a session plus its messages, in order."""
+ db = sqlite3.connect(path)
+ db.execute("CREATE TABLE sessions (id TEXT PRIMARY KEY, started_at REAL)")
+ db.execute("CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT, "
+ "role TEXT, content TEXT, tool_calls TEXT, tool_call_id TEXT)")
+ db.execute("INSERT INTO sessions VALUES ('sess-1', ?)", (started_at,))
+ for role, content in rows:
+ db.execute("INSERT INTO messages (session_id, role, content, tool_calls, tool_call_id) "
+ "VALUES ('sess-1', ?, ?, NULL, NULL)", (role, content))
+ db.commit(); db.close()
+ return str(path)
+
+
+def _ro_opener(db_file: str):
+ """Stand-in for _hermes_db_ro. row_factory matters: without it every row read raises inside
+ the sweep's broad except and the guard sees no output at all — which would make these tests
+ pass for the wrong reason."""
+ def open_ro(_path):
+ db = sqlite3.connect(f"file:{db_file}?mode=ro", uri=True)
+ db.row_factory = sqlite3.Row
+ return db
+ return open_ro
+
+
+def test_startup_timeout_kills_a_hung_fresh_turn(monkeypatch, tmp_path):
+ # No session row ever appears (simulates the CLI blocked before its first DB write).
+ monkeypatch.setattr(rn, "_hermes_db_ro", lambda db_path: None)
+ monkeypatch.setattr(rn, "_HERMES_STARTUP_TIMEOUT_S", 0.2)
+ monkeypatch.setattr(rn, "_HERMES_POLL_S", 0.05)
+ monkeypatch.setattr(rn, "MAX_TURN_SECONDS", 999) # must NOT be what stops this turn
+ monkeypatch.setattr(rn.subprocess, "Popen", _fake_popen_spawning(["sleep", "100"]))
+
+ turn_id = "startup-hang"
+ rn._turns[turn_id] = {"events": [], "done": False}
+ env = {**os.environ, "HERMES_HOME": str(tmp_path)}
+
+ t0 = time.time()
+ rn._run_hermes_bg(turn_id, str(tmp_path), env, "gpt-5.4", "azure", "hi", None, None)
+ elapsed = time.time() - t0
+
+ rec = rn._turns[turn_id]
+ assert elapsed < 5, "startup guard did not fire — turn ran to the full per-turn cap"
+ assert rec["status"] == "timeout"
+ assert rec.get("capped") is True
+ assert "produced no output" in (rec.get("error") or "")
+
+
+def test_startup_guard_survives_the_session_row_and_the_prompt_echo(monkeypatch, tmp_path):
+ # The regression that motivated the guard's current shape: hermes HAS written its session row
+ # and echoed the user's prompt, but the provider call behind it never returns. A guard keyed on
+ # the session row would have disarmed here and let the turn run to the six-hour cap.
+ started = time.time()
+ db_file = _state_db(tmp_path / "echo.db", started, [("user", "hi")])
+ monkeypatch.setattr(rn, "_hermes_db_ro", _ro_opener(db_file))
+ monkeypatch.setattr(rn, "_HERMES_STARTUP_TIMEOUT_S", 0.3)
+ monkeypatch.setattr(rn, "_HERMES_POLL_S", 0.05)
+ monkeypatch.setattr(rn, "MAX_TURN_SECONDS", 999)
+ monkeypatch.setattr(rn.subprocess, "Popen", _fake_popen_spawning(["sleep", "100"]))
+
+ rn._turns["echo-only"] = {"events": [], "done": False}
+ rn._run_hermes_bg("echo-only", str(tmp_path), {**os.environ, "HERMES_HOME": str(tmp_path)},
+ "gpt-5.4", "azure", "hi", None, None)
+
+ rec = rn._turns["echo-only"]
+ assert rec.get("capped") is True, "the prompt echo disarmed the guard — it must not count as output"
+ assert rec["status"] == "timeout"
+
+
+def test_startup_guard_disarms_once_the_model_produces_output(monkeypatch, tmp_path):
+ # The other side of the same rule: real output means the turn is alive, and a tool call that
+ # runs for minutes must never be killed. hermes writes the assistant message BEFORE executing
+ # the call, so output exists by then.
+ started = time.time()
+ db_file = _state_db(tmp_path / "live.db", started, [("user", "hi"), ("assistant", "working on it")])
+ monkeypatch.setattr(rn, "_hermes_db_ro", _ro_opener(db_file))
+ monkeypatch.setattr(rn, "_HERMES_STARTUP_TIMEOUT_S", 0.3)
+ monkeypatch.setattr(rn, "_HERMES_POLL_S", 0.05)
+ monkeypatch.setattr(rn, "MAX_TURN_SECONDS", 999)
+ # Outlives the startup window several times over, the way a long tool call does.
+ monkeypatch.setattr(rn.subprocess, "Popen", _fake_popen_spawning(["sleep", "1.5"]))
+
+ rn._turns["live"] = {"events": [], "done": False}
+ rn._run_hermes_bg("live", str(tmp_path), {**os.environ, "HERMES_HOME": str(tmp_path)},
+ "gpt-5.4", "azure", "hi", None, None)
+
+ rec = rn._turns["live"]
+ assert not rec.get("capped"), "a turn that produced output was killed by the startup guard"
+ assert rec["status"] == "done"
+
+
+def test_resume_lost_emits_visible_event_not_just_a_log_line(monkeypatch, tmp_path):
+ # The requested resume target is never found (_hermes_session_row -> None) — the driver must
+ # emit a resume_lost event into the turn's OWN stream, not just print() to server logs.
+ monkeypatch.setattr(rn, "_hermes_session_row", lambda db_path, sid: None)
+ monkeypatch.setattr(rn, "_hermes_db_ro", lambda db_path: None) # no session ever seen mid-run either
+ monkeypatch.setattr(rn, "_HERMES_STARTUP_TIMEOUT_S", 10.0) # must not race the fast fake process
+ monkeypatch.setattr(rn, "_HERMES_POLL_S", 0.02)
+ monkeypatch.setattr(rn.subprocess, "Popen", _fake_popen_spawning(["sh", "-c", "exit 0"]))
+
+ turn_id = "resume-lost"
+ rn._turns[turn_id] = {"events": [], "done": False}
+ env = {**os.environ, "HERMES_HOME": str(tmp_path)}
+
+ rn._run_hermes_bg(turn_id, str(tmp_path), env, "gpt-5.4", "azure", "hi", "prior-not-here", None)
+
+ events = rn._turns[turn_id]["events"]
+ assert any(e.get("type") == "system" and e.get("subtype") == "resume_lost"
+ and e.get("requested_session_id") == "prior-not-here" for e in events), (
+ "resume-lost was not signaled in the turn's event stream")
diff --git a/runner/tests/test_token_usage_norm.py b/runner/tests/test_token_usage_norm.py
new file mode 100644
index 00000000..a52cf0ee
--- /dev/null
+++ b/runner/tests/test_token_usage_norm.py
@@ -0,0 +1,45 @@
+"""Guard: codex token usage must report FRESH input (cached read netted out).
+
+Regression this pins: codex's inputTokens INCLUDES the cached read. Billing charges input_1k on
+input_tokens, so if the cached read isn't subtracted, cached tokens are billed at the full input
+rate (~10x the cached rate). On a long conversation (each turn resends the whole cache-hit
+transcript) that inflated cost several-fold. _norm_token_usage must net cached out so input_tokens
+is fresh-only, uniform with the Anthropic contract.
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import server as rn # noqa: E402
+
+
+def test_codex_input_is_netted_of_cached():
+ # codex app-server shape: counts under tokenUsage.total, cached included in inputTokens
+ u = {"tokenUsage": {"total": {
+ "inputTokens": 13024524, "outputTokens": 264690, "cachedInputTokens": 11233792}}}
+ r = rn._norm_token_usage(u)
+ assert r["input_tokens"] == 13024524 - 11233792 # fresh only
+ assert r["cache_read_tokens"] == 11233792
+ assert r["output_tokens"] == 264690
+
+
+def test_snake_case_exec_shape_netted():
+ u = {"input_tokens": 1000, "output_tokens": 200, "cached_input_tokens": 900}
+ r = rn._norm_token_usage(u)
+ assert r["input_tokens"] == 100
+ assert r["cache_read_tokens"] == 900
+
+
+def test_no_cache_leaves_input_untouched():
+ u = {"input_tokens": 500, "output_tokens": 50}
+ r = rn._norm_token_usage(u)
+ assert r["input_tokens"] == 500
+ assert "cache_read_tokens" not in r
+
+
+def test_never_negative_when_provider_reports_net_input():
+ # defensive: a provider that already nets input (cache_read > reported input) must not go negative
+ u = {"input_tokens": 100, "output_tokens": 10, "cache_read_input_tokens": 900}
+ r = rn._norm_token_usage(u)
+ assert r["input_tokens"] == 0
+ assert r["cache_read_tokens"] == 900