HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.
Python agent-apiagent-frameworkagent-harnessai-agentsai-infrastructurecoding-agentconformance-testingdeepseek-harnessharness-engineeringhermesllmopen-protocolopen-sourceopen-standardpi-coding-agentself-hosteduhpunified-apiunified-harness-protocolunified-interface
benchmark: a foreign run keeps its trace, the finding is decided before cleanup (#253)
github-actions[bot]0fc05df47780
diff --git a/scripts/benchmark/run.py b/scripts/benchmark/run.pyindex d2657002..c993e3ed 100644--- a/scripts/benchmark/run.py+++ b/scripts/benchmark/run.py@@ -205,6 +205,13 @@ def provider_failure(turn_error) -> str:return msg[:160] if msg and PROVIDER_ERROR.search(msg) else ""+def foreign_connection(rec: dict, expect: str) -> str:+ """The connection a run was served by when it is not the one under test, else "": the matrix's+ rule 1. Decided before a session is cleaned up, so a foreign run keeps its trace (#243)."""+ got = str(rec.get("connection") or "")+ return got if expect and got and got != expect else ""++def prompt_matches(stored: str, prompt: str) -> bool:"""Whether a session's stored prompt is this prompt. The session list keeps the prompt's head(1500 characters, measured), sometimes behind an attachment line, so the stored head — whole,@@ -250,7 +257,8 @@ def recover_session(before: set[str], harness_id: str, prompt: str, t0: float, ctime.sleep(10)-def run_task(pack, root: str, task: dict, label: str, harness_id: str, model: str, provider: str, workdir: str) -> dict:+def run_task(pack, root: str, task: dict, label: str, harness_id: str, model: str, provider: str, workdir: str,+ expect: str = "") -> dict:staged = pack.stage(task, root)content = [{"type": "input_text", "text": staged["prompt"]}]for name, data in staged["files"]:@@ -299,6 +307,11 @@ def run_task(pack, root: str, task: dict, label: str, harness_id: str, model: stassistant=(turn.get("assistant") or "")[:200])if rec.get("turn_error") is None and turn.get("error"):rec["turn_error"] = turn.get("error")+ # Rule 1 is decided HERE, before the session is cleaned up: a run served by another connection+ # is a finding, and a finding keeps its session and trace as the evidence. Deciding it after+ # this function returned deleted the trace first and printed FINDING foreign second (#243).+ if foreign_connection(rec, expect):+ rec["foreign"] = rec["connection"]# A turn the provider refused before the agent did anything measures the account, not the# harness: 150 records of "402 Insufficient Balance" arrived in eleven minutes once a key ran# dry (2026-09-18). That is this runner's problem to re-run, never the harness's failure.@@ -396,12 +409,10 @@ def main() -> None:if halt.is_set():returntry:- rec = run_task(pack, root, t, label, hid, model, provider, workdir)+ rec = run_task(pack, root, t, label, hid, model, provider, workdir, expect=expect)except Exception as e: # noqa: BLE001 — this runner's own failure, re-run next launchrec = {"provider": provider, "harness": label, "harness_id": hid, "model": model,"pack": pack.NAME, "task": t["id"], "error": repr(e)[:300]}- if expect and rec.get("connection") and rec["connection"] != expect:- rec["foreign"] = rec["connection"]# three provider refusals in a row: the account, the key or the provider is down,# and every further task would only record the same lineif provider_streak(rec) and not halt.is_set():diff --git a/scripts/benchmark/test_run.py b/scripts/benchmark/test_run.pyindex 96a06c33..c3ac9020 100644--- a/scripts/benchmark/test_run.py+++ b/scripts/benchmark/test_run.py@@ -10,7 +10,7 @@ import syssys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))from render import render # noqa: E402-from run import lookup_use, network_use, prompt_matches, provider_failure, provider_streak, tool_outcomes, usage_of # noqa: E402+from run import foreign_connection, lookup_use, network_use, prompt_matches, provider_failure, provider_streak, tool_outcomes, usage_of # noqa: E402def test_three_provider_failures_in_a_row_trip_the_halt_and_a_result_resets_it():@@ -197,3 +197,17 @@ def test_a_runner_error_is_listed_as_an_error_not_a_fail():assert "Runner errors" in md and "opencode x m b: request failed" in mdrow = next(line for line in md.splitlines() if line.startswith("| opencode |"))assert "| 1 | 1 (100%) |" in row+++def test_a_foreign_run_is_decided_before_its_session_is_cleaned_up():+ """Rule 1 used to be decided in the caller after run_task had already deleted the session, so+ the trace of a foreign run was gone before the run was printed as a finding (#243, reported by+ trifonnt). The decision is a function run_task calls before cleanup."""+ assert foreign_connection({"connection": "integration:other"}, "integration:deepseek") == "integration:other"+ assert foreign_connection({"connection": "integration:deepseek"}, "integration:deepseek") == ""+ assert foreign_connection({"connection": ""}, "integration:deepseek") == "" # unreported: not a finding+ assert foreign_connection({"connection": "integration:other"}, "") == "" # nothing under test+ import inspect+ from run import run_task+ src = inspect.getsource(run_task)+ assert src.index("foreign_connection(rec, expect)") < src.index('api("DELETE", f"/v1/sessions/{sid}")', src.index("kept"))