HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.
Python agent-apiagent-frameworkagent-harnessai-agentsai-infrastructurecoding-agentconformance-testingdeepseek-harnessharness-engineeringhermesllmopen-protocolopen-sourceopen-standardpi-coding-agentself-hosteduhpunified-apiunified-harness-protocolunified-interface
benchmark: the provider-failure streak has its own lock; a note on the run goes under the table
Hao Luf3aa0f110977
diff --git a/scripts/benchmark/render.py b/scripts/benchmark/render.pyindex 1d010a9b..87b06fdb 100644--- a/scripts/benchmark/render.py+++ b/scripts/benchmark/render.py@@ -101,8 +101,20 @@ def render(records: list[dict]) -> str:if __name__ == "__main__":+ # `--note "..."` (repeatable) records what the run ran on: the instance version, the date, the+ # connection, anything a reader needs to reproduce the column. The document is generated, so+ # this is where such a sentence lives.recs: list[dict] = []- for path in sys.argv[1:]:- data = json.load(open(path, encoding="utf-8"))+ notes: list[str] = []+ args = sys.argv[1:]+ while args:+ a = args.pop(0)+ if a == "--note":+ notes.append(args.pop(0))+ continue+ data = json.load(open(a, encoding="utf-8"))recs += list(data.values()) if isinstance(data, dict) else list(data)- print(render(recs))+ md = render(recs)+ if notes:+ md += "\n## Run notes\n\n" + "\n".join(f"- {n}" for n in notes) + "\n"+ print(md)diff --git a/scripts/benchmark/run.py b/scripts/benchmark/run.pyindex e2c5cac3..3d701c72 100644--- a/scripts/benchmark/run.py+++ b/scripts/benchmark/run.py@@ -162,6 +162,17 @@ PROVIDER_ERROR = re.compile(r"\b(?:40[1239]|429|5\d\d)\b|insufficient balance|rare.I)+_streak = [0]+_streak_lock = threading.Lock() # its own lock: log() takes _lock, and a caller may hold this while logging+++def provider_streak(rec: dict, limit: int = 3) -> bool:+ """True once `limit` provider failures have arrived in a row; any other record resets the count."""+ with _streak_lock:+ _streak[0] = _streak[0] + 1 if str(rec.get("error") or "").startswith("provider failure") else 0+ return _streak[0] >= limit++def provider_failure(turn_error) -> str:"""The provider's refusal in a failed turn's error, or '' when the error is something else."""msg = turn_error.get("message") if isinstance(turn_error, dict) else turn_error@@ -340,7 +351,6 @@ def main() -> None:return f"{rec['provider']}|{rec['harness']}|{rec['model']}|{rec['pack']}|{rec['task']}"halt = threading.Event()- streak = [0]for label, hid in harnesses:for model in models:if halt.is_set():@@ -366,11 +376,9 @@ def main() -> None:rec["foreign"] = rec["connection"]# three provider refusals in a row: the account, the key or the provider is down,# and every further task would only record the same line- with _lock:- streak[0] = streak[0] + 1 if str(rec.get("error", "")).startswith("provider failure") else 0- if streak[0] >= 3 and not halt.is_set():- halt.set()- log(f"HALT after three provider failures in a row: {rec['error'][:160]}")+ if provider_streak(rec) and not halt.is_set():+ halt.set()+ log(f"HALT after three provider failures in a row: {rec['error'][:160]}")save(rec)verdict = ("ERROR " + rec["error"][:80]) if rec.get("error") else \("FINDING network" if rec.get("network") else "FINDING lookup" if rec.get("lookup") elsediff --git a/scripts/benchmark/test_run.py b/scripts/benchmark/test_run.pyindex 5eb0e650..14ec64e5 100644--- a/scripts/benchmark/test_run.py+++ b/scripts/benchmark/test_run.py@@ -10,7 +10,18 @@ import syssys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))from render import render # noqa: E402-from run import lookup_use, network_use, prompt_matches, provider_failure, tool_outcomes, usage_of # noqa: E402+from run import lookup_use, network_use, prompt_matches, provider_failure, provider_streak, tool_outcomes, usage_of # noqa: E402+++def test_three_provider_failures_in_a_row_trip_the_halt_and_a_result_resets_it():+ """The first cut counted the streak under the same lock log() takes and deadlocked on the third+ failure (measured 2026-09-18: three threads in futex wait, the run silent for half an hour)."""+ bad = {"error": "provider failure, not a result: 402 Insufficient Balance"}+ good = {"resolved": True}+ assert not provider_streak(bad) and not provider_streak(bad) and provider_streak(bad)+ assert not provider_streak(good)+ assert not provider_streak(bad) and not provider_streak(bad)+ assert not provider_streak({"error": "request failed and no session found: 502"}) # not the provider'sdef test_a_providers_refusal_is_the_runners_problem_not_the_harnesss():