HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.
Python agent-apiagent-frameworkagent-harnessai-agentsai-infrastructurecoding-agentconformance-testingdeepseek-harnessharness-engineeringhermesllmopen-protocolopen-sourceopen-standardpi-coding-agentself-hosteduhpunified-apiunified-harness-protocolunified-interface
relay: a route's totals are folded and read under one lock (#483)
Rudy Mizrahi Celekli526c595122f8
diff --git a/runner/server.py b/runner/server.pyindex 79ab9604..9e48f876 100644--- a/runner/server.py+++ b/runner/server.py@@ -3844,7 +3844,15 @@ def _hermes_mcp_section(servers: list[dict] | None) -> dict:# turn with a fresh env, so each turn registers its upstream and the entry simply outlives the# turn (a few dozen bytes per turn, process lifetime — the sandbox recycles long before this# matters).-_HERMES_RELAY: dict = {"server": None, "port": 0, "routes": {}, "lock": threading.Lock()}+_HERMES_RELAY: dict = {"server": None, "port": 0, "routes": {}, "lock": threading.Lock(),+ # A route's totals (`usage`, `reasoning_tokens`) are folded in by the relay's+ # request threads, one per call, and a CLI can have two calls in flight on one+ # route (a background call beside the main one, parallel subagents) while the+ # turn reads them. Every fold and every read of a total assembled from several+ # writes holds this lock, so a reader sees a whole fold or the one before it and+ # no increment is lost (#449). A single store of one scalar (last_data,+ # served_model, last_finish, a repair flag) needs none.+ "totals": threading.Lock()}# The thinking of the turn being built, while /turn runs a backend's builder: {"asked": the level# the turn asked for ("" for none), "applied": what a backend whose calls never pass the relay set# for its CLI (Claude Code, Codex on OpenAI's own endpoint), "routes": the flags of every route the@@ -4995,7 +5003,8 @@ def _thinking_in_sse_line(line: bytes) -> int | None:def _thinking_add(flags: dict, count: int | None) -> None:"""Fold one call's thinking count into the route's total; the key exists only once a provider gave one."""if count is not None:- flags["reasoning_tokens"] = int(flags.get("reasoning_tokens") or 0) + count+ with _HERMES_RELAY["totals"]:+ flags["reasoning_tokens"] = int(flags.get("reasoning_tokens") or 0) + countdef _stream_fold(flags: dict, call_usage: dict, call_thinking: int | None, folded: dict) -> None:@@ -5008,27 +5017,29 @@ def _stream_fold(flags: dict, call_usage: dict, call_thinking: int | None, foldewas on it. CheetahClaws's in-process driver is such a client, and its turns came back with nousage at all (the thinking column on a candidate, 2026-10-05: the relay had read 329 outputtokens and 312 thinking tokens for a call whose turn recorded none)."""- if call_usage:- total = flags.setdefault("usage", {})- for k, v in call_usage.items():- delta = int(v) - int(folded.get(k, 0))- if delta or k not in total: # a zero is still a figure the provider gave- total[k] = int(total.get(k, 0)) + delta- folded[k] = int(v)- if call_thinking is not None:- delta = call_thinking - int(folded.get("_thinking", 0))- if delta or "_thinking" not in folded:- flags["reasoning_tokens"] = int(flags.get("reasoning_tokens") or 0) + delta- folded["_thinking"] = call_thinking+ with _HERMES_RELAY["totals"]:+ if call_usage:+ total = flags.setdefault("usage", {})+ for k, v in call_usage.items():+ delta = int(v) - int(folded.get(k, 0))+ if delta or k not in total: # a zero is still a figure the provider gave+ total[k] = int(total.get(k, 0)) + delta+ folded[k] = int(v)+ if call_thinking is not None:+ delta = call_thinking - int(folded.get("_thinking", 0))+ if delta or "_thinking" not in folded:+ flags["reasoning_tokens"] = int(flags.get("reasoning_tokens") or 0) + delta+ folded["_thinking"] = call_thinkingdef _usage_add(flags: dict, call_usage: dict) -> None:"""Fold one call's usage into the route's running total: a turn is many provider calls."""if not call_usage:return- total = flags.setdefault("usage", {})- for k, v in call_usage.items():- total[k] = int(total.get(k, 0)) + int(v)+ with _HERMES_RELAY["totals"]:+ total = flags.setdefault("usage", {})+ for k, v in call_usage.items():+ total[k] = int(total.get(k, 0)) + int(v)def _model_metadata_with_context_length(data: bytes) -> bytes:@@ -5148,8 +5159,10 @@ def _relay_usage(env: dict) -> dict:statement of what it counted, read off the bytes as they pass, rather than a number the CLIestimated or nothing at all. A backend that reports its own usage keeps it; this only fills aresult event that arrived empty."""- u = _turn_relay_flags(env).get("usage")- return dict(u) if isinstance(u, dict) and u else {}+ flags = _turn_relay_flags(env)+ with _HERMES_RELAY["totals"]: # a whole fold, never part of one (#449)+ u = flags.get("usage")+ return dict(u) if isinstance(u, dict) and u else {}# A local model server's own discovery endpoints (Ollama, llama.cpp, LM Studio, vLLM), as the pathdiff --git a/runner/tests/test_relay_totals_lock.py b/runner/tests/test_relay_totals_lock.pynew file mode 100644index 00000000..21bb66e3--- /dev/null+++ b/runner/tests/test_relay_totals_lock.py@@ -0,0 +1,154 @@+"""A route's totals are folded by the relay's request threads and read by the turn, on different+threads by construction (#449). These pin the ordering by holding a fold at its worst moment and+acting from a second thread, rather than by asking whether a lock object exists:++- a reader sees the whole fold or the one before it, never a mixture of two answers' figures;+- two folds racing on one route both land, so no call's tokens are lost.+"""+import pathlib+import sys+import threading++sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))+import server as rs # noqa: E402++GRACE = 0.5 # how long a second thread is given to act early; a fixed tree only waits longer+++class _Gated(dict):+ """A call's usage that stops after handing over its first figure, until released: the fold has+ written one key of the route's total and not the next."""++ def __init__(self, *a, **kw):+ super().__init__(*a, **kw)+ self.inside = threading.Event()+ self.release = threading.Event()++ def items(self):+ for n, kv in enumerate(super().items()):+ if n == 1:+ self.inside.set()+ assert self.release.wait(10)+ yield kv+++class _SlowInt:+ """A figure that is read only once released: the fold has read the route's current total and+ has not written its sum yet, the read-modify-write window where another fold can land."""++ def __init__(self, n):+ self.n = n+ self.inside = threading.Event()+ self.release = threading.Event()++ def _wait(self):+ self.inside.set()+ assert self.release.wait(10)++ def __int__(self):+ self._wait()+ return self.n++ def __radd__(self, other):+ self._wait()+ return other + self.n+++def _route(flags: dict) -> dict:+ tok = "hr-relay-test-totals-%d" % id(flags)+ rs._HERMES_RELAY["routes"][tok] = ("http://upstream.invalid", "k", flags)+ return {"OPENAI_API_KEY": tok}+++def _run(fn, *args):+ out = {}+ t = threading.Thread(target=lambda: out.setdefault("v", fn(*args)), daemon=True)+ t.start()+ return t, out+++def _read_during(fold, flags: dict, gate) -> dict:+ """Start `fold`, wait until it is held midway, read the route's usage from another thread, then+ let the fold finish. Returns what the reader saw."""+ env = _route(flags)+ folder, _ = _run(fold)+ assert gate.inside.wait(10)+ reader, seen = _run(rs._relay_usage, env)+ reader.join(GRACE)+ gate.release.set()+ folder.join(10)+ reader.join(10)+ assert not folder.is_alive() and not reader.is_alive()+ return seen["v"]+++def test_a_reader_sees_a_whole_usage_add_or_the_one_before_it():+ before = {"input_tokens": 10, "output_tokens": 4}+ flags = {"usage": dict(before)}+ call = _Gated(input_tokens=5, output_tokens=3)+ seen = _read_during(lambda: rs._usage_add(flags, call), flags, call)+ after = {"input_tokens": 15, "output_tokens": 7}+ assert seen in (before, after), f"the reader saw part of a fold: {seen}"+ assert flags["usage"] == after+++def test_a_reader_sees_a_whole_stream_fold_or_the_one_before_it():+ before = {"input_tokens": 10, "output_tokens": 4}+ flags = {"usage": dict(before)}+ call = _Gated(input_tokens=5, output_tokens=3)+ seen = _read_during(lambda: rs._stream_fold(flags, call, None, {}), flags, call)+ after = {"input_tokens": 15, "output_tokens": 7}+ assert seen in (before, after), f"the reader saw part of a fold: {seen}"+ assert flags["usage"] == after+++def _race(first, second):+ """Hold `first` inside its read-modify-write, run `second` from another thread, then release."""+ gate = first[1]+ a, _ = _run(*first[0])+ assert gate.inside.wait(10)+ b, _ = _run(*second)+ b.join(GRACE)+ gate.release.set()+ a.join(10)+ b.join(10)+ assert not a.is_alive() and not b.is_alive()+++def test_two_usage_folds_racing_on_one_route_both_land():+ flags = {"usage": {"output_tokens": 100}}+ slow = _SlowInt(3)+ _race(((rs._usage_add, flags, {"output_tokens": slow}), slow),+ (rs._usage_add, flags, {"output_tokens": 4}))+ assert flags["usage"]["output_tokens"] == 107, "one call's tokens were lost"+++def test_two_thinking_folds_racing_on_one_route_both_land():+ flags = {"reasoning_tokens": 100}+ slow = _SlowInt(3)+ _race(((rs._thinking_add, flags, slow), slow), (rs._thinking_add, flags, 4))+ assert flags["reasoning_tokens"] == 107, "one call's thinking tokens were lost"+++def test_many_calls_on_one_route_sum_exactly():+ flags: dict = {}+ n_threads, n_calls = 8, 200++ def calls():+ for _ in range(n_calls):+ rs._usage_add(flags, {"input_tokens": 1, "output_tokens": 2})+ rs._thinking_add(flags, 1)++ threads = [threading.Thread(target=calls) for _ in range(n_threads)]+ old = sys.getswitchinterval()+ sys.setswitchinterval(1e-6)+ try:+ for t in threads:+ t.start()+ for t in threads:+ t.join()+ finally:+ sys.setswitchinterval(old)+ total = n_threads * n_calls+ assert flags["usage"] == {"input_tokens": total, "output_tokens": 2 * total}+ assert flags["reasoning_tokens"] == total