The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(cli): wait for the consumer when the gateway client's event queue is full
David Dudok de Wit5d6017d82437
diff --git a/hermes_cli/gateway_client.py b/hermes_cli/gateway_client.pyindex f369f2cfe51..495e1f7167a 100644--- a/hermes_cli/gateway_client.py+++ b/hermes_cli/gateway_client.py@@ -20,6 +20,9 @@ DEFAULT_RPC_TIMEOUT = 30# request, ``compression.context_total_ceiling_seconds`` to 600 s). Same budget as Desktop's# ``SESSION_COMPRESS_TIMEOUT_MS`` so every surface gives up at the same point.COMPRESS_RPC_TIMEOUT = 660+# A full event queue parks the socket reader; a consumer that takes no event for this long has+# stalled and is failed. Same bound as the owner's own send deadline (``tui_gateway/ws.py``).+EVENT_STALL_TIMEOUT = 30def gateway_ws_target(endpoint, ticket):@@ -119,8 +122,15 @@ class GatewayClient:else:future.set_result(frame.get("result", {}))elif "method" in frame:- self.events.put_nowait(frame)- except (ConnectionClosed, OSError, ValueError, asyncio.QueueFull):+ try:+ self.events.put_nowait(frame)+ except asyncio.QueueFull:+ # Frames the socket already buffered (a CLI resumed from Ctrl-Z) are read+ # without yielding, so the consumer has not had a turn yet: wait for room.+ # Not wait_for: on 3.11 it swallows a close that lands as the put completes.+ async with asyncio.timeout(EVENT_STALL_TIMEOUT):+ await self.events.put(frame)+ except (ConnectionClosed, OSError, ValueError, TimeoutError):passfinally:error = GatewayClientError("Gateway disconnected; turn outcome is unknown. "diff --git a/hermes_cli/kanban_worker_client.py b/hermes_cli/kanban_worker_client.pyindex 42e7fca60d8..1528a3705ba 100644--- a/hermes_cli/kanban_worker_client.py+++ b/hermes_cli/kanban_worker_client.py@@ -29,13 +29,25 @@ async def run(params, board_db):while True:try:async with connect_gateway() as client:- return await _submit_claim(client, params, board_db)+ # kanban.run subscribes this socket to the card's session, but only receipts matter+ # here: discard the turn's events, or they fill the client's bounded queue and its+ # reader stops, with the next receipt behind it.+ drain = asyncio.create_task(_discard(client.events))+ try:+ return await _submit_claim(client, params, board_db)+ finally:+ drain.cancel()except GatewayUnavailableError as exc:if exc.state not in _TRANSIENT_OWNER_STATES or time.monotonic() >= deadline:raiseawait asyncio.sleep(1)+async def _discard(events):+ while True:+ await events.get()++async def _submit_claim(client, params, board_db):from hermes_cli.gateway_client import GatewayClientErrordb = str(await asyncio.to_thread(Path(board_db).resolve))diff --git a/tests/agent/test_kanban_turn_recovery.py b/tests/agent/test_kanban_turn_recovery.pyindex ee77480812a..87cc0033213 100644--- a/tests/agent/test_kanban_turn_recovery.py+++ b/tests/agent/test_kanban_turn_recovery.py@@ -585,6 +585,7 @@ def test_worker_client_reports_the_owner_exit_and_reply(owner_worker, monkeypatc"""The dispatcher-spawned process is the client half: it exits with the code the ownerbooked for this exact attempt, ends its log with the attempt's reply, and writes theexit trailer the dead-worker sweep reads — so an exhausted wall still releases as 75."""+ import asynciofrom contextlib import asynccontextmanagerfrom hermes_cli import gateway_client, kanban_worker_client@@ -595,6 +596,8 @@ def test_worker_client_reports_the_owner_exit_and_reply(owner_worker, monkeypatcsubmitted = []class _Owner:+ events: asyncio.Queue = asyncio.Queue()+async def rpc(self, method, **params):submitted.append(method)assert method == "kanban.run" and params["task_id"] == context["task_id"]diff --git a/tests/hermes_cli/test_gateway_client_backlog.py b/tests/hermes_cli/test_gateway_client_backlog.pynew file mode 100644index 00000000000..b55ec49f4d7--- /dev/null+++ b/tests/hermes_cli/test_gateway_client_backlog.py@@ -0,0 +1,126 @@+"""A client's own event backlog never fails its turn (#134889).++The frames a socket already buffered (a CLI resumed from Ctrl-Z) are read in one burst that never+yields to the loop, so the 4,096-frame client queue filled before its consumer had a turn: the+reader reported the turn's outcome as unknown, and the next prompt failed while the socket closed.+A Kanban worker, subscribed by ``kanban.run`` but reading only receipts, filled it the same way.+"""+import asyncio+import json+from contextlib import asynccontextmanager++import pytest++from hermes_cli import gateway_client+from hermes_cli.gateway_client import GatewayClient, GatewayClientError+from hermes_cli.gateway_connection import GatewayConnection++SID = "local-backlog"+++def _frame(kind, text):+ return json.dumps({"jsonrpc": "2.0", "method": "event", "params": {+ "type": kind, "session_id": SID, "payload": {"text": text}}})+++class Backlog:+ """A socket whose frames are all buffered already: iterating them never yields to the loop."""++ def __init__(self, frames):+ self.frames = frames++ async def __aiter__(self):+ for raw in self.frames:+ yield raw+ await asyncio.Event().wait() # the socket stays open+++def test_backlog_larger_than_the_client_queue_reaches_the_cli_intact():+ deltas = [str(i) for i in range(6000)]+ frames = [_frame("message.delta", text) for text in deltas] + [_frame("message.complete", "done")]++ @asynccontextmanager+ async def connect():+ async with GatewayClient(Backlog(frames)) as client:+ yield client++ conn = GatewayConnection(connect=connect)+ conn.start(timeout=5)+ try:+ seen = []+ while not seen or not (isinstance(seen[-1], Exception) or seen[-1]["params"]["type"] == "message.complete"):+ seen.append(conn.events.get(timeout=5))+ finally:+ conn.close()+ assert not isinstance(seen[-1], Exception), seen[-1]+ assert [frame["params"]["payload"]["text"] for frame in seen[:-1]] == deltas+++async def test_a_consumer_that_stops_draining_is_still_failed_with_the_queue_bounded(monkeypatch):+ monkeypatch.setattr(gateway_client, "EVENT_STALL_TIMEOUT", 0.05, raising=False)+ client = GatewayClient(Backlog([_frame("message.delta", str(i)) for i in range(6000)]))+ pending = asyncio.get_running_loop().create_future()+ client.pending[1] = pending+ await asyncio.wait_for(client._read(), 5)+ with pytest.raises(GatewayClientError, match="outcome is unknown"):+ await pending+ assert client.events.full()+ queued = [client.events.get_nowait() for _ in range(client.events.maxsize)]+ assert isinstance(queued[-1], GatewayClientError)+++class CardOwner:+ """A Kanban card's owner as the worker's socket sees it: ``kanban.run`` subscribes the socket,+ the card's turn streams ``events`` frames, and the next receipt poll finds it terminal."""++ def __init__(self, events):+ self.events = events+ self.inbox = asyncio.Queue()++ async def send(self, raw):+ request = json.loads(raw)+ if request["method"] == "kanban.run":+ result = {"session_id": SID, "receipt": {"status": "running", "admission_id": "a1"}}+ frames = [{"jsonrpc": "2.0", "id": request["id"], "result": result}]+ frames += [json.loads(_frame("message.delta", str(i))) for i in range(self.events)]+ else:+ frames = [{"jsonrpc": "2.0", "id": request["id"], "result": {"status": "terminal", "admission_id": "a1"}}]+ for frame in frames:+ self.inbox.put_nowait(json.dumps(frame))++ async def __aiter__(self):+ while True:+ yield await self.inbox.get()+++async def test_kanban_worker_settles_a_card_whose_turn_outstreams_the_client_queue(monkeypatch, tmp_path):+ from gateway import session_kanban+ from hermes_cli import kanban_worker_client++ @asynccontextmanager+ async def connect():+ async with GatewayClient(CardOwner(6000)) as client:+ yield client++ monkeypatch.setattr(gateway_client, "connect_gateway", connect)+ monkeypatch.setattr(session_kanban, "worker_result", lambda *_: {"exit_code": 0, "last_output": "card closed"})+ monkeypatch.delenv("HERMES_TUI_GATEWAY_URL", raising=False)+ assert await asyncio.wait_for(kanban_worker_client.run({"task_id": "t1"}, str(tmp_path / "kanban.db")), 5) == 0+++async def test_closing_as_a_waiting_frame_lands_still_ends_the_reader():+ """A close that lands just as the consumer makes room still ends the reader, so ``__aexit__``+ returns instead of waiting on the open socket forever."""+ client = await GatewayClient(Backlog([_frame("message.delta", str(i)) for i in range(4097)])).__aenter__()+ while not client.events.full(): # the reader yields only once it waits on the last frame+ await asyncio.sleep(0)+ client.events.get_nowait()+ closing = asyncio.create_task(client.__aexit__(None, None, None))+ try:+ await asyncio.wait_for(asyncio.shield(closing), 2)+ finally:+ client.reader.cancel()+ await asyncio.gather(closing, return_exceptions=True)diff --git a/tests/hermes_cli/test_kanban_worker_owner_wait.py b/tests/hermes_cli/test_kanban_worker_owner_wait.pyindex c8973e6b679..2b31658760a 100644--- a/tests/hermes_cli/test_kanban_worker_owner_wait.py+++ b/tests/hermes_cli/test_kanban_worker_owner_wait.py@@ -6,6 +6,7 @@ of those trip the breaker and park a card that never ran (the in-flight handoffmust never submit the claim twice, and a non-transient refusal still fails at once."""+import asynciofrom contextlib import asynccontextmanagerimport pytest@@ -25,6 +26,8 @@ def _owner(states, submitted):raise unavailable(f"Gateway {state}: update_paused", state)class _Client:+ events = asyncio.Queue()+async def rpc(self, method, **params):submitted.append(method)return {"session_id": "s1", "receipt": {"status": "terminal", "admission_id": "a1"}}