Skip to content

The agent that grows with you

fix(cli): wait for the consumer when the gateway client's event queue is full

David Dudok de Wit5d6017d82437
diff --git a/hermes_cli/gateway_client.py b/hermes_cli/gateway_client.py
index f369f2cfe51..495e1f7167a 100644
--- a/hermes_cli/gateway_client.py
+++ b/hermes_cli/gateway_client.py
@@ -20,6 +20,9 @@ DEFAULT_RPC_TIMEOUT = 30
# request, ``compression.context_total_ceiling_seconds`` to 600 s). Same budget as Desktop's
# ``SESSION_COMPRESS_TIMEOUT_MS`` so every surface gives up at the same point.
COMPRESS_RPC_TIMEOUT = 660
+# A full event queue parks the socket reader; a consumer that takes no event for this long has
+# stalled and is failed. Same bound as the owner's own send deadline (``tui_gateway/ws.py``).
+EVENT_STALL_TIMEOUT = 30
def gateway_ws_target(endpoint, ticket):
@@ -119,8 +122,15 @@ class GatewayClient:
else:
future.set_result(frame.get("result", {}))
elif "method" in frame:
- self.events.put_nowait(frame)
- except (ConnectionClosed, OSError, ValueError, asyncio.QueueFull):
+ try:
+ self.events.put_nowait(frame)
+ except asyncio.QueueFull:
+ # Frames the socket already buffered (a CLI resumed from Ctrl-Z) are read
+ # without yielding, so the consumer has not had a turn yet: wait for room.
+ # Not wait_for: on 3.11 it swallows a close that lands as the put completes.
+ async with asyncio.timeout(EVENT_STALL_TIMEOUT):
+ await self.events.put(frame)
+ except (ConnectionClosed, OSError, ValueError, TimeoutError):
pass
finally:
error = GatewayClientError("Gateway disconnected; turn outcome is unknown. "
diff --git a/hermes_cli/kanban_worker_client.py b/hermes_cli/kanban_worker_client.py
index 42e7fca60d8..1528a3705ba 100644
--- a/hermes_cli/kanban_worker_client.py
+++ b/hermes_cli/kanban_worker_client.py
@@ -29,13 +29,25 @@ async def run(params, board_db):
while True:
try:
async with connect_gateway() as client:
- return await _submit_claim(client, params, board_db)
+ # kanban.run subscribes this socket to the card's session, but only receipts matter
+ # here: discard the turn's events, or they fill the client's bounded queue and its
+ # reader stops, with the next receipt behind it.
+ drain = asyncio.create_task(_discard(client.events))
+ try:
+ return await _submit_claim(client, params, board_db)
+ finally:
+ drain.cancel()
except GatewayUnavailableError as exc:
if exc.state not in _TRANSIENT_OWNER_STATES or time.monotonic() >= deadline:
raise
await asyncio.sleep(1)
+async def _discard(events):
+ while True:
+ await events.get()
+
+
async def _submit_claim(client, params, board_db):
from hermes_cli.gateway_client import GatewayClientError
db = str(await asyncio.to_thread(Path(board_db).resolve))
diff --git a/tests/agent/test_kanban_turn_recovery.py b/tests/agent/test_kanban_turn_recovery.py
index ee77480812a..87cc0033213 100644
--- a/tests/agent/test_kanban_turn_recovery.py
+++ b/tests/agent/test_kanban_turn_recovery.py
@@ -585,6 +585,7 @@ def test_worker_client_reports_the_owner_exit_and_reply(owner_worker, monkeypatc
"""The dispatcher-spawned process is the client half: it exits with the code the owner
booked for this exact attempt, ends its log with the attempt's reply, and writes the
exit trailer the dead-worker sweep reads — so an exhausted wall still releases as 75."""
+ import asyncio
from contextlib import asynccontextmanager
from hermes_cli import gateway_client, kanban_worker_client
@@ -595,6 +596,8 @@ def test_worker_client_reports_the_owner_exit_and_reply(owner_worker, monkeypatc
submitted = []
class _Owner:
+ events: asyncio.Queue = asyncio.Queue()
+
async def rpc(self, method, **params):
submitted.append(method)
assert method == "kanban.run" and params["task_id"] == context["task_id"]
diff --git a/tests/hermes_cli/test_gateway_client_backlog.py b/tests/hermes_cli/test_gateway_client_backlog.py
new file mode 100644
index 00000000000..b55ec49f4d7
--- /dev/null
+++ b/tests/hermes_cli/test_gateway_client_backlog.py
@@ -0,0 +1,126 @@
+"""A client's own event backlog never fails its turn (#134889).
+
+The frames a socket already buffered (a CLI resumed from Ctrl-Z) are read in one burst that never
+yields to the loop, so the 4,096-frame client queue filled before its consumer had a turn: the
+reader reported the turn's outcome as unknown, and the next prompt failed while the socket closed.
+A Kanban worker, subscribed by ``kanban.run`` but reading only receipts, filled it the same way.
+"""
+import asyncio
+import json
+from contextlib import asynccontextmanager
+
+import pytest
+
+from hermes_cli import gateway_client
+from hermes_cli.gateway_client import GatewayClient, GatewayClientError
+from hermes_cli.gateway_connection import GatewayConnection
+
+SID = "local-backlog"
+
+
+def _frame(kind, text):
+ return json.dumps({"jsonrpc": "2.0", "method": "event", "params": {
+ "type": kind, "session_id": SID, "payload": {"text": text}}})
+
+
+class Backlog:
+ """A socket whose frames are all buffered already: iterating them never yields to the loop."""
+
+ def __init__(self, frames):
+ self.frames = frames
+
+ async def __aiter__(self):
+ for raw in self.frames:
+ yield raw
+ await asyncio.Event().wait() # the socket stays open
+
+
+def test_backlog_larger_than_the_client_queue_reaches_the_cli_intact():
+ deltas = [str(i) for i in range(6000)]
+ frames = [_frame("message.delta", text) for text in deltas] + [_frame("message.complete", "done")]
+
+ @asynccontextmanager
+ async def connect():
+ async with GatewayClient(Backlog(frames)) as client:
+ yield client
+
+ conn = GatewayConnection(connect=connect)
+ conn.start(timeout=5)
+ try:
+ seen = []
+ while not seen or not (isinstance(seen[-1], Exception) or seen[-1]["params"]["type"] == "message.complete"):
+ seen.append(conn.events.get(timeout=5))
+ finally:
+ conn.close()
+ assert not isinstance(seen[-1], Exception), seen[-1]
+ assert [frame["params"]["payload"]["text"] for frame in seen[:-1]] == deltas
+
+
+async def test_a_consumer_that_stops_draining_is_still_failed_with_the_queue_bounded(monkeypatch):
+ monkeypatch.setattr(gateway_client, "EVENT_STALL_TIMEOUT", 0.05, raising=False)
+ client = GatewayClient(Backlog([_frame("message.delta", str(i)) for i in range(6000)]))
+ pending = asyncio.get_running_loop().create_future()
+ client.pending[1] = pending
+ await asyncio.wait_for(client._read(), 5)
+ with pytest.raises(GatewayClientError, match="outcome is unknown"):
+ await pending
+ assert client.events.full()
+ queued = [client.events.get_nowait() for _ in range(client.events.maxsize)]
+ assert isinstance(queued[-1], GatewayClientError)
+
+
+class CardOwner:
+ """A Kanban card's owner as the worker's socket sees it: ``kanban.run`` subscribes the socket,
+ the card's turn streams ``events`` frames, and the next receipt poll finds it terminal."""
+
+ def __init__(self, events):
+ self.events = events
+ self.inbox = asyncio.Queue()
+
+ async def send(self, raw):
+ request = json.loads(raw)
+ if request["method"] == "kanban.run":
+ result = {"session_id": SID, "receipt": {"status": "running", "admission_id": "a1"}}
+ frames = [{"jsonrpc": "2.0", "id": request["id"], "result": result}]
+ frames += [json.loads(_frame("message.delta", str(i))) for i in range(self.events)]
+ else:
+ frames = [{"jsonrpc": "2.0", "id": request["id"], "result": {"status": "terminal", "admission_id": "a1"}}]
+ for frame in frames:
+ self.inbox.put_nowait(json.dumps(frame))
+
+ async def __aiter__(self):
+ while True:
+ yield await self.inbox.get()
+
+
+async def test_kanban_worker_settles_a_card_whose_turn_outstreams_the_client_queue(monkeypatch, tmp_path):
+ from gateway import session_kanban
+ from hermes_cli import kanban_worker_client
+
+ @asynccontextmanager
+ async def connect():
+ async with GatewayClient(CardOwner(6000)) as client:
+ yield client
+
+ monkeypatch.setattr(gateway_client, "connect_gateway", connect)
+ monkeypatch.setattr(session_kanban, "worker_result", lambda *_: {"exit_code": 0, "last_output": "card closed"})
+ monkeypatch.delenv("HERMES_TUI_GATEWAY_URL", raising=False)
+ assert await asyncio.wait_for(kanban_worker_client.run({"task_id": "t1"}, str(tmp_path / "kanban.db")), 5) == 0
+
+
+async def test_closing_as_a_waiting_frame_lands_still_ends_the_reader():
+ """A close that lands just as the consumer makes room still ends the reader, so ``__aexit__``
+ returns instead of waiting on the open socket forever."""
+ client = await GatewayClient(Backlog([_frame("message.delta", str(i)) for i in range(4097)])).__aenter__()
+ while not client.events.full(): # the reader yields only once it waits on the last frame
+ await asyncio.sleep(0)
+ client.events.get_nowait()
+ closing = asyncio.create_task(client.__aexit__(None, None, None))
+ try:
+ await asyncio.wait_for(asyncio.shield(closing), 2)
+ finally:
+ client.reader.cancel()
+ await asyncio.gather(closing, return_exceptions=True)
diff --git a/tests/hermes_cli/test_kanban_worker_owner_wait.py b/tests/hermes_cli/test_kanban_worker_owner_wait.py
index c8973e6b679..2b31658760a 100644
--- a/tests/hermes_cli/test_kanban_worker_owner_wait.py
+++ b/tests/hermes_cli/test_kanban_worker_owner_wait.py
@@ -6,6 +6,7 @@ of those trip the breaker and park a card that never ran (the in-flight handoff
must never submit the claim twice, and a non-transient refusal still fails at once.
"""
+import asyncio
from contextlib import asynccontextmanager
import pytest
@@ -25,6 +26,8 @@ def _owner(states, submitted):
raise unavailable(f"Gateway {state}: update_paused", state)
class _Client:
+ events = asyncio.Queue()
+
async def rpc(self, method, **params):
submitted.append(method)
return {"session_id": "s1", "receipt": {"status": "terminal", "admission_id": "a1"}}