The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(cli): keep reading toward an awaited answer and never drop ACP's post-ack recovery
Teknium3ad36cf9cdcc
diff --git a/acp_adapter/gateway_server.py b/acp_adapter/gateway_server.pyindex b5327facf0c..f24cf5e3a6d 100644--- a/acp_adapter/gateway_server.py+++ b/acp_adapter/gateway_server.py@@ -706,8 +706,9 @@ class GatewayACPAgent(GatewayReplyMixin, acp.Agent):await self._changed.wait_for(lambda: self._adopted.get(session_id, 0) > resumesor self._failure is not None or self._link != link)if self._adopted.get(session_id, 0) > resumes:- with suppress(asyncio.QueueFull): # full: the reader's own overflow ends this link- self._gateway.events.put_nowait((session_id, admission_id))+ # Held behind a full queue, never dropped: a full queue parks the reader, not the link.+ from hermes_cli.gateway_client import put_behind+ put_behind(self._gateway.events, (session_id, admission_id))async def _recover(self, session_id, admission_id):"""Queued: it lost nothing. Running: cut until its live completion. Unknown: the waiter refuses it.diff --git a/hermes_cli/gateway_client.py b/hermes_cli/gateway_client.pyindex 495e1f7167a..ba9f0a74444 100644--- a/hermes_cli/gateway_client.py+++ b/hermes_cli/gateway_client.py@@ -2,6 +2,7 @@from __future__ import annotationsimport asyncio+from collections import dequefrom contextlib import asynccontextmanager, suppressimport jsonimport os@@ -23,6 +24,11 @@ COMPRESS_RPC_TIMEOUT = 660# A full event queue parks the socket reader; a consumer that takes no event for this long has# stalled and is failed. Same bound as the owner's own send deadline (``tui_gateway/ws.py``).EVENT_STALL_TIMEOUT = 30+# Events read past a full queue while an RPC answer is awaited (the owner sends a re-attached+# session's frames ahead of its ``session.resume`` answer, and a recovery RPC is awaited from the+# queue's only consumer). Bounded: past either cap the reader waits for the consumer again.+EVENT_OVERFLOW_FRAMES = 16384+EVENT_OVERFLOW_BYTES = 8 * 1024 * 1024def gateway_ws_target(endpoint, ticket):@@ -93,14 +99,73 @@ class GatewayUnavailableError(GatewayClientError):self.state = state+class EventQueue(asyncio.Queue):+ """The client's bounded event queue plus an ordered overflow behind it: each ``get`` refills the+ queue from the overflow, so items arrive in read order and ``qsize``/``full`` see only the queue."""++ def __init__(self, maxsize):+ super().__init__(maxsize)+ self.overflow = deque() # (size, item)+ self.overflow_bytes = 0+ self.moved = asyncio.Event() # an item left the overflow (or an RPC started waiting)++ def _get(self):+ item = super()._get()+ if self.overflow:+ size, held = self.overflow.popleft()+ self.overflow_bytes -= size+ super()._put(held)+ self.moved.set()+ return item++ def hold(self, item, size=0):+ """Queue *item* behind everything already read, never dropping it."""+ if not self.overflow:+ try:+ self.put_nowait(item)+ return+ except asyncio.QueueFull:+ pass+ self.overflow.append((size, item))+ self.overflow_bytes += size+++def put_behind(events, item):+ """Queue a local item (ACP's post-ack recovery) behind every frame read so far; never dropped."""+ if isinstance(events, EventQueue):+ events.hold(item)+ else:+ events.put_nowait(item)++class GatewayClient:def __init__(self, websocket):self.websocket = websocket- self.events = asyncio.Queue(maxsize=4096)+ self.events = EventQueue(maxsize=4096)self.pending = {}self.sequence = 0self.reader = None+ def _read_on(self):+ """Keep reading past a full queue: nothing is held, or an awaited answer may sit behind the+ held frames (its waiter may be the queue's only consumer) and the overflow has room."""+ events = self.events+ return not events.overflow or (+ len(events.overflow) < EVENT_OVERFLOW_FRAMES and events.overflow_bytes < EVENT_OVERFLOW_BYTES+ and any(not future.done() for future in self.pending.values()))++ async def _deliver(self, frame, size):+ self.events.hold(frame, size)+ if self._read_on():+ return+ # Frames the socket already buffered (a CLI resumed from Ctrl-Z) are read without yielding,+ # so the consumer has not had a turn yet: wait for it (or for an RPC that needs the reader).+ # Not wait_for: on 3.11 it swallows a close that lands as the wait completes.+ async with asyncio.timeout(EVENT_STALL_TIMEOUT):+ while not self._read_on():+ self.events.moved.clear()+ await self.events.moved.wait()+async def __aenter__(self):self.reader = asyncio.create_task(self._read())return self@@ -122,14 +187,7 @@ class GatewayClient:else:future.set_result(frame.get("result", {}))elif "method" in frame:- try:- self.events.put_nowait(frame)- except asyncio.QueueFull:- # Frames the socket already buffered (a CLI resumed from Ctrl-Z) are read- # without yielding, so the consumer has not had a turn yet: wait for room.- # Not wait_for: on 3.11 it swallows a close that lands as the put completes.- async with asyncio.timeout(EVENT_STALL_TIMEOUT):- await self.events.put(frame)+ await self._deliver(frame, len(raw))except (ConnectionClosed, OSError, ValueError, TimeoutError):passfinally:@@ -138,10 +196,11 @@ class GatewayClient:for future in self.pending.values():if not future.done():future.set_exception(error)- # A stalled consumer is failed rather than silently losing terminal events.- if self.events.full():+ # A stalled consumer is failed rather than silently losing terminal events: behind+ # whatever is held, or in place of the oldest queued frame when nothing is.+ if self.events.full() and not self.events.overflow:self.events.get_nowait()- self.events.put_nowait(error)+ self.events.hold(error)async def rpc(self, method, *, _timeout=None, **params):"""Send one request and await its answer; ``_timeout`` (seconds, never sent on the wire)@@ -151,6 +210,7 @@ class GatewayClient:rid = self.sequencefuture = asyncio.get_running_loop().create_future()self.pending[rid] = future+ self.events.moved.set() # a reader waiting on the consumer reads on toward this answertry:await self.websocket.send(json.dumps({"jsonrpc": "2.0", "id": rid, "method": method, "params": params}))budget = DEFAULT_RPC_TIMEOUT if _timeout is None else _timeoutdiff --git a/tests/hermes_cli/test_gateway_client_backlog.py b/tests/hermes_cli/test_gateway_client_backlog.pyindex b55ec49f4d7..845b2ed4246 100644--- a/tests/hermes_cli/test_gateway_client_backlog.py+++ b/tests/hermes_cli/test_gateway_client_backlog.py@@ -58,16 +58,19 @@ def test_backlog_larger_than_the_client_queue_reaches_the_cli_intact():@pytest.mark.asyncioasync def test_a_consumer_that_stops_draining_is_still_failed_with_the_queue_bounded(monkeypatch):+ """An awaited answer lets the reader read past a full queue, but only into a bounded overflow."""monkeypatch.setattr(gateway_client, "EVENT_STALL_TIMEOUT", 0.05, raising=False)+ monkeypatch.setattr(gateway_client, "EVENT_OVERFLOW_FRAMES", 100)client = GatewayClient(Backlog([_frame("message.delta", str(i)) for i in range(6000)]))pending = asyncio.get_running_loop().create_future()client.pending[1] = pendingawait asyncio.wait_for(client._read(), 5)with pytest.raises(GatewayClientError, match="outcome is unknown"):await pending- assert client.events.full()- queued = [client.events.get_nowait() for _ in range(client.events.maxsize)]- assert isinstance(queued[-1], GatewayClientError)+ assert client.events.full() and len(client.events.overflow) == 101 # 100 frames + the failure+ queued = [client.events.get_nowait() for _ in range(client.events.maxsize + 101)]+ assert client.events.empty() and isinstance(queued[-1], GatewayClientError)+ assert [frame["params"]["payload"]["text"] for frame in queued[:-1]] == [str(i) for i in range(4196)]class CardOwner:diff --git a/tests/hermes_cli/test_gateway_client_recovery_backlog.py b/tests/hermes_cli/test_gateway_client_recovery_backlog.pynew file mode 100644index 00000000000..feadcc01bab--- /dev/null+++ b/tests/hermes_cli/test_gateway_client_recovery_backlog.py@@ -0,0 +1,148 @@+"""Recovery RPCs awaited from a client's only event consumer (#134889 CLI-35/CLI-37 review).++``GatewayChatView`` and ACP await ``session.resume`` / ``prompt.receipt`` from the task that drains+``GatewayClient.events``. The owner attaches the new subscription before it answers, so a burst+larger than the client queue can sit ahead of the answer: the reader must keep reading toward it,+and ACP's post-ack recovery must never be dropped when that queue is full. Real ``GatewayClient``+over a fake socket; probes from @dokterdok's review.+"""+import asyncio+import json+from unittest.mock import AsyncMock++import pytest+++def event(kind, seq=1, sid='s', admission=None):+ return {'jsonrpc': '2.0', 'method': 'event', 'params': {+ 'session_id': sid, 'type': kind, 'replay_epoch': 'e', 'seq': seq, 'admission_id': admission,+ 'payload': {'text': 'x'}}}+++class ResumeWire:+ def __init__(self, burst):+ self.burst = burst+ self.frames = asyncio.Queue()+ self.answer_read = asyncio.Event()++ async def send(self, raw):+ req = json.loads(raw)+ assert req['method'] == 'session.resume'+ # An attached owner can send a batch before its resume answer (owner fanout: 16,384+ # frames; the client queue: 4,096).+ for seq in range(2, self.burst + 2):+ self.frames.put_nowait(event('message.delta', seq))+ self.frames.put_nowait({'jsonrpc': '2.0', 'id': req['id'], 'result': {+ 'stored_session_id': 's', 'replay_epoch': 'e', 'last_sequence': 1, 'pending': [], 'prompts': []}})++ async def __aiter__(self):+ while True:+ frame = await self.frames.get()+ if 'id' in frame:+ self.answer_read.set()+ yield json.dumps(frame)++[email protected]('consumer', ['chat', 'acp'])[email protected]('burst', [4096, 4097, 6000])+async def test_resume_answer_behind_a_burst_stays_reachable(monkeypatch, consumer, burst):+ from hermes_cli import gateway_client+ monkeypatch.setattr(gateway_client, 'EVENT_STALL_TIMEOUT', 30)+ wire = ResumeWire(burst)+ async with gateway_client.GatewayClient(wire) as client:+ if consumer == 'chat':+ from hermes_cli.gateway_chat_view import GatewayChatView+ view = GatewayChatView(client, {'stored_session_id': 's'}, quiet=True)+ pump = asyncio.create_task(view.render())+ failure = lambda: view.failure+ else:+ from acp_adapter.gateway_server import GatewayACPAgent+ view = GatewayACPAgent()+ view._gateway, view._conn = client, AsyncMock()+ view._snapshots['s'] = {'replay_epoch': 'e', 'last_sequence': 0}+ pump = view._event_task = asyncio.create_task(view._events())+ failure = lambda: view._failure+ answered = asyncio.create_task(wire.answer_read.wait())+ try:+ wire.frames.put_nowait(event('session.replay_gap'))+ # Well under EVENT_STALL_TIMEOUT: a parked reader would not reach the answer in time.+ await asyncio.wait({pump, answered}, timeout=10, return_when=asyncio.FIRST_COMPLETED)+ assert wire.answer_read.is_set(), f'{consumer}: answer unread, queued={client.events.qsize()}'+ assert failure() is None+ finally:+ pump.cancel()+ answered.cancel()+ await asyncio.gather(pump, answered, return_exceptions=True)+++class PostAckWire:+ def __init__(self):+ self.frames = asyncio.Queue()+ self.submit_id = None+ self.receipts = []++ async def send(self, raw):+ req = json.loads(raw)+ if req['method'] == 'prompt.submit':+ self.submit_id = req['id']+ self.frames.put_nowait(event('session.replay_gap', admission='other'))+ elif req['method'] == 'session.resume':+ # The turn completed in the gap before its ack reached ACP.+ self.frames.put_nowait({'id': req['id'], 'result': {+ 'stored_session_id': 's', 'replay_epoch': 'e', 'last_sequence': 1, 'pending': [], 'prompts': []}})+ self.frames.put_nowait(event('message.delta', sid='t', admission='other'))+ elif req['method'] == 'prompt.receipt':+ self.receipts.append(req['params']['admission_id'])+ self.frames.put_nowait({'id': req['id'], 'result': {+ 'admission_id': 'a', 'status': 'terminal', 'outcome': 'completed',+ 'result': {'final_response': 'finished'}}})+ else:+ raise AssertionError(req)++ async def __aiter__(self):+ while True:+ yield json.dumps(await self.frames.get())+++async def test_a_full_queue_never_drops_a_post_ack_recovery():+ """Session t fills the queue while session s's ack lands after a gap re-attach covering it."""+ import acp+ from acp_adapter.gateway_server import GatewayACPAgent+ from hermes_cli.gateway_client import GatewayClient+ from tests.waiting import async_wait_until++ wire = PostAckWire()+ entered, release = asyncio.Event(), asyncio.Event()+ async with GatewayClient(wire) as client:+ agent = GatewayACPAgent()+ agent._gateway, agent._conn = client, AsyncMock()+ agent._snapshots.update({sid: {'replay_epoch': 'e', 'last_sequence': 0} for sid in ('s', 't')})++ async def deliver(**kwargs):+ if kwargs['session_id'] == 't' and not entered.is_set():+ entered.set()+ await release.wait()++ agent._conn.session_update.side_effect = deliver+ pump = agent._event_task = asyncio.create_task(agent._events())+ prompt = asyncio.create_task(agent.prompt([acp.text_block('q')], 's'))+ try:+ await async_wait_until(lambda: entered.is_set() or prompt.done() or agent._failure is not None, timeout=5)+ assert entered.is_set(), f'recovery failed before the post-ack race: {agent._failure}'+ for seq in range(2, client.events.maxsize + 2):+ wire.frames.put_nowait(event('message.delta', seq, sid='t', admission='other'))+ wire.frames.put_nowait({'id': wire.submit_id, 'result': {'admission_id': 'a', 'status': 'queued'}})+ await async_wait_until(lambda: agent._admissions.get('s') == 'a', timeout=5)+ assert client.events.full(), 'probe did not exercise the full-queue branch'+ await asyncio.sleep(0) # let _after_ack hand its recovery over+ release.set()+ response = await asyncio.wait_for(prompt, 10)+ assert response.stop_reason == 'end_turn', (wire.receipts, agent._failure)+ assert wire.receipts == ['a']+ finally:+ release.set()+ prompt.cancel()+ pump.cancel()+ await asyncio.gather(prompt, pump, return_exceptions=True)