Skip to content

The agent that grows with you

fix(acp): refresh lost-turn state before refusing the next prompt

David Dudok de Wita9e38dce929a
diff --git a/acp_adapter/gateway_server.py b/acp_adapter/gateway_server.py
index fe74fc90bf0..e334165ce15 100644
--- a/acp_adapter/gateway_server.py
+++ b/acp_adapter/gateway_server.py
@@ -566,6 +566,8 @@ class GatewayACPAgent(GatewayReplyMixin, acp.Agent):
held_id, session_id = session_id, self._aliases.get(session_id, session_id)
if session_id not in self._snapshots:
raise GatewayClientError("not_found")
+ if self._has_unknown(session_id) and await self._refresh_unknown_for_prompt(session_id):
+ return PromptResponse(stop_reason='cancelled')
if self._has_unknown(session_id):
raise GatewayClientError(self._unknown_recovery(session_id, submitted=False))
prepared = await self._prepare_submit(session_id, prompt)
@@ -726,6 +728,52 @@ class GatewayACPAgent(GatewayReplyMixin, acp.Agent):
return any(row.get("status") == "unknown"
for row in self._snapshots[session_id].get("pending", []))
+ async def _refresh_unknown_for_prompt(self, session_id):
+ # Stop during the owner's read must cancel the still-unsent editor input.
+ self._submitting.add(session_id)
+ try:
+ await self._refresh_unknown(session_id)
+ except BaseException:
+ self._pending_cancels.discard(session_id)
+ raise
+ finally:
+ self._submitting.discard(session_id)
+ cancelled = session_id in self._pending_cancels
+ self._pending_cancels.discard(session_id)
+ return cancelled
+
+ async def _refresh_unknown(self, session_id):
+ """Ask the owner whether another viewer has acknowledged a cached lost turn."""
+ client = await self._client() # A reconnect may already supply a fresh snapshot.
+ cached = self._snapshots.get(session_id)
+ if cached is None:
+ raise GatewayClientError("not_found")
+ if not self._has_unknown(session_id):
+ return
+ # Freeze the snapshot's pending watermark before the pump can advance its reply cursor.
+ cached.setdefault("_pending_watermark", (cached.get("replay_epoch"), cached.get("last_sequence", 0)))
+ fresh = await client.rpc("session.resume", session_id=session_id)
+ cached = self._snapshots.get(session_id)
+ if cached is None:
+ raise GatewayClientError("not_found")
+ # Only pending state received while the RPC was in flight can supersede its snapshot.
+ epoch, sequence = cached.get("_pending_watermark",
+ (cached.get("replay_epoch"), cached.get("last_sequence", 0)))
+ if "pending" in fresh and (
+ fresh.get("replay_epoch") != epoch or fresh.get("last_sequence", 0) >= sequence
+ ):
+ cached["pending"] = fresh["pending"]
+ # The snapshot supplies only pending state here. Do not consume earlier message,
+ # approval, or tool events that the editor's event pump has not projected yet.
+ cached["_pending_watermark"] = (fresh.get("replay_epoch"), fresh.get("last_sequence", 0))
+
+ def _pending_event_is_new(self, session_id, event):
+ """Keep a pre-refresh pending event from restoring an already-discarded lost turn."""
+ fence = self._snapshots.get(session_id, {}).get("_pending_watermark")
+ sequence = event.get("seq")
+ return (fence is None or sequence is None or event.get("replay_epoch") != fence[0]
+ or sequence > fence[1])
+
def _blocked_admission(self, session_id, admission_id):
return self._has_unknown(session_id) and any(
row.get("admission_id") == admission_id and row.get("status") in {"queued", "unknown"}
@@ -764,7 +812,7 @@ class GatewayACPAgent(GatewayReplyMixin, acp.Agent):
sid, editor_id = event["session_id"], self._editor_id(event["session_id"])
aid = event.get("admission_id")
if kind in {"session.info", "session.updated", "session.usage", "session.tools_changed"}:
- await self._project_session_state(kind, sid, editor_id, payload)
+ await self._project_session_state(kind, sid, editor_id, payload, event)
return
if kind == "session.replay_gap":
raise GatewayClientError("session_replay_gap")
@@ -815,7 +863,7 @@ class GatewayACPAgent(GatewayReplyMixin, acp.Agent):
finally:
self._settling.discard(aid)
- async def _project_session_state(self, kind, sid, editor_id, payload):
+ async def _project_session_state(self, kind, sid, editor_id, payload, event):
if kind == "session.usage":
await self._project_usage(sid, editor_id, payload)
return
@@ -841,8 +889,10 @@ class GatewayACPAgent(GatewayReplyMixin, acp.Agent):
return
if kind == "session.info" and sid in self._snapshots and "pending" in payload:
async with self._changed:
- self._snapshots[sid]["pending"] = payload["pending"]
- self._changed.notify_all()
+ if self._pending_event_is_new(sid, event):
+ self._snapshots[sid]["pending"] = payload["pending"]
+ self._snapshots[sid]["_pending_watermark"] = (event.get("replay_epoch"), event.get("seq", 0))
+ self._changed.notify_all()
await self._project_title(sid, editor_id, payload.get("title"))
async def _project_usage(self, sid, editor_id, stats):
diff --git a/tests/acp_adapter/test_acp_reattach_unknown_notice.py b/tests/acp_adapter/test_acp_reattach_unknown_notice.py
index 3f86e7779f1..3bc9437af33 100644
--- a/tests/acp_adapter/test_acp_reattach_unknown_notice.py
+++ b/tests/acp_adapter/test_acp_reattach_unknown_notice.py
@@ -1,10 +1,12 @@
"""A reconnect that re-attaches a session with a lost turn tells the editor at once, like load/resume
-(JM44 r1-P1-4): real websockets, the real GatewayClient, two owners."""
+(JM44 r1-P1-4), and one that finds the turn discarded admits the next prompt (#140521): real
+websockets, the real GatewayClient, two owners."""
import asyncio
from contextlib import asynccontextmanager
import json
from unittest.mock import AsyncMock
+import acp
import pytest
from websockets.asyncio.client import connect
from websockets.asyncio.server import serve
@@ -14,26 +16,29 @@ from acp_adapter.gateway_server import GatewayACPAgent
from hermes_cli.gateway_client import GatewayClient
-def _owner(profile_id, epoch, pending):
+def _owner(profile_id, epoch, pending, calls=None):
async def handler(ws):
async for raw in ws:
frame = json.loads(raw)
- result = {"profile_id": profile_id} if frame["method"] == "runtime.describe" else {}
- if frame["method"] == "session.resume":
+ method = frame["method"]
+ if calls is not None:
+ calls.append(method)
+ result = {"profile_id": profile_id} if method == "runtime.describe" else {}
+ if method == "session.resume":
result = {"session_id": frame["params"]["session_id"], "messages": [], "prompts": [],
"pending": pending, "replay_epoch": epoch, "last_sequence": 0}
+ if method == "prompt.submit":
+ result = {"admission_id": "next"}
await ws.send(json.dumps({"jsonrpc": "2.0", "id": frame["id"], "result": result}))
+ if method == "prompt.submit":
+ await ws.send(json.dumps({"jsonrpc": "2.0", "method": "event", "params": {
+ "session_id": frame["params"]["session_id"], "type": "message.complete", "seq": 1,
+ "replay_epoch": epoch, "admission_id": "next", "payload": {"outcome": "completed"}}}))
return handler
-async def test_reattach_after_restart_posts_the_unknown_recovery_notice(monkeypatch):
- agent = GatewayACPAgent()
- agent._conn = AsyncMock()
- home = str(agent._home)
- first = await serve(_owner(home, "first", []), "127.0.0.1", 0)
- second = await serve(_owner(home, "second", [{"admission_id": "lost-1", "status": "unknown"}]), "127.0.0.1", 0)
- order = iter([first, second])
+def _connector(monkeypatch, *servers):
+ order = iter(servers)
@asynccontextmanager
async def connect_gateway():
@@ -42,6 +47,16 @@ async def test_reattach_after_restart_posts_the_unknown_recovery_notice(monkeypa
yield client
monkeypatch.setattr(server, "connect_gateway", connect_gateway)
+
+
+async def test_reattach_after_restart_posts_the_unknown_recovery_notice(monkeypatch):
+ agent = GatewayACPAgent()
+ agent._conn = AsyncMock()
+ home = str(agent._home)
+ first = await serve(_owner(home, "first", []), "127.0.0.1", 0)
+ second = await serve(_owner(home, "second", [{"admission_id": "lost-1", "status": "unknown"}]), "127.0.0.1", 0)
+ _connector(monkeypatch, first, second)
try:
await agent.load_session(cwd="/", session_id="s")
before = agent._conn.session_update.await_count # load's own notices (no unknown yet)
@@ -57,3 +72,26 @@ async def test_reattach_after_restart_posts_the_unknown_recovery_notice(monkeypa
await agent.aclose()
second.close()
+
+async def test_next_prompt_reconnects_after_the_lost_turn_was_discarded_while_offline(monkeypatch):
+ """The editor cached the lost turn, then lost its gateway connection (another restart, a dropped
+ socket) while the classic chat discarded that turn: its next prompt reaches the owner."""
+ agent = GatewayACPAgent()
+ agent._conn = AsyncMock()
+ home, calls = str(agent._home), []
+ first = await serve(_owner(home, "first", [{"admission_id": "lost-1", "status": "unknown"}]), "127.0.0.1", 0)
+ second = await serve(_owner(home, "second", [], calls), "127.0.0.1", 0)
+ _connector(monkeypatch, first, second)
+ try:
+ await agent.load_session(cwd="/", session_id="s")
+ await asyncio.gather(*agent._background) # load's post-attach reads stay on the first owner
+ first.close()
+ await first.wait_closed()
+ await asyncio.wait_for(agent._event_task, 5)
+ response = await asyncio.wait_for(agent.prompt([acp.text_block("next")], "s"), 5)
+ assert response.stop_reason == "end_turn"
+ assert calls == ["runtime.describe", "session.resume", "config.set", "prompt.submit"]
+ finally:
+ await agent.aclose()
+ second.close()
diff --git a/tests/acp_adapter/test_gateway_queued_unknown.py b/tests/acp_adapter/test_gateway_queued_unknown.py
index 97046cf9b49..ef7abd823ae 100644
--- a/tests/acp_adapter/test_gateway_queued_unknown.py
+++ b/tests/acp_adapter/test_gateway_queued_unknown.py
@@ -1,5 +1,6 @@
"""Editor prompts fail recoverably when their FIFO becomes unknown."""
import asyncio
+from contextlib import suppress
from unittest.mock import AsyncMock
import acp
@@ -47,9 +48,168 @@ async def test_editor_refuses_new_prompt_until_lost_turn_is_resolved():
agent = GatewayACPAgent()
agent._gateway = AsyncMock()
agent._snapshots['s'] = {'pending': [{'admission_id': 'lost', 'status': 'unknown'}]}
+ agent._gateway.rpc.return_value = {'pending': [{'admission_id': 'lost', 'status': 'unknown'}]}
with pytest.raises(GatewayClientError, match='unknown_execution'):
await asyncio.wait_for(agent.prompt([acp.text_block('new input')], 's'), 5)
- agent._gateway.rpc.assert_not_awaited()
+ agent._gateway.rpc.assert_awaited_once_with('session.resume', session_id='s')
+
+
+async def test_editor_rechecks_lost_turn_after_discard_in_another_viewer():
+ agent = GatewayACPAgent()
+ agent._gateway = AsyncMock()
+ lost = [{'admission_id': 'lost', 'status': 'unknown'}]
+ agent._snapshots['s'] = {'pending': lost, 'replay_epoch': 'owner', 'last_sequence': 1}
+ owner_pending = list(lost)
+ calls = []
+
+ async def rpc(method, **params):
+ calls.append(method)
+ if method == 'session.resume':
+ return {'pending': list(owner_pending), 'replay_epoch': 'owner', 'last_sequence': 2}
+ assert method == 'prompt.submit' and params['session_id'] == 's'
+ await agent._project({'session_id': 's', 'type': 'message.complete',
+ 'admission_id': 'next', 'payload': {'outcome': 'completed'}})
+ return {'admission_id': 'next'}
+
+ agent._gateway.rpc.side_effect = rpc
+ with pytest.raises(GatewayClientError, match='unknown_execution'):
+ await agent.prompt([acp.text_block('still blocked')], 's')
+ assert 'prompt.submit' not in calls
+
+ owner_pending.clear() # The classic chat acknowledged the lost turn with /discard.
+ response = await asyncio.wait_for(agent.prompt([acp.text_block('next prompt')], 's'), 5)
+ assert response.stop_reason == 'end_turn'
+ assert calls.count('prompt.submit') == 1
+
+
+async def test_unknown_refresh_keeps_unread_reply_events_without_restoring_old_pending():
+ agent = GatewayACPAgent()
+ agent._gateway = AsyncMock()
+ agent._gateway.events = asyncio.Queue()
+ agent._conn = AsyncMock()
+ approval_seen, release_approval = asyncio.Event(), asyncio.Event()
+
+ async def show_approval(_session_id, _prompt):
+ approval_seen.set()
+ await release_approval.wait()
+ return True
+
+ agent._answer_permission = show_approval
+ agent._snapshots['s'] = {'pending': [{'admission_id': 'lost', 'status': 'unknown'}],
+ 'replay_epoch': 'owner', 'last_sequence': 1}
+ agent._gateway.rpc.return_value = {'pending': [], 'replay_epoch': 'owner', 'last_sequence': 4}
+
+ await agent._refresh_unknown('s')
+ assert not agent._has_unknown('s')
+ # The event pump was busy while the owner discarded the head and ran a follower.
+ # Its old pending event must not undo the refresh; its reply still reaches the editor.
+ await agent._gateway.events.put({'params': {
+ 'session_id': 's', 'replay_epoch': 'owner', 'seq': 2, 'type': 'session.info',
+ 'payload': {'pending': [{'admission_id': 'lost', 'status': 'unknown'}]}}})
+ await agent._gateway.events.put({'params': {
+ 'session_id': 's', 'replay_epoch': 'owner', 'seq': 3, 'type': 'approval.request',
+ 'payload': {'kind': 'approval', 'prompt_id': 'approval-1', 'execution_generation': 1}}})
+ await agent._gateway.events.put({'params': {
+ 'session_id': 's', 'replay_epoch': 'owner', 'seq': 4, 'type': 'message.delta',
+ 'admission_id': 'follower', 'payload': {'text': 'Follower answer'}}})
+ task = asyncio.create_task(agent._events())
+ try:
+ await async_wait_until(lambda: approval_seen.is_set() and agent._conn.session_update.await_count == 1,
+ what='the buffered approval and follower reply reached the editor')
+ assert agent._conn.session_update.await_args.kwargs['update'].content.text == 'Follower answer'
+ assert not agent._has_unknown('s')
+ finally:
+ release_approval.set()
+ task.cancel()
+ with suppress(asyncio.CancelledError):
+ await task
+ await asyncio.gather(*agent._permissions.values(), return_exceptions=True)
+
+
[email protected]('newer_pending', [False, True])
+async def test_unknown_refresh_orders_pending_state_independently_of_reply_events(newer_pending):
+ agent = GatewayACPAgent()
+ agent._gateway = AsyncMock()
+ agent._gateway.events = asyncio.Queue()
+ agent._conn = AsyncMock()
+ lost = [{'admission_id': 'lost', 'status': 'unknown'}]
+ agent._snapshots['s'] = {'pending': lost, 'replay_epoch': 'owner', 'last_sequence': 1}
+ entered, release = asyncio.Event(), asyncio.Event()
+
+ async def rpc(method, **params):
+ if method == 'session.resume':
+ entered.set()
+ await release.wait()
+ return {'pending': [], 'replay_epoch': 'owner', 'last_sequence': 3}
+ assert method == 'prompt.submit'
+ await agent._project({'session_id': 's', 'type': 'message.complete',
+ 'admission_id': 'next', 'payload': {'outcome': 'completed'}})
+ return {'admission_id': 'next'}
+
+ agent._gateway.rpc.side_effect = rpc
+ pump = asyncio.create_task(agent._events())
+ prompt = asyncio.create_task(agent.prompt([acp.text_block('next prompt')], 's'))
+ try:
+ await asyncio.wait_for(entered.wait(), 5)
+ # While the resume reply is assembled, the pump drains old pending state and a
+ # follower's newer reply. Only a newer pending event can supersede the snapshot.
+ await agent._gateway.events.put({'params': {
+ 'session_id': 's', 'replay_epoch': 'owner', 'seq': 2,
+ 'type': 'session.info', 'payload': {'pending': lost}}})
+ await agent._gateway.events.put({'params': {
+ 'session_id': 's', 'replay_epoch': 'owner', 'seq': 4, 'type': 'message.delta',
+ 'admission_id': 'follower', 'payload': {'text': 'Follower answer'}}})
+ if newer_pending:
+ await agent._gateway.events.put({'params': {
+ 'session_id': 's', 'replay_epoch': 'owner', 'seq': 5, 'type': 'session.info',
+ 'payload': {'pending': [{'admission_id': 'follower', 'status': 'unknown'}]}}})
+ await async_wait_until(lambda: agent._snapshots['s']['last_sequence'] == (5 if newer_pending else 4),
+ what='the follower events reached the editor before the resume response')
+ assert agent._conn.session_update.await_args.kwargs['update'].content.text == 'Follower answer'
+ release.set()
+ if newer_pending:
+ with pytest.raises(GatewayClientError, match='unknown_execution'):
+ await asyncio.wait_for(prompt, 5)
+ else:
+ assert (await asyncio.wait_for(prompt, 5)).stop_reason == 'end_turn'
+ methods = [call.args[0] for call in agent._gateway.rpc.await_args_list]
+ assert methods.count('prompt.submit') == (0 if newer_pending else 1)
+ finally:
+ release.set()
+ prompt.cancel()
+ pump.cancel()
+ await asyncio.gather(prompt, pump, return_exceptions=True)
+
+
+async def test_stop_during_unknown_refresh_cancels_the_unsent_prompt():
+ agent = GatewayACPAgent()
+ agent._gateway = AsyncMock()
+ agent._snapshots['s'] = {'pending': [{'admission_id': 'lost', 'status': 'unknown'}]}
+ entered, release = asyncio.Event(), asyncio.Event()
+ calls = []
+
+ async def rpc(method, **params):
+ calls.append(method)
+ if method == 'session.resume':
+ entered.set()
+ await release.wait()
+ return {'pending': []}
+ assert method == 'prompt.submit'
+ await agent._project({'session_id': 's', 'type': 'message.complete',
+ 'admission_id': 'next', 'payload': {'outcome': 'completed'}})
+ return {'admission_id': 'next'}
+
+ agent._gateway.rpc.side_effect = rpc
+ prompt = asyncio.create_task(agent.prompt([acp.text_block('new input')], 's'))
+ await asyncio.wait_for(entered.wait(), 5)
+ await agent.cancel('s')
+ release.set()
+ assert (await asyncio.wait_for(prompt, 5)).stop_reason == 'cancelled'
+ assert calls == ['session.resume']
@pytest.mark.asyncio
@@ -73,9 +233,9 @@ async def test_resumed_unknown_turn_names_the_existing_discard_recovery():
assert 'chat --cli --resume s' in text and f'/discard {lost}' in text, text
assert 'do not resend' in text
assert 'nothing was submitted' in str(refused.value)
- # The load's own RPCs only (the session's reasoning flag included): no prompt.submit.
+ # The prompt refreshed the owner's pending rows; no prompt was submitted.
asked = [call.args[0] for call in agent._gateway.rpc.await_args_list]
- assert asked == ['session.info', 'session.resume', 'config.set']
+ assert asked == ['session.info', 'session.resume', 'config.set', 'session.resume']
@pytest.mark.asyncio