Skip to content

The agent that grows with you

fix(cli): a local turn no longer loses a started /queue turn's start

Tekniume84df131a4bc
diff --git a/hermes_cli/gateway_agent.py b/hermes_cli/gateway_agent.py
index 1565322a535..59354523a94 100644
--- a/hermes_cli/gateway_agent.py
+++ b/hermes_cli/gateway_agent.py
@@ -695,8 +695,10 @@ class GatewayAgent:
adopted = take_admitted_turn(self, persist_user_message if isinstance(persist_user_message, str)
else user_message)
if adopted is not None:
- # A notification turn the gateway already admitted and is running (its frames are queued).
+ # A turn the gateway already admitted and is running (its frames are queued). A recovered
+ # ``/queue`` turn missed some frames: its reply is completed from the receipt.
receipt = adopted
+ turn.gapped = bool(adopted.get("recovered"))
else:
self._drain_stale()
self._taken_turn_route = None
@@ -730,7 +732,8 @@ class GatewayAgent:
self._generation = receipt["execution_generation"]
starts: dict = {}
try:
- outcome = self._pump(stream_callback, starts, turn)
+ settled = (adopted or {}).get("receipt")
+ outcome = {"_receipt": settled} if settled else self._pump(stream_callback, starts, turn)
finally:
if self._owns(turn):
self._generation_done()
@@ -745,7 +748,9 @@ class GatewayAgent:
return result
def _drain_stale(self):
- """Frames left from an earlier turn of this viewer (a late settlement) are not this turn's."""
+ """Frames left from an earlier turn of this viewer (a late settlement) are not this turn's. A
+ ``/queue`` admission's start among them is remembered: that turn is painted after this one."""
+ from hermes_cli.gateway_tools_notify import missed_start
while True:
try:
frame = self.connection.events.get_nowait()
@@ -756,6 +761,9 @@ class GatewayAgent:
continue # a connection a reconnect already replaced
self.connection.events.put(frame)
return
+ params = frame.get("params") if isinstance(frame, dict) else None
+ if isinstance(params, dict) and params.get("session_id") == self.session_id:
+ missed_start(self._state(), params)
def _generation_done(self):
self._generation = None
@@ -803,6 +811,8 @@ class GatewayAgent:
kind, payload = params.get("type"), params.get("payload") or {}
admission = params.get("admission_id") or (payload.get("admission_id") if isinstance(payload, dict) else None)
if self._admission and admission and admission != self._admission:
+ from hermes_cli.gateway_tools_notify import missed_start
+ missed_start(self._state(), params) # a /queue turn run ahead of this one: painted after it
continue
if params.get("execution_generation") is not None:
self._generation = params["execution_generation"]
diff --git a/hermes_cli/gateway_connection.py b/hermes_cli/gateway_connection.py
index e53fd9312d3..5d765b56e79 100644
--- a/hermes_cli/gateway_connection.py
+++ b/hermes_cli/gateway_connection.py
@@ -70,6 +70,12 @@ class SessionState:
# one whose item the input loop just took (``gateway_tools_notify.arm_adopted``).
self.adopted: dict = {}
self.armed_admission: str | None = None
+ # Each pending list the owner published (or a snapshot carried) bumps ``pending_seq``; a
+ # ``/queue`` admission whose ``message.start`` this viewer dropped unpainted (a local turn's
+ # drain or pump) is in ``missed``. Both make the idle tick read its receipt (``recover_queued``).
+ self.pending_seq = 0
+ self.recovered_seq = 0
+ self.missed: set = set()
# ``session.replay_gap``s seen, and how many a re-attach covered: the owner retired this viewer's
# subscription on overflow, so its frames stop until ``GatewayConnection.reattach``.
self.gaps = self.attached_gaps = 0
@@ -96,6 +102,7 @@ class SessionState:
self.info.update({k: v for k, v in payload.items() if k != "session_id"})
if isinstance(payload.get("pending"), list):
self.pending = list(payload["pending"])
+ self.pending_seq += 1
if payload.get("title"):
self.title = payload["title"]
elif kind == "todo.updated" and isinstance(payload, dict):
diff --git a/hermes_cli/gateway_tools_notify.py b/hermes_cli/gateway_tools_notify.py
index e792c4ed248..cb4236c3242 100644
--- a/hermes_cli/gateway_tools_notify.py
+++ b/hermes_cli/gateway_tools_notify.py
@@ -41,12 +41,15 @@ def queue_admitted_turn(cli, store) -> None:
from tools.process_registry_notifications import TimelineNotification
connection, session_id = store.connection, getattr(cli, "session_id", None)
state = connection.state(session_id)
+ if _adoption_waiting(cli, state):
+ return # an adopted /queue turn is waiting for the input loop: its frames stay queued for its pump
state.admitted_turn = None # a header queued earlier whose turn never ran is stale now
events = connection.events
while True:
try:
frame = events.get_nowait()
except queue.Empty:
+ recover_queued(cli, store)
return
if isinstance(frame, Exception):
from hermes_cli.gateway_connection import lost_before
@@ -66,16 +69,15 @@ def queue_admitted_turn(cli, store) -> None:
continue
if params.get("type") != "message.start" or not params.get("admission_id"):
continue
- queued_text = state.queued_inputs.pop(params["admission_id"], None)
+ queued_text = state.queued_inputs.get(params["admission_id"])
if queued_text is not None:
# A ``/queue`` prompt this viewer admitted (#134889 N5): the owner runs it now, and main's
# input path paints and pumps it as a typed turn. Literal: it is a prompt, never a command.
# Adopted by admission id (the item carries it), never by text: the input path may rewrite
# the text, and a second ``prompt.submit`` would run the prompt twice.
from cli import _SeededQueryMessage
- state.adopted[params["admission_id"]] = {"admission_id": params["admission_id"],
- "execution_generation": params.get("execution_generation")}
- _put_first(cli._pending_input, _SeededQueryMessage(queued_text, admission_id=params["admission_id"]))
+ _adopt(cli, state, {"admission_id": params["admission_id"],
+ "execution_generation": params.get("execution_generation")}, queued_text)
return
card = _card(params)
if card is None:
@@ -88,6 +90,81 @@ def queue_admitted_turn(cli, store) -> None:
return
+def _adopt(cli, state, adopted, text):
+ from cli import _SeededQueryMessage
+ state.missed.discard(adopted["admission_id"])
+ state.adopted[adopted["admission_id"]] = adopted
+ _put_first(cli._pending_input, _SeededQueryMessage(text, admission_id=adopted["admission_id"]))
+
+
+def _adoption_waiting(cli, state) -> bool:
+ """An adopted item is still in ``_pending_input`` (one ``/queue clear`` dropped is forgotten)."""
+ if not state.adopted:
+ return False
+ pending = getattr(cli, "_pending_input", None)
+ if pending is None:
+ return False
+ with pending.mutex:
+ waiting = {getattr(item, "admission_id", None) for item in pending.queue}
+ waiting.add(state.armed_admission) # taken by the input loop, its run_conversation not started yet
+ for admission in [a for a in state.adopted if a not in waiting]:
+ state.adopted.pop(admission, None)
+ return bool(state.adopted)
+
+
+_NEVER_RAN = {"cancelled", "rejected"}
+
+
+def recover_queued(cli, store) -> None:
+ """Idle tick, after the stream had no ``message.start`` for them: this viewer's ``/queue``
+ admissions the owner started or settled without this viewer painting the turn (a local turn's
+ drain or pump dropped the start, or it ran across a reconnect, which replays no frames). Read on
+ a new published pending list or a dropped start only, never every tick.
+
+ Still running: adopted and pumped live, its reply completed from the receipt (``recovered``).
+ Settled: adopted with its saved result, painted without a pump. One that never ran (cancelled,
+ rejected) is forgotten: the only point where ``queued_inputs`` drops a row unpainted.
+ """
+ from hermes_cli.gateway_client import GatewayClientError
+ state = store.connection.state(getattr(cli, "session_id", None))
+ if not state.queued_inputs or (state.recovered_seq == state.pending_seq and not state.missed):
+ return
+ state.recovered_seq = state.pending_seq
+ listed = {row.get("admission_id"): row.get("status") for row in state.pending or () if isinstance(row, dict)}
+ for admission, text in list(state.queued_inputs.items()):
+ if admission in state.adopted or (admission not in state.missed and listed.get(admission) == "queued"):
+ continue
+ try:
+ receipt = store.call("prompt.receipt", session_id=cli.session_id, admission_id=admission,
+ include_result=True) or {}
+ except GatewayClientError:
+ state.recovered_seq = -1 # unread: try again on the next tick
+ continue
+ status = receipt.get("status")
+ if status == "queued":
+ state.missed.discard(admission)
+ continue
+ if status == "terminal" and receipt.get("outcome") in _NEVER_RAN:
+ state.queued_inputs.pop(admission, None)
+ state.missed.discard(admission)
+ continue
+ if status not in ("started", "terminal"):
+ continue # unknown: the owner resolves it; nothing to paint yet
+ adopted = {"admission_id": admission, "execution_generation": receipt.get("execution_generation"),
+ "recovered": True}
+ if status == "terminal":
+ adopted["receipt"] = receipt
+ _adopt(cli, state, adopted, text)
+ return # one turn at a time: the next one is recovered after this one ran
+
+
+def missed_start(state, params) -> None:
+ """A frame dropped outside its turn: a ``/queue`` admission's ``message.start`` is remembered so
+ the idle tick paints that turn from its receipt instead of losing it."""
+ if params.get("type") == "message.start" and params.get("admission_id") in state.queued_inputs:
+ state.missed.add(params["admission_id"])
+
+
def _put_first(pending, item):
"""Ahead of anything already queued: the gateway is running this turn NOW (typed inputs the
post-turn path re-queued are admitted behind it), and a typed turn run first would drop it."""
@@ -114,6 +191,7 @@ def take_admitted_turn(agent, text):
armed, state.armed_admission = getattr(state, "armed_admission", None), None
adopted = state.adopted.pop(armed, None) if armed else None
if adopted is not None:
+ state.queued_inputs.pop(armed, None) # painted now: the admission's turn is this one
return adopted
adopted = getattr(state, "admitted_turn", None)
state.admitted_turn = None
diff --git a/tests/hermes_cli/test_cli_queue_gateway_fifo.py b/tests/hermes_cli/test_cli_queue_gateway_fifo.py
index 30afd51b32c..e8bd3265acf 100644
--- a/tests/hermes_cli/test_cli_queue_gateway_fifo.py
+++ b/tests/hermes_cli/test_cli_queue_gateway_fifo.py
@@ -165,13 +165,13 @@ def test_the_owner_claim_of_a_queued_prompt_runs_as_a_typed_turn_without_a_secon
cli._drain_process_notifications("cli-idle")
item = cli._pending_input.get_nowait()
assert str(item) == TEXT and type(item).__name__ == "_SeededQueryMessage" # literal, never a command
- assert conn.state(SID).queued_inputs == {}
cli._tui_unwrap_input(item) # the input loop takes it: its admission is armed by id
agent = GatewayAgent(connection=conn, session_id=SID)
agent._pending_cli_user_message = {"role": "user", "content": TEXT}
result = agent.run_conversation(TEXT, conversation_history=[])
assert "prompt.submit" not in [method for method, _ in client.calls]
assert result["final_response"] == "ran"
+ assert conn.state(SID).queued_inputs == {} # cleared once its turn is the one painted
finally:
conn.close()
@@ -227,3 +227,71 @@ def test_a_typed_turn_with_the_same_text_is_never_taken_for_the_adopted_admissio
assert "prompt.submit" in [m for m, _ in client.calls]
finally:
conn.close()
+
+
+def _run_adopted(cli, conn):
+ """The input loop takes the adopted item and chat() runs it: ``run_conversation`` result."""
+ from hermes_cli.gateway_agent import GatewayAgent
+ from tests.hermes_cli.test_gateway_agent import SID
+ item = cli._pending_input.get_nowait()
+ text, _, seeded = cli._tui_unwrap_input(item)
+ assert seeded and item.admission_id
+ agent = GatewayAgent(connection=conn, session_id=SID)
+ agent._pending_cli_user_message = {"role": "user", "content": text}
+ return agent.run_conversation(text, conversation_history=[])
+
+
+def test_a_local_turn_does_not_swallow_the_start_of_a_cli_queued_admission():
+ """Review N5-F2: the owner started the CLI's queued admission, but a local input (busy-mode Enter,
+ a /goal continuation) reached run_conversation before the idle tick adopted it. Its drain must not
+ lose that turn: the idle tick after the local turn paints it from its receipt (no second submit),
+ and only then is its ``queued_inputs`` entry cleared."""
+ from hermes_cli.gateway_agent import GatewayAgent
+ from tests.hermes_cli.test_gateway_agent import SID, frame
+ from tests.hermes_cli.test_gateway_tools_notify import _connected, _idle_cli, _play
+ conn, client = _connected()
+ try:
+ cli = _idle_cli(conn)
+ state = conn.state(SID)
+ state.queued_inputs["adm-q"] = TEXT
+ _play(conn, client, [frame("message.start", {}, admission="adm-q"),
+ frame("message.delta", {"text": "queued reply"}, admission="adm-q")])
+ client.script = [frame("message.complete", {"text": "local", "status": "complete"}, admission="adm-1")]
+ agent = GatewayAgent(connection=conn, session_id=SID)
+ agent._pending_cli_user_message = {"role": "user", "content": "local"}
+ agent.run_conversation("local", conversation_history=[])
+ assert "adm-q" in state.queued_inputs # not painted yet: still owed
+ submits = [m for m, _ in client.calls].count("prompt.submit")
+ client.result = {"final_response": "queued reply"}
+ cli._drain_process_notifications("cli-post-turn")
+ result = _run_adopted(cli, conn)
+ assert [m for m, _ in client.calls].count("prompt.submit") == submits # never submitted again
+ assert result["final_response"] == "queued reply"
+ assert state.queued_inputs == {}
+ finally:
+ conn.close()
+
+
+def test_a_queued_admission_another_client_cancelled_is_forgotten_not_painted():
+ """A remembered admission that settled without running (cancelled by Ink/Desktop) is dropped from
+ ``queued_inputs`` on the next published list; nothing is painted for it."""
+ from tests.hermes_cli.test_gateway_agent import SID
+ from tests.hermes_cli.test_gateway_tools_notify import _connected, _idle_cli
+ conn, client = _connected()
+ try:
+ cli = _idle_cli(conn)
+ state = conn.state(SID)
+ state.queued_inputs["adm-q"] = TEXT
+ real = client.rpc
+
+ async def rpc(method, **params):
+ if method == "prompt.receipt":
+ client.calls.append((method, params))
+ return {"admission_id": "adm-q", "status": "terminal", "outcome": "cancelled"}
+ return await real(method, **params)
+ client.rpc = rpc
+ state.apply_event("session.info", {"pending": []})
+ cli._drain_process_notifications("cli-idle")
+ assert cli._pending_input.empty() and state.queued_inputs == {}
+ finally:
+ conn.close()