The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): a Ctrl+D close pending at gateway stop still ends the row
Tekniumca7b7175035b
diff --git a/gateway/session_authority.py b/gateway/session_authority.pyindex f8ffafa8303..22914bc4fa2 100644--- a/gateway/session_authority.py+++ b/gateway/session_authority.py@@ -276,6 +276,9 @@ class SessionAuthority:snapshot = SubscriptionSnapshot(subscription, handle, epoch,sequence, tuple(local_history(self, ref)),tuple(self._pending_receipt(r) for r in self._pending_rows(ref)), prompts)+ from gateway.session_viewer_close import CLOSE_PREFIX, forget_persisted_close+ if await asyncio.to_thread(self.db.get_meta, CLOSE_PREFIX + ref.session_id) is not None:+ await asyncio.to_thread(forget_persisted_close, self, ref.session_id) # the viewer cancels itfrom gateway.session_deferred_pick import reask_on_attachawait reask_on_attach(self, ref, live)return snapshotdiff --git a/gateway/session_local_recovery.py b/gateway/session_local_recovery.pyindex e3c468a5875..222ab32a225 100644--- a/gateway/session_local_recovery.py+++ b/gateway/session_local_recovery.py@@ -176,3 +176,8 @@ def recover_local_sessions(authority, *, schedule=False):authority._schedule(ref)except RuntimeStoreError as exc:logging.getLogger(__name__).warning('Local session %s paused: %s', sid, exc.reason)+ if schedule:+ # A last-viewer close the previous owner stopped before completing (#134889 LC-6b).+ from gateway.session_viewer_close import rearm_persisted_closes+ for sid in rearm_persisted_closes(authority):+ authority._schedule(SessionRef(authority.profile_id, sid))diff --git a/gateway/session_viewer_close.py b/gateway/session_viewer_close.pyindex f7b3666c9fb..51868b6c0ba 100644--- a/gateway/session_viewer_close.py+++ b/gateway/session_viewer_close.py@@ -6,6 +6,9 @@ closes the Relay conversation scope). Under the gateway those clients are viewermemory manager and the Relay scope live in the owner, so ``session.detach {end_reason}`` from thelast viewer is where the owner runs them. A close that lands while a turn is still queued or runningis remembered and completed when the FIFO goes idle, so exiting mid-turn still ends the row.+The pending close is also kept in ``state_meta`` (``CLOSE_PREFIX + session_id``): a gateway stop or+restart before the FIFO goes idle would otherwise drop it, and nothing else ends a local row. Boot+recovery re-arms it (``rearm_persisted_closes``), so the restarted owner's first idle point ends it."""import asyncioimport logging@@ -16,6 +19,8 @@ from hermes_state_runtime import RuntimeStoreErrorlogger = logging.getLogger(__name__)+CLOSE_PREFIX = 'gateway.viewer_close.v1:'+_BUSY_SQL = ("SELECT 1 FROM session_admissions WHERE target_session_id=? AND status IN ('queued','started','unknown') ""UNION ALL SELECT 1 FROM worker_executions WHERE session_id IN (?,?) ""AND status IN ('registered','running','unknown') LIMIT 1")@@ -62,10 +67,51 @@ async def close_viewer_session(authority, session_id, reason):refuse_for_session((authority.profile_id, session_id), session_key=route, parent_session_ids=parents)for parent in parents:interrupt_for_session(session_key=route, parent_session_id=parent, reason=f'viewer_close:{reason}')- authority.pending_closes[session_id] = _PendingClose(reason)+ authority.pending_closes[session_id] = close = _PendingClose(reason)+ await asyncio.to_thread(_persist_close, authority, session_id, reason)+ if not _still_pending(authority, session_id, live, close):+ # A viewer attached during the write: its own clear may have run before this commit.+ await asyncio.to_thread(forget_persisted_close, authority, session_id)+ returnawait settle_viewer_close(authority, session_id)+def _persist_close(authority, session_id, reason):+ from hermes_state_runtime import _epoch++ def write(conn):+ _epoch(conn, authority.epoch)+ conn.execute('INSERT INTO state_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',+ (CLOSE_PREFIX + session_id, reason))+ authority.db._execute_write(write)+++def forget_persisted_close(authority, session_id):+ """Blocking: drop *session_id*'s stored close (a viewer re-attached, or the close completed)."""+ authority.db._execute_write(lambda conn: conn.execute('DELETE FROM state_meta WHERE key=?',+ (CLOSE_PREFIX + session_id,)))+++def rearm_persisted_closes(authority):+ """Boot/serve recovery, after the local sessions are restored: every stored close whose session+ is mounted with no viewer goes back to ``pending_closes``. The caller schedules the drain, whose+ idle point (after any queued turn; never past an ``unknown`` one) completes it. Returns the ids."""+ with authority.db._read_ctx() as conn:+ stored = conn.execute('SELECT key, value FROM state_meta WHERE key LIKE ?', (CLOSE_PREFIX + '%',)).fetchall()+ armed = []+ for key, reason in stored:+ session_id = key[len(CLOSE_PREFIX):]+ live = authority.sessions.get(session_id)+ if live is None or live.superseded:+ continue # not restored (paused, retired): left stored for a later recovery+ if live.subscribers:+ forget_persisted_close(authority, session_id)+ continue+ authority.pending_closes.setdefault(session_id, _PendingClose(reason))+ armed.append(session_id)+ return armed++async def settle_viewer_close(authority, session_id):"""Complete a pending viewer close once *session_id* has no viewer and an idle FIFO: stamp therow with the viewer's reason, commit the cached agent's memory and fire ``on_session_finalize``.@@ -93,6 +139,7 @@ async def settle_viewer_close(authority, session_id):if not ended:await _requeue_if_busy(authority, session_id, live, target, close)return+ await asyncio.to_thread(forget_persisted_close, authority, session_id)if live.subscribers:return # a viewer reattached during the stamp; reopen_if_reattached reopened the rowplatform = (row or {}).get('source') or (live.source.platform.value if live.source else '')@@ -107,6 +154,8 @@ async def _requeue_if_busy(authority, session_id, live, target, close):"""The stamp was refused. If input was admitted after the idle check, the close goes back to thedrain's next idle point; any other refusal (row already ended) leaves nothing to finish."""if not await asyncio.to_thread(_busy, authority, session_id, target):+ if session_id not in authority.pending_closes:+ await asyncio.to_thread(forget_persisted_close, authority, session_id) # already endedreturnif live.subscribers or session_id in authority.pending_closes:return # a reattach cancelled it, or a newer close replaced itdiff --git a/hermes_state_mutation_retirement.py b/hermes_state_mutation_retirement.pyindex 17a73ac45a0..8558b9208b2 100644--- a/hermes_state_mutation_retirement.py+++ b/hermes_state_mutation_retirement.py@@ -200,6 +200,7 @@ def retire_routes(conn, session_ids):conn.execute('INSERT OR REPLACE INTO state_meta(key,value) VALUES(?,?)',(RETIRED_PREFIX + sid, json.dumps({'retired_at': time.time()})))conn.execute('DELETE FROM state_meta WHERE key=?', ('gateway.api.settings.v1.' + sid,))+ conn.execute('DELETE FROM state_meta WHERE key=?', ('gateway.viewer_close.v1:' + sid,))targets = set(session_ids)for row in conn.execute('SELECT scope,session_key,entry_json FROM gateway_routing').fetchall():if entry_session_id(row['entry_json']) in targets:diff --git a/tests/gateway/test_viewer_close_lifecycle.py b/tests/gateway/test_viewer_close_lifecycle.pyindex 9e7e30408d7..4746ff83717 100644--- a/tests/gateway/test_viewer_close_lifecycle.py+++ b/tests/gateway/test_viewer_close_lifecycle.py@@ -300,3 +300,60 @@ async def test_concurrent_settlers_run_the_close_lifecycle_once(tmp_path, monkeyassert [name for name, _ in calls] == ['commit', 'finalize']assert db.get_session('chat')['end_reason'] == 'cli_close'+++async def _restarted_owner(tmp_path, monkeypatch, db):+ """The next gateway process on the same state.db: a new authority whose memory starts empty."""+ db2, authority, calls = await _owner(tmp_path, monkeypatch)+ assert authority.pending_closes == {}+ return db2, authority, calls+++async def test_a_close_pending_at_gateway_stop_completes_after_restart(tmp_path, monkeypatch):+ """#134889 LC-6b, independent review P2-4: Ctrl+D mid-turn, then the gateway stops before the+ turn's idle point. The deferred close lived only in memory, so the row stayed open forever."""+ from gateway.session_local_recovery import recover_local_sessions+ from hermes_state_runtime import admit_session_input, cancel_session_input++ db, authority, _calls = await _owner(tmp_path, monkeypatch)+ viewer, subscription = await _open(db, authority, tmp_path)+ row = admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',+ request_id='r1', payload={'text': 'mid-turn'})+ await _call(viewer, 'session.detach', session_id='chat', subscription_id=subscription, end_reason='cli_close')+ assert db.get_session('chat')['ended_at'] is None and 'chat' in authority.pending_closes+ # The shutdown settles the turn (interrupted) without reaching the drain's idle point.+ cancel_session_input(db, epoch=authority.epoch, admission_id=row['admission_id'])++ db2, restarted, calls2 = await _restarted_owner(tmp_path, monkeypatch, db)+ recover_local_sessions(restarted, schedule=True)+ await async_wait_until(lambda: len(calls2) == 2, timeout=10)+ assert db2.get_session('chat')['end_reason'] == 'cli_close'+ assert [name for name, _ in calls2] == ['commit', 'finalize']+ assert db2.get_meta('gateway.viewer_close.v1:chat') is None # settled: nothing left to re-arm+++async def test_a_viewer_reattaching_cancels_the_persisted_close(tmp_path, monkeypatch):+ from gateway.session_local_recovery import recover_local_sessions+ from gateway.session_viewer_close import CLOSE_PREFIX+ from hermes_state_runtime import admit_session_input, cancel_session_input++ db, authority, _calls = await _owner(tmp_path, monkeypatch)+ viewer, subscription = await _open(db, authority, tmp_path)+ row = admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',+ request_id='r1', payload={'text': 'mid-turn'})+ await _call(viewer, 'session.detach', session_id='chat', subscription_id=subscription, end_reason='cli_close')+ assert db.get_meta(CLOSE_PREFIX + 'chat') == 'cli_close'++ # Re-attach before settlement (same process): the stored close goes with the in-memory one.+ await _call(_viewer(authority, tmp_path), 'session.resume', session_id='chat')+ assert authority.pending_closes == {} and db.get_meta(CLOSE_PREFIX + 'chat') is None++ # And after a restart (the turn settled meanwhile) nothing re-arms it.+ cancel_session_input(db, epoch=authority.epoch, admission_id=row['admission_id'])+ db2, restarted, calls2 = await _restarted_owner(tmp_path, monkeypatch, db)+ recover_local_sessions(restarted, schedule=True)+ assert restarted.sessions['chat'].task is None # no queued turn and no close: no drain at all+ assert restarted.pending_closes == {} and calls2 == []+ assert db2.get_session('chat')['ended_at'] is None