Skip to content

The agent that grows with you

fix(gateway): a Ctrl+D close pending at gateway stop still ends the row

Tekniumca7b7175035b
diff --git a/gateway/session_authority.py b/gateway/session_authority.py
index f8ffafa8303..22914bc4fa2 100644
--- a/gateway/session_authority.py
+++ b/gateway/session_authority.py
@@ -276,6 +276,9 @@ class SessionAuthority:
snapshot = SubscriptionSnapshot(subscription, handle, epoch,
sequence, tuple(local_history(self, ref)),
tuple(self._pending_receipt(r) for r in self._pending_rows(ref)), prompts)
+ from gateway.session_viewer_close import CLOSE_PREFIX, forget_persisted_close
+ if await asyncio.to_thread(self.db.get_meta, CLOSE_PREFIX + ref.session_id) is not None:
+ await asyncio.to_thread(forget_persisted_close, self, ref.session_id) # the viewer cancels it
from gateway.session_deferred_pick import reask_on_attach
await reask_on_attach(self, ref, live)
return snapshot
diff --git a/gateway/session_local_recovery.py b/gateway/session_local_recovery.py
index e3c468a5875..222ab32a225 100644
--- a/gateway/session_local_recovery.py
+++ b/gateway/session_local_recovery.py
@@ -176,3 +176,8 @@ def recover_local_sessions(authority, *, schedule=False):
authority._schedule(ref)
except RuntimeStoreError as exc:
logging.getLogger(__name__).warning('Local session %s paused: %s', sid, exc.reason)
+ if schedule:
+ # A last-viewer close the previous owner stopped before completing (#134889 LC-6b).
+ from gateway.session_viewer_close import rearm_persisted_closes
+ for sid in rearm_persisted_closes(authority):
+ authority._schedule(SessionRef(authority.profile_id, sid))
diff --git a/gateway/session_viewer_close.py b/gateway/session_viewer_close.py
index f7b3666c9fb..51868b6c0ba 100644
--- a/gateway/session_viewer_close.py
+++ b/gateway/session_viewer_close.py
@@ -6,6 +6,9 @@ closes the Relay conversation scope). Under the gateway those clients are viewer
memory manager and the Relay scope live in the owner, so ``session.detach {end_reason}`` from the
last viewer is where the owner runs them. A close that lands while a turn is still queued or running
is remembered and completed when the FIFO goes idle, so exiting mid-turn still ends the row.
+The pending close is also kept in ``state_meta`` (``CLOSE_PREFIX + session_id``): a gateway stop or
+restart before the FIFO goes idle would otherwise drop it, and nothing else ends a local row. Boot
+recovery re-arms it (``rearm_persisted_closes``), so the restarted owner's first idle point ends it.
"""
import asyncio
import logging
@@ -16,6 +19,8 @@ from hermes_state_runtime import RuntimeStoreError
logger = logging.getLogger(__name__)
+CLOSE_PREFIX = 'gateway.viewer_close.v1:'
+
_BUSY_SQL = ("SELECT 1 FROM session_admissions WHERE target_session_id=? AND status IN ('queued','started','unknown') "
"UNION ALL SELECT 1 FROM worker_executions WHERE session_id IN (?,?) "
"AND status IN ('registered','running','unknown') LIMIT 1")
@@ -62,10 +67,51 @@ async def close_viewer_session(authority, session_id, reason):
refuse_for_session((authority.profile_id, session_id), session_key=route, parent_session_ids=parents)
for parent in parents:
interrupt_for_session(session_key=route, parent_session_id=parent, reason=f'viewer_close:{reason}')
- authority.pending_closes[session_id] = _PendingClose(reason)
+ authority.pending_closes[session_id] = close = _PendingClose(reason)
+ await asyncio.to_thread(_persist_close, authority, session_id, reason)
+ if not _still_pending(authority, session_id, live, close):
+ # A viewer attached during the write: its own clear may have run before this commit.
+ await asyncio.to_thread(forget_persisted_close, authority, session_id)
+ return
await settle_viewer_close(authority, session_id)
+def _persist_close(authority, session_id, reason):
+ from hermes_state_runtime import _epoch
+
+ def write(conn):
+ _epoch(conn, authority.epoch)
+ conn.execute('INSERT INTO state_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',
+ (CLOSE_PREFIX + session_id, reason))
+ authority.db._execute_write(write)
+
+
+def forget_persisted_close(authority, session_id):
+ """Blocking: drop *session_id*'s stored close (a viewer re-attached, or the close completed)."""
+ authority.db._execute_write(lambda conn: conn.execute('DELETE FROM state_meta WHERE key=?',
+ (CLOSE_PREFIX + session_id,)))
+
+
+def rearm_persisted_closes(authority):
+ """Boot/serve recovery, after the local sessions are restored: every stored close whose session
+ is mounted with no viewer goes back to ``pending_closes``. The caller schedules the drain, whose
+ idle point (after any queued turn; never past an ``unknown`` one) completes it. Returns the ids."""
+ with authority.db._read_ctx() as conn:
+ stored = conn.execute('SELECT key, value FROM state_meta WHERE key LIKE ?', (CLOSE_PREFIX + '%',)).fetchall()
+ armed = []
+ for key, reason in stored:
+ session_id = key[len(CLOSE_PREFIX):]
+ live = authority.sessions.get(session_id)
+ if live is None or live.superseded:
+ continue # not restored (paused, retired): left stored for a later recovery
+ if live.subscribers:
+ forget_persisted_close(authority, session_id)
+ continue
+ authority.pending_closes.setdefault(session_id, _PendingClose(reason))
+ armed.append(session_id)
+ return armed
+
+
async def settle_viewer_close(authority, session_id):
"""Complete a pending viewer close once *session_id* has no viewer and an idle FIFO: stamp the
row with the viewer's reason, commit the cached agent's memory and fire ``on_session_finalize``.
@@ -93,6 +139,7 @@ async def settle_viewer_close(authority, session_id):
if not ended:
await _requeue_if_busy(authority, session_id, live, target, close)
return
+ await asyncio.to_thread(forget_persisted_close, authority, session_id)
if live.subscribers:
return # a viewer reattached during the stamp; reopen_if_reattached reopened the row
platform = (row or {}).get('source') or (live.source.platform.value if live.source else '')
@@ -107,6 +154,8 @@ async def _requeue_if_busy(authority, session_id, live, target, close):
"""The stamp was refused. If input was admitted after the idle check, the close goes back to the
drain's next idle point; any other refusal (row already ended) leaves nothing to finish."""
if not await asyncio.to_thread(_busy, authority, session_id, target):
+ if session_id not in authority.pending_closes:
+ await asyncio.to_thread(forget_persisted_close, authority, session_id) # already ended
return
if live.subscribers or session_id in authority.pending_closes:
return # a reattach cancelled it, or a newer close replaced it
diff --git a/hermes_state_mutation_retirement.py b/hermes_state_mutation_retirement.py
index 17a73ac45a0..8558b9208b2 100644
--- a/hermes_state_mutation_retirement.py
+++ b/hermes_state_mutation_retirement.py
@@ -200,6 +200,7 @@ def retire_routes(conn, session_ids):
conn.execute('INSERT OR REPLACE INTO state_meta(key,value) VALUES(?,?)',
(RETIRED_PREFIX + sid, json.dumps({'retired_at': time.time()})))
conn.execute('DELETE FROM state_meta WHERE key=?', ('gateway.api.settings.v1.' + sid,))
+ conn.execute('DELETE FROM state_meta WHERE key=?', ('gateway.viewer_close.v1:' + sid,))
targets = set(session_ids)
for row in conn.execute('SELECT scope,session_key,entry_json FROM gateway_routing').fetchall():
if entry_session_id(row['entry_json']) in targets:
diff --git a/tests/gateway/test_viewer_close_lifecycle.py b/tests/gateway/test_viewer_close_lifecycle.py
index 9e7e30408d7..4746ff83717 100644
--- a/tests/gateway/test_viewer_close_lifecycle.py
+++ b/tests/gateway/test_viewer_close_lifecycle.py
@@ -300,3 +300,60 @@ async def test_concurrent_settlers_run_the_close_lifecycle_once(tmp_path, monkey
assert [name for name, _ in calls] == ['commit', 'finalize']
assert db.get_session('chat')['end_reason'] == 'cli_close'
+
+
+async def _restarted_owner(tmp_path, monkeypatch, db):
+ """The next gateway process on the same state.db: a new authority whose memory starts empty."""
+ db2, authority, calls = await _owner(tmp_path, monkeypatch)
+ assert authority.pending_closes == {}
+ return db2, authority, calls
+
+
+async def test_a_close_pending_at_gateway_stop_completes_after_restart(tmp_path, monkeypatch):
+ """#134889 LC-6b, independent review P2-4: Ctrl+D mid-turn, then the gateway stops before the
+ turn's idle point. The deferred close lived only in memory, so the row stayed open forever."""
+ from gateway.session_local_recovery import recover_local_sessions
+ from hermes_state_runtime import admit_session_input, cancel_session_input
+
+ db, authority, _calls = await _owner(tmp_path, monkeypatch)
+ viewer, subscription = await _open(db, authority, tmp_path)
+ row = admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',
+ request_id='r1', payload={'text': 'mid-turn'})
+ await _call(viewer, 'session.detach', session_id='chat', subscription_id=subscription, end_reason='cli_close')
+ assert db.get_session('chat')['ended_at'] is None and 'chat' in authority.pending_closes
+ # The shutdown settles the turn (interrupted) without reaching the drain's idle point.
+ cancel_session_input(db, epoch=authority.epoch, admission_id=row['admission_id'])
+
+ db2, restarted, calls2 = await _restarted_owner(tmp_path, monkeypatch, db)
+ recover_local_sessions(restarted, schedule=True)
+ await async_wait_until(lambda: len(calls2) == 2, timeout=10)
+ assert db2.get_session('chat')['end_reason'] == 'cli_close'
+ assert [name for name, _ in calls2] == ['commit', 'finalize']
+ assert db2.get_meta('gateway.viewer_close.v1:chat') is None # settled: nothing left to re-arm
+
+
+async def test_a_viewer_reattaching_cancels_the_persisted_close(tmp_path, monkeypatch):
+ from gateway.session_local_recovery import recover_local_sessions
+ from gateway.session_viewer_close import CLOSE_PREFIX
+ from hermes_state_runtime import admit_session_input, cancel_session_input
+
+ db, authority, _calls = await _owner(tmp_path, monkeypatch)
+ viewer, subscription = await _open(db, authority, tmp_path)
+ row = admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',
+ request_id='r1', payload={'text': 'mid-turn'})
+ await _call(viewer, 'session.detach', session_id='chat', subscription_id=subscription, end_reason='cli_close')
+ assert db.get_meta(CLOSE_PREFIX + 'chat') == 'cli_close'
+
+ # Re-attach before settlement (same process): the stored close goes with the in-memory one.
+ await _call(_viewer(authority, tmp_path), 'session.resume', session_id='chat')
+ assert authority.pending_closes == {} and db.get_meta(CLOSE_PREFIX + 'chat') is None
+
+ # And after a restart (the turn settled meanwhile) nothing re-arms it.
+ cancel_session_input(db, epoch=authority.epoch, admission_id=row['admission_id'])
+ db2, restarted, calls2 = await _restarted_owner(tmp_path, monkeypatch, db)
+ recover_local_sessions(restarted, schedule=True)
+ assert restarted.sessions['chat'].task is None # no queued turn and no close: no drain at all
+ assert restarted.pending_closes == {} and calls2 == []
+ assert db2.get_session('chat')['ended_at'] is None