The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): a cancelled older viewer close no longer erases a newer one
Tekniumdd9bfa9c815b
diff --git a/gateway/session_authority.py b/gateway/session_authority.pyindex 22914bc4fa2..bb644ebf92a 100644--- a/gateway/session_authority.py+++ b/gateway/session_authority.py@@ -276,9 +276,8 @@ class SessionAuthority:snapshot = SubscriptionSnapshot(subscription, handle, epoch,sequence, tuple(local_history(self, ref)),tuple(self._pending_receipt(r) for r in self._pending_rows(ref)), prompts)- from gateway.session_viewer_close import CLOSE_PREFIX, forget_persisted_close- if await asyncio.to_thread(self.db.get_meta, CLOSE_PREFIX + ref.session_id) is not None:- await asyncio.to_thread(forget_persisted_close, self, ref.session_id) # the viewer cancels it+ from gateway.session_viewer_close import cancel_persisted_close+ await asyncio.to_thread(cancel_persisted_close, self, ref.session_id) # the viewer cancels itfrom gateway.session_deferred_pick import reask_on_attachawait reask_on_attach(self, ref, live)return snapshotdiff --git a/gateway/session_viewer_close.py b/gateway/session_viewer_close.pyindex 51868b6c0ba..58edf8dba4a 100644--- a/gateway/session_viewer_close.py+++ b/gateway/session_viewer_close.py@@ -9,9 +9,14 @@ is remembered and completed when the FIFO goes idle, so exiting mid-turn still eThe pending close is also kept in ``state_meta`` (``CLOSE_PREFIX + session_id``): a gateway stop orrestart before the FIFO goes idle would otherwise drop it, and nothing else ends a local row. Bootrecovery re-arms it (``rearm_persisted_closes``), so the restarted owner's first idle point ends it.+Each stored close carries its own operation id, and every delete is compare-and-delete on that exact+record under the owner-epoch fence: a cancelled older close, a reattach or a stale owner can only+remove the record it saw, never a newer close stored since."""import asyncio+import jsonimport logging+import uuidfrom functools import partialfrom hermes_state_local import local_receipt@@ -44,11 +49,21 @@ _RESET_REASONS = frozenset({'new_session', 'tui_close'})class _PendingClose:- """One viewer close; identity, not the reason, tells a settler its close is still the pending one."""- __slots__ = ('reason',)+ """One viewer close; identity, not the reason, tells a settler its close is still the pending one.+ ``record`` is its ``state_meta`` value: a unique operation id plus the reason."""+ __slots__ = ('reason', 'record')- def __init__(self, reason):+ def __init__(self, reason, record=None):self.reason = reason+ self.record = record or json.dumps({'op': uuid.uuid4().hex, 'reason': reason}, sort_keys=True)++ @classmethod+ def from_record(cls, record):+ try:+ reason = json.loads(record)['reason']+ except (ValueError, TypeError, KeyError):+ reason = record # a bare-reason record from before operation ids+ return cls(reason, record)async def close_viewer_session(authority, session_id, reason):@@ -68,28 +83,52 @@ async def close_viewer_session(authority, session_id, reason):for parent in parents:interrupt_for_session(session_key=route, parent_session_id=parent, reason=f'viewer_close:{reason}')authority.pending_closes[session_id] = close = _PendingClose(reason)- await asyncio.to_thread(_persist_close, authority, session_id, reason)+ await asyncio.to_thread(_persist_close, authority, session_id, close.record)if not _still_pending(authority, session_id, live, close):- # A viewer attached during the write: its own clear may have run before this commit.- await asyncio.to_thread(forget_persisted_close, authority, session_id)+ # A viewer attached during the write: its own clear may have run before this commit. Only+ # this close's record goes; a newer close stored since keeps its own.+ await asyncio.to_thread(forget_persisted_close, authority, session_id, close.record)returnawait settle_viewer_close(authority, session_id)-def _persist_close(authority, session_id, reason):+def _persist_close(authority, session_id, record):from hermes_state_runtime import _epochdef write(conn):_epoch(conn, authority.epoch)conn.execute('INSERT INTO state_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',- (CLOSE_PREFIX + session_id, reason))+ (CLOSE_PREFIX + session_id, record))authority.db._execute_write(write)-def forget_persisted_close(authority, session_id):- """Blocking: drop *session_id*'s stored close (a viewer re-attached, or the close completed)."""- authority.db._execute_write(lambda conn: conn.execute('DELETE FROM state_meta WHERE key=?',- (CLOSE_PREFIX + session_id,)))+def forget_persisted_close(authority, session_id, record):+ """Blocking: drop *session_id*'s stored close only if it is still exactly *record* (that close+ completed or was cancelled), and only while this authority owns the store. Returns whether a+ row went."""+ from hermes_state_runtime import _epoch++ def write(conn):+ _epoch(conn, authority.epoch)+ return conn.execute('DELETE FROM state_meta WHERE key=? AND value=?',+ (CLOSE_PREFIX + session_id, record)).rowcount+ try:+ return bool(authority.db._execute_write(write))+ except RuntimeStoreError as exc:+ logger.info('Stored close for %s left in place: %s', session_id, exc)+ return False+++def cancel_persisted_close(authority, session_id):+ """Blocking, for a viewer attach: drop the stored close unless it is the one now pending in+ memory (a detach that ran after the attach stored it; it is that close's own to settle)."""+ record = authority.db.get_meta(CLOSE_PREFIX + session_id)+ if record is None:+ return+ pending = authority.pending_closes.get(session_id)+ if pending is not None and pending.record == record:+ return+ forget_persisted_close(authority, session_id, record)def rearm_persisted_closes(authority):@@ -99,15 +138,15 @@ def rearm_persisted_closes(authority):with authority.db._read_ctx() as conn:stored = conn.execute('SELECT key, value FROM state_meta WHERE key LIKE ?', (CLOSE_PREFIX + '%',)).fetchall()armed = []- for key, reason in stored:+ for key, record in stored:session_id = key[len(CLOSE_PREFIX):]live = authority.sessions.get(session_id)if live is None or live.superseded:continue # not restored (paused, retired): left stored for a later recoveryif live.subscribers:- forget_persisted_close(authority, session_id)+ forget_persisted_close(authority, session_id, record)continue- authority.pending_closes.setdefault(session_id, _PendingClose(reason))+ authority.pending_closes.setdefault(session_id, _PendingClose.from_record(record))armed.append(session_id)return armed@@ -139,7 +178,7 @@ async def settle_viewer_close(authority, session_id):if not ended:await _requeue_if_busy(authority, session_id, live, target, close)return- await asyncio.to_thread(forget_persisted_close, authority, session_id)+ await asyncio.to_thread(forget_persisted_close, authority, session_id, close.record)if live.subscribers:return # a viewer reattached during the stamp; reopen_if_reattached reopened the rowplatform = (row or {}).get('source') or (live.source.platform.value if live.source else '')@@ -155,7 +194,7 @@ async def _requeue_if_busy(authority, session_id, live, target, close):drain's next idle point; any other refusal (row already ended) leaves nothing to finish."""if not await asyncio.to_thread(_busy, authority, session_id, target):if session_id not in authority.pending_closes:- await asyncio.to_thread(forget_persisted_close, authority, session_id) # already ended+ await asyncio.to_thread(forget_persisted_close, authority, session_id, close.record) # already endedreturnif live.subscribers or session_id in authority.pending_closes:return # a reattach cancelled it, or a newer close replaced itdiff --git a/tests/gateway/test_viewer_close_lifecycle.py b/tests/gateway/test_viewer_close_lifecycle.pyindex 4746ff83717..7910bad6a10 100644--- a/tests/gateway/test_viewer_close_lifecycle.py+++ b/tests/gateway/test_viewer_close_lifecycle.py@@ -302,6 +302,18 @@ async def test_concurrent_settlers_run_the_close_lifecycle_once(tmp_path, monkeyassert db.get_session('chat')['end_reason'] == 'cli_close'+def _stored(db):+ import json+ from gateway.session_viewer_close import CLOSE_PREFIX+ record = db.get_meta(CLOSE_PREFIX + 'chat')+ if record is None:+ return None+ try:+ return json.loads(record)+ except ValueError:+ return {'reason': record} # a bare-reason record (before operation ids)++async def _restarted_owner(tmp_path, monkeypatch, db):"""The next gateway process on the same state.db: a new authority whose memory starts empty."""db2, authority, calls = await _owner(tmp_path, monkeypatch)@@ -344,7 +356,7 @@ async def test_a_viewer_reattaching_cancels_the_persisted_close(tmp_path, monkeyrow = admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',request_id='r1', payload={'text': 'mid-turn'})await _call(viewer, 'session.detach', session_id='chat', subscription_id=subscription, end_reason='cli_close')- assert db.get_meta(CLOSE_PREFIX + 'chat') == 'cli_close'+ assert _stored(db)['reason'] == 'cli_close'# Re-attach before settlement (same process): the stored close goes with the in-memory one.await _call(_viewer(authority, tmp_path), 'session.resume', session_id='chat')@@ -357,3 +369,120 @@ async def test_a_viewer_reattaching_cancels_the_persisted_close(tmp_path, monkeyassert restarted.sessions['chat'].task is None # no queued turn and no close: no drain at allassert restarted.pending_closes == {} and calls2 == []assert db2.get_session('chat')['ended_at'] is None+++def _gate_persist(monkeypatch, reason):+ """Hold *reason*'s close after its real SQLite commit, before its off-loop write returns."""+ import gateway.session_viewer_close as close+ original = close._persist_close+ entered, release = threading.Event(), threading.Event()++ def gated(authority, session_id, record):+ original(authority, session_id, record)+ if reason in record:+ entered.set()+ assert release.wait(10)+ monkeypatch.setattr(close, '_persist_close', gated)+ return entered, release+++async def test_a_cancelled_older_close_keeps_the_newer_close_durable(tmp_path, monkeypatch):+ """#134889 LC-6b, independent review round 2: close A commits, a viewer reattaches (cancelling A)+ and closes again as B; A's late continuation deleted B's stored close by key, so a restart+ before the idle point left the row open for good."""+ from gateway.session_local_recovery import recover_local_sessions+ from hermes_state_runtime import admit_session_input, cancel_session_input++ db, authority, _calls = await _owner(tmp_path, monkeypatch)+ viewer, subscription = await _open(db, authority, tmp_path)+ row = admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',+ request_id='r1', payload={'text': 'mid-turn'})+ entered, release = _gate_persist(monkeypatch, 'close_a')+ old = asyncio.create_task(_call(viewer, 'session.detach', session_id='chat',+ subscription_id=subscription, end_reason='close_a'))+ await async_wait_until(entered.is_set)+ newcomer = _viewer(authority, tmp_path)+ resumed = await _call(newcomer, 'session.resume', session_id='chat')+ await _call(newcomer, 'session.detach', session_id='chat', subscription_id=resumed['subscription_id'],+ end_reason='close_b')+ stored_b = _stored(db)+ assert stored_b['reason'] == 'close_b'+ release.set()+ await old++ # A's continuation sees it lost and removes only its own record: B's stays, op id and all.+ assert _stored(db) == stored_b+ assert authority.pending_closes['chat'].reason == 'close_b'++ # The old owner's memory is gone; the next authority recovers B and ends the row.+ cancel_session_input(db, epoch=authority.epoch, admission_id=row['admission_id'])+ db2, restarted, calls2 = await _restarted_owner(tmp_path, monkeypatch, db)+ recover_local_sessions(restarted, schedule=True)+ await async_wait_until(lambda: len(calls2) == 2, timeout=10)+ assert db2.get_session('chat')['end_reason'] == 'close_b'+ assert db2.get_session('chat')['ended_at'] is not None+ assert [name for name, _ in calls2] == ['commit', 'finalize'] and _stored(db2) is None+++async def test_the_newer_close_settles_and_the_older_one_leaves_no_trace(tmp_path, monkeypatch):+ db, authority, calls = await _owner(tmp_path, monkeypatch)+ viewer, subscription = await _open(db, authority, tmp_path)+ entered, release = _gate_persist(monkeypatch, 'close_a')+ old = asyncio.create_task(_call(viewer, 'session.detach', session_id='chat',+ subscription_id=subscription, end_reason='close_a'))+ await async_wait_until(entered.is_set)+ newcomer = _viewer(authority, tmp_path)+ resumed = await _call(newcomer, 'session.resume', session_id='chat')+ # Idle FIFO: B settles immediately, while A is still parked after its commit.+ await _call(newcomer, 'session.detach', session_id='chat', subscription_id=resumed['subscription_id'],+ end_reason='close_b')+ assert db.get_session('chat')['end_reason'] == 'close_b' and _stored(db) is None+ release.set()+ await old+ assert [name for name, _ in calls] == ['commit', 'finalize'] # once, for B only+ assert db.get_session('chat')['end_reason'] == 'close_b'+ assert authority.pending_closes == {} and _stored(db) is None+++async def test_a_reattach_during_the_close_write_cancels_it_durably(tmp_path, monkeypatch):+ from gateway.session_local_recovery import recover_local_sessions+ from hermes_state_runtime import admit_session_input, cancel_session_input++ db, authority, calls = await _owner(tmp_path, monkeypatch)+ viewer, subscription = await _open(db, authority, tmp_path)+ row = admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',+ request_id='r1', payload={'text': 'mid-turn'})+ entered, release = _gate_persist(monkeypatch, 'close_a')+ old = asyncio.create_task(_call(viewer, 'session.detach', session_id='chat',+ subscription_id=subscription, end_reason='close_a'))+ await async_wait_until(entered.is_set)+ await _call(_viewer(authority, tmp_path), 'session.resume', session_id='chat')+ release.set()+ await old+ assert authority.pending_closes == {} and _stored(db) is None and calls == []++ cancel_session_input(db, epoch=authority.epoch, admission_id=row['admission_id'])+ db2, restarted, calls2 = await _restarted_owner(tmp_path, monkeypatch, db)+ recover_local_sessions(restarted, schedule=True)+ assert restarted.pending_closes == {} and calls2 == []+ assert db2.get_session('chat')['ended_at'] is None+++async def test_a_stale_owner_cannot_delete_a_stored_close(tmp_path, monkeypatch):+ """Compare-and-delete is epoch-fenced: an owner that lost the store leaves its successor's close."""+ import json+ from gateway.session_viewer_close import CLOSE_PREFIX, forget_persisted_close++ db, authority, _calls = await _owner(tmp_path, monkeypatch)+ record = json.dumps({'op': 'x', 'reason': 'cli_close'})+ db.set_meta(CLOSE_PREFIX + 'chat', record)+ stale = SimpleNamespace(db=db, epoch=authority.epoch - 1)+ assert forget_persisted_close(stale, 'chat', record) is False+ assert forget_persisted_close(authority, 'chat', 'another close') is False+ assert db.get_meta(CLOSE_PREFIX + 'chat') == record+ assert forget_persisted_close(authority, 'chat', record) is True+ assert db.get_meta(CLOSE_PREFIX + 'chat') is None