Skip to content

The agent that grows with you

fix(gateway): a cancelled older viewer close no longer erases a newer one

Tekniumdd9bfa9c815b
diff --git a/gateway/session_authority.py b/gateway/session_authority.py
index 22914bc4fa2..bb644ebf92a 100644
--- a/gateway/session_authority.py
+++ b/gateway/session_authority.py
@@ -276,9 +276,8 @@ class SessionAuthority:
snapshot = SubscriptionSnapshot(subscription, handle, epoch,
sequence, tuple(local_history(self, ref)),
tuple(self._pending_receipt(r) for r in self._pending_rows(ref)), prompts)
- from gateway.session_viewer_close import CLOSE_PREFIX, forget_persisted_close
- if await asyncio.to_thread(self.db.get_meta, CLOSE_PREFIX + ref.session_id) is not None:
- await asyncio.to_thread(forget_persisted_close, self, ref.session_id) # the viewer cancels it
+ from gateway.session_viewer_close import cancel_persisted_close
+ await asyncio.to_thread(cancel_persisted_close, self, ref.session_id) # the viewer cancels it
from gateway.session_deferred_pick import reask_on_attach
await reask_on_attach(self, ref, live)
return snapshot
diff --git a/gateway/session_viewer_close.py b/gateway/session_viewer_close.py
index 51868b6c0ba..58edf8dba4a 100644
--- a/gateway/session_viewer_close.py
+++ b/gateway/session_viewer_close.py
@@ -9,9 +9,14 @@ is remembered and completed when the FIFO goes idle, so exiting mid-turn still e
The pending close is also kept in ``state_meta`` (``CLOSE_PREFIX + session_id``): a gateway stop or
restart before the FIFO goes idle would otherwise drop it, and nothing else ends a local row. Boot
recovery re-arms it (``rearm_persisted_closes``), so the restarted owner's first idle point ends it.
+Each stored close carries its own operation id, and every delete is compare-and-delete on that exact
+record under the owner-epoch fence: a cancelled older close, a reattach or a stale owner can only
+remove the record it saw, never a newer close stored since.
"""
import asyncio
+import json
import logging
+import uuid
from functools import partial
from hermes_state_local import local_receipt
@@ -44,11 +49,21 @@ _RESET_REASONS = frozenset({'new_session', 'tui_close'})
class _PendingClose:
- """One viewer close; identity, not the reason, tells a settler its close is still the pending one."""
- __slots__ = ('reason',)
+ """One viewer close; identity, not the reason, tells a settler its close is still the pending one.
+ ``record`` is its ``state_meta`` value: a unique operation id plus the reason."""
+ __slots__ = ('reason', 'record')
- def __init__(self, reason):
+ def __init__(self, reason, record=None):
self.reason = reason
+ self.record = record or json.dumps({'op': uuid.uuid4().hex, 'reason': reason}, sort_keys=True)
+
+ @classmethod
+ def from_record(cls, record):
+ try:
+ reason = json.loads(record)['reason']
+ except (ValueError, TypeError, KeyError):
+ reason = record # a bare-reason record from before operation ids
+ return cls(reason, record)
async def close_viewer_session(authority, session_id, reason):
@@ -68,28 +83,52 @@ async def close_viewer_session(authority, session_id, reason):
for parent in parents:
interrupt_for_session(session_key=route, parent_session_id=parent, reason=f'viewer_close:{reason}')
authority.pending_closes[session_id] = close = _PendingClose(reason)
- await asyncio.to_thread(_persist_close, authority, session_id, reason)
+ await asyncio.to_thread(_persist_close, authority, session_id, close.record)
if not _still_pending(authority, session_id, live, close):
- # A viewer attached during the write: its own clear may have run before this commit.
- await asyncio.to_thread(forget_persisted_close, authority, session_id)
+ # A viewer attached during the write: its own clear may have run before this commit. Only
+ # this close's record goes; a newer close stored since keeps its own.
+ await asyncio.to_thread(forget_persisted_close, authority, session_id, close.record)
return
await settle_viewer_close(authority, session_id)
-def _persist_close(authority, session_id, reason):
+def _persist_close(authority, session_id, record):
from hermes_state_runtime import _epoch
def write(conn):
_epoch(conn, authority.epoch)
conn.execute('INSERT INTO state_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',
- (CLOSE_PREFIX + session_id, reason))
+ (CLOSE_PREFIX + session_id, record))
authority.db._execute_write(write)
-def forget_persisted_close(authority, session_id):
- """Blocking: drop *session_id*'s stored close (a viewer re-attached, or the close completed)."""
- authority.db._execute_write(lambda conn: conn.execute('DELETE FROM state_meta WHERE key=?',
- (CLOSE_PREFIX + session_id,)))
+def forget_persisted_close(authority, session_id, record):
+ """Blocking: drop *session_id*'s stored close only if it is still exactly *record* (that close
+ completed or was cancelled), and only while this authority owns the store. Returns whether a
+ row went."""
+ from hermes_state_runtime import _epoch
+
+ def write(conn):
+ _epoch(conn, authority.epoch)
+ return conn.execute('DELETE FROM state_meta WHERE key=? AND value=?',
+ (CLOSE_PREFIX + session_id, record)).rowcount
+ try:
+ return bool(authority.db._execute_write(write))
+ except RuntimeStoreError as exc:
+ logger.info('Stored close for %s left in place: %s', session_id, exc)
+ return False
+
+
+def cancel_persisted_close(authority, session_id):
+ """Blocking, for a viewer attach: drop the stored close unless it is the one now pending in
+ memory (a detach that ran after the attach stored it; it is that close's own to settle)."""
+ record = authority.db.get_meta(CLOSE_PREFIX + session_id)
+ if record is None:
+ return
+ pending = authority.pending_closes.get(session_id)
+ if pending is not None and pending.record == record:
+ return
+ forget_persisted_close(authority, session_id, record)
def rearm_persisted_closes(authority):
@@ -99,15 +138,15 @@ def rearm_persisted_closes(authority):
with authority.db._read_ctx() as conn:
stored = conn.execute('SELECT key, value FROM state_meta WHERE key LIKE ?', (CLOSE_PREFIX + '%',)).fetchall()
armed = []
- for key, reason in stored:
+ for key, record in stored:
session_id = key[len(CLOSE_PREFIX):]
live = authority.sessions.get(session_id)
if live is None or live.superseded:
continue # not restored (paused, retired): left stored for a later recovery
if live.subscribers:
- forget_persisted_close(authority, session_id)
+ forget_persisted_close(authority, session_id, record)
continue
- authority.pending_closes.setdefault(session_id, _PendingClose(reason))
+ authority.pending_closes.setdefault(session_id, _PendingClose.from_record(record))
armed.append(session_id)
return armed
@@ -139,7 +178,7 @@ async def settle_viewer_close(authority, session_id):
if not ended:
await _requeue_if_busy(authority, session_id, live, target, close)
return
- await asyncio.to_thread(forget_persisted_close, authority, session_id)
+ await asyncio.to_thread(forget_persisted_close, authority, session_id, close.record)
if live.subscribers:
return # a viewer reattached during the stamp; reopen_if_reattached reopened the row
platform = (row or {}).get('source') or (live.source.platform.value if live.source else '')
@@ -155,7 +194,7 @@ async def _requeue_if_busy(authority, session_id, live, target, close):
drain's next idle point; any other refusal (row already ended) leaves nothing to finish."""
if not await asyncio.to_thread(_busy, authority, session_id, target):
if session_id not in authority.pending_closes:
- await asyncio.to_thread(forget_persisted_close, authority, session_id) # already ended
+ await asyncio.to_thread(forget_persisted_close, authority, session_id, close.record) # already ended
return
if live.subscribers or session_id in authority.pending_closes:
return # a reattach cancelled it, or a newer close replaced it
diff --git a/tests/gateway/test_viewer_close_lifecycle.py b/tests/gateway/test_viewer_close_lifecycle.py
index 4746ff83717..7910bad6a10 100644
--- a/tests/gateway/test_viewer_close_lifecycle.py
+++ b/tests/gateway/test_viewer_close_lifecycle.py
@@ -302,6 +302,18 @@ async def test_concurrent_settlers_run_the_close_lifecycle_once(tmp_path, monkey
assert db.get_session('chat')['end_reason'] == 'cli_close'
+def _stored(db):
+ import json
+ from gateway.session_viewer_close import CLOSE_PREFIX
+ record = db.get_meta(CLOSE_PREFIX + 'chat')
+ if record is None:
+ return None
+ try:
+ return json.loads(record)
+ except ValueError:
+ return {'reason': record} # a bare-reason record (before operation ids)
+
+
async def _restarted_owner(tmp_path, monkeypatch, db):
"""The next gateway process on the same state.db: a new authority whose memory starts empty."""
db2, authority, calls = await _owner(tmp_path, monkeypatch)
@@ -344,7 +356,7 @@ async def test_a_viewer_reattaching_cancels_the_persisted_close(tmp_path, monkey
row = admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',
request_id='r1', payload={'text': 'mid-turn'})
await _call(viewer, 'session.detach', session_id='chat', subscription_id=subscription, end_reason='cli_close')
- assert db.get_meta(CLOSE_PREFIX + 'chat') == 'cli_close'
+ assert _stored(db)['reason'] == 'cli_close'
# Re-attach before settlement (same process): the stored close goes with the in-memory one.
await _call(_viewer(authority, tmp_path), 'session.resume', session_id='chat')
@@ -357,3 +369,120 @@ async def test_a_viewer_reattaching_cancels_the_persisted_close(tmp_path, monkey
assert restarted.sessions['chat'].task is None # no queued turn and no close: no drain at all
assert restarted.pending_closes == {} and calls2 == []
assert db2.get_session('chat')['ended_at'] is None
+
+
+def _gate_persist(monkeypatch, reason):
+ """Hold *reason*'s close after its real SQLite commit, before its off-loop write returns."""
+ import gateway.session_viewer_close as close
+ original = close._persist_close
+ entered, release = threading.Event(), threading.Event()
+
+ def gated(authority, session_id, record):
+ original(authority, session_id, record)
+ if reason in record:
+ entered.set()
+ assert release.wait(10)
+ monkeypatch.setattr(close, '_persist_close', gated)
+ return entered, release
+
+
+async def test_a_cancelled_older_close_keeps_the_newer_close_durable(tmp_path, monkeypatch):
+ """#134889 LC-6b, independent review round 2: close A commits, a viewer reattaches (cancelling A)
+ and closes again as B; A's late continuation deleted B's stored close by key, so a restart
+ before the idle point left the row open for good."""
+ from gateway.session_local_recovery import recover_local_sessions
+ from hermes_state_runtime import admit_session_input, cancel_session_input
+
+ db, authority, _calls = await _owner(tmp_path, monkeypatch)
+ viewer, subscription = await _open(db, authority, tmp_path)
+ row = admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',
+ request_id='r1', payload={'text': 'mid-turn'})
+ entered, release = _gate_persist(monkeypatch, 'close_a')
+ old = asyncio.create_task(_call(viewer, 'session.detach', session_id='chat',
+ subscription_id=subscription, end_reason='close_a'))
+ await async_wait_until(entered.is_set)
+ newcomer = _viewer(authority, tmp_path)
+ resumed = await _call(newcomer, 'session.resume', session_id='chat')
+ await _call(newcomer, 'session.detach', session_id='chat', subscription_id=resumed['subscription_id'],
+ end_reason='close_b')
+ stored_b = _stored(db)
+ assert stored_b['reason'] == 'close_b'
+ release.set()
+ await old
+
+ # A's continuation sees it lost and removes only its own record: B's stays, op id and all.
+ assert _stored(db) == stored_b
+ assert authority.pending_closes['chat'].reason == 'close_b'
+
+ # The old owner's memory is gone; the next authority recovers B and ends the row.
+ cancel_session_input(db, epoch=authority.epoch, admission_id=row['admission_id'])
+ db2, restarted, calls2 = await _restarted_owner(tmp_path, monkeypatch, db)
+ recover_local_sessions(restarted, schedule=True)
+ await async_wait_until(lambda: len(calls2) == 2, timeout=10)
+ assert db2.get_session('chat')['end_reason'] == 'close_b'
+ assert db2.get_session('chat')['ended_at'] is not None
+ assert [name for name, _ in calls2] == ['commit', 'finalize'] and _stored(db2) is None
+
+
+async def test_the_newer_close_settles_and_the_older_one_leaves_no_trace(tmp_path, monkeypatch):
+ db, authority, calls = await _owner(tmp_path, monkeypatch)
+ viewer, subscription = await _open(db, authority, tmp_path)
+ entered, release = _gate_persist(monkeypatch, 'close_a')
+ old = asyncio.create_task(_call(viewer, 'session.detach', session_id='chat',
+ subscription_id=subscription, end_reason='close_a'))
+ await async_wait_until(entered.is_set)
+ newcomer = _viewer(authority, tmp_path)
+ resumed = await _call(newcomer, 'session.resume', session_id='chat')
+ # Idle FIFO: B settles immediately, while A is still parked after its commit.
+ await _call(newcomer, 'session.detach', session_id='chat', subscription_id=resumed['subscription_id'],
+ end_reason='close_b')
+ assert db.get_session('chat')['end_reason'] == 'close_b' and _stored(db) is None
+ release.set()
+ await old
+ assert [name for name, _ in calls] == ['commit', 'finalize'] # once, for B only
+ assert db.get_session('chat')['end_reason'] == 'close_b'
+ assert authority.pending_closes == {} and _stored(db) is None
+
+
+async def test_a_reattach_during_the_close_write_cancels_it_durably(tmp_path, monkeypatch):
+ from gateway.session_local_recovery import recover_local_sessions
+ from hermes_state_runtime import admit_session_input, cancel_session_input
+
+ db, authority, calls = await _owner(tmp_path, monkeypatch)
+ viewer, subscription = await _open(db, authority, tmp_path)
+ row = admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',
+ request_id='r1', payload={'text': 'mid-turn'})
+ entered, release = _gate_persist(monkeypatch, 'close_a')
+ old = asyncio.create_task(_call(viewer, 'session.detach', session_id='chat',
+ subscription_id=subscription, end_reason='close_a'))
+ await async_wait_until(entered.is_set)
+ await _call(_viewer(authority, tmp_path), 'session.resume', session_id='chat')
+ release.set()
+ await old
+ assert authority.pending_closes == {} and _stored(db) is None and calls == []
+
+ cancel_session_input(db, epoch=authority.epoch, admission_id=row['admission_id'])
+ db2, restarted, calls2 = await _restarted_owner(tmp_path, monkeypatch, db)
+ recover_local_sessions(restarted, schedule=True)
+ assert restarted.pending_closes == {} and calls2 == []
+ assert db2.get_session('chat')['ended_at'] is None
+
+
+async def test_a_stale_owner_cannot_delete_a_stored_close(tmp_path, monkeypatch):
+ """Compare-and-delete is epoch-fenced: an owner that lost the store leaves its successor's close."""
+ import json
+ from gateway.session_viewer_close import CLOSE_PREFIX, forget_persisted_close
+
+ db, authority, _calls = await _owner(tmp_path, monkeypatch)
+ record = json.dumps({'op': 'x', 'reason': 'cli_close'})
+ db.set_meta(CLOSE_PREFIX + 'chat', record)
+ stale = SimpleNamespace(db=db, epoch=authority.epoch - 1)
+ assert forget_persisted_close(stale, 'chat', record) is False
+ assert forget_persisted_close(authority, 'chat', 'another close') is False
+ assert db.get_meta(CLOSE_PREFIX + 'chat') == record
+ assert forget_persisted_close(authority, 'chat', record) is True
+ assert db.get_meta(CLOSE_PREFIX + 'chat') is None