Skip to content

The agent that grows with you

fix(gateway): a Ctrl+D close survives a hard-killed gateway's unknown turn

Teknium13cc28a5ce42
diff --git a/gateway/session_authority.py b/gateway/session_authority.py
index bb644ebf92a..a1f441a89f2 100644
--- a/gateway/session_authority.py
+++ b/gateway/session_authority.py
@@ -667,10 +667,16 @@ class SessionAuthority:
settlement write failed (its exact result is still in ``pending_results``) commits that
result instead of discarding it: the answer was produced, only its receipt was lost."""
self.authorize(actor, ref, 'session:control')
- before = await self.receipt(actor, ref, admission_id)
+ await self.receipt(actor, ref, admission_id)
+ return await self.resolve_unknown_as_owner(ref, admission_id, generation)
+
+ async def resolve_unknown_as_owner(self, ref, admission_id, generation):
+ """``resolve_unknown`` past authorization: also the owner's own call when the last viewer
+ closed a session whose turn recovered ``unknown`` (gateway/session_viewer_close)."""
captured = self.pending_results.get(admission_id)
+ before = get_session_admission(self.db, admission_id=admission_id)
prepared = None
- if captured is not None and before.status == 'unknown':
+ if captured is not None and before is not None and before['status'] == 'unknown':
from gateway.session_results import prepare_result
row = get_session_admission(self.db, admission_id=admission_id)
prepared = prepare_result(row, captured['result'].get('final_response') or '', 'completed', captured)
diff --git a/gateway/session_local_recovery.py b/gateway/session_local_recovery.py
index 222ab32a225..6daf7ee2a82 100644
--- a/gateway/session_local_recovery.py
+++ b/gateway/session_local_recovery.py
@@ -178,6 +178,10 @@ def recover_local_sessions(authority, *, schedule=False):
logging.getLogger(__name__).warning('Local session %s paused: %s', sid, exc.reason)
if schedule:
# A last-viewer close the previous owner stopped before completing (#134889 LC-6b).
- from gateway.session_viewer_close import rearm_persisted_closes
+ from gateway.session_viewer_close import rearm_persisted_closes, settle_viewer_close
+ from gateway.session_runtime_workers import track_mutation
for sid in rearm_persisted_closes(authority):
authority._schedule(SessionRef(authority.profile_id, sid))
+ if any(row['status'] == 'unknown' for row in list_session_admissions(authority.db, session_id=sid)):
+ # A hard-killed owner's turn: the drain pauses on it, so the close resolves it.
+ track_mutation(authority, settle_viewer_close(authority, sid))
diff --git a/gateway/session_viewer_close.py b/gateway/session_viewer_close.py
index 58edf8dba4a..a41c220e9da 100644
--- a/gateway/session_viewer_close.py
+++ b/gateway/session_viewer_close.py
@@ -160,6 +160,8 @@ async def settle_viewer_close(authority, session_id):
if close is None or live is None or live.subscribers:
authority.pending_closes.pop(session_id, None)
return
+ if not await _resolve_abandoned_unknown(authority, session_id, live, close):
+ return # left paused (a viewer came back, or the resolution was refused)
target = _physical_target(authority, session_id)
# Every await below can let a viewer reattach (attach drops the pending close) or another
# settler run, so the close is re-checked after each one.
@@ -185,6 +187,36 @@ async def settle_viewer_close(authority, session_id):
await _run_close_lifecycle(authority, live, target, platform, close.reason)
+async def _resolve_abandoned_unknown(authority, session_id, live, close):
+ """A turn the previous owner lost (hard kill) recovers ``unknown`` and pauses the FIFO until
+ someone Discards it, but the last viewer closed the session: nobody is left to. The viewer's
+ close is that decision, so the owner resolves it as Discard does (interrupted, never replayed,
+ transcript boundary written) and the drain moves on; a queued successor still runs before the
+ close. A viewer back first cancels the close and keeps the decision. Returns whether the
+ close may go on to its idle check (no ``unknown`` turn is left)."""
+ from hermes_state_runtime import list_session_admissions
+ rows = await asyncio.to_thread(list_session_admissions, authority.db, session_id=session_id)
+ unknown = [row for row in rows if row['status'] == 'unknown']
+ if not unknown:
+ return True
+ from gateway.session_contract import SessionRef
+ ref = SessionRef(authority.profile_id, session_id)
+ for row in unknown:
+ if not _still_pending(authority, session_id, live, close):
+ return False
+ try:
+ await authority.resolve_unknown_as_owner(ref, row['admission_id'], row['generation'])
+ except RuntimeStoreError as exc:
+ logger.warning('Unknown turn %s on closed session %s left paused: %s',
+ row['admission_id'], session_id, exc.reason)
+ return False
+ logger.info('Unknown turn %s resolved interrupted: its session %s was closed by its last viewer',
+ row['admission_id'], session_id)
+ # The resolution also rescheduled the drain; the idle check below and its next idle point
+ # are both safe (the close is claimed exactly once).
+ return _still_pending(authority, session_id, live, close)
+
+
def _still_pending(authority, session_id, live, close):
return not live.subscribers and authority.pending_closes.get(session_id) is close
diff --git a/tests/gateway/test_viewer_close_lifecycle.py b/tests/gateway/test_viewer_close_lifecycle.py
index 7910bad6a10..39b25508d2a 100644
--- a/tests/gateway/test_viewer_close_lifecycle.py
+++ b/tests/gateway/test_viewer_close_lifecycle.py
@@ -486,3 +486,60 @@ async def test_a_stale_owner_cannot_delete_a_stored_close(tmp_path, monkeypatch)
assert db.get_meta(CLOSE_PREFIX + 'chat') == record
assert forget_persisted_close(authority, 'chat', record) is True
assert db.get_meta(CLOSE_PREFIX + 'chat') is None
+
+
+async def _hard_killed_mid_turn(tmp_path, monkeypatch):
+ """Ctrl+D mid-turn, then the owner dies with no shutdown code (SIGKILL): the started admission
+ and the stored close are all that is left; the next owner recovers the admission ``unknown``."""
+ from hermes_state_runtime import admit_session_input, claim_session_input
+
+ db, authority, _calls = await _owner(tmp_path, monkeypatch)
+ viewer, subscription = await _open(db, authority, tmp_path)
+ admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',
+ request_id='r1', payload={'text': 'mid-turn'})
+ row = claim_session_input(db, epoch=authority.epoch, session_id='chat')
+ assert row['status'] == 'started'
+ await _call(viewer, 'session.detach', session_id='chat', subscription_id=subscription, end_reason='cli_close')
+ assert _stored(db)['reason'] == 'cli_close' and db.get_session('chat')['ended_at'] is None
+ db2, restarted, calls2 = await _restarted_owner(tmp_path, monkeypatch, db)
+ from hermes_state_runtime import get_session_admission
+ assert get_session_admission(db2, admission_id=row['admission_id'])['status'] == 'unknown'
+ return db2, restarted, calls2, row
+
+
+async def test_a_close_stored_before_a_hard_kill_ends_the_row_past_the_unknown_turn(tmp_path, monkeypatch):
+ """#134889 LC-6b, @alt-glitch warning 1 on #140489: after SIGKILL the in-flight turn recovers
+ ``unknown``, the drain pauses on ``unknown_execution`` before the idle point, and the stored close
+ never applied: the row stayed open forever. Nobody is left to Discard the turn (the viewer closed
+ the session), so the owner resolves it as interrupted (never replayed) and the close completes."""
+ from gateway.session_local_recovery import recover_local_sessions
+ from hermes_state_runtime import get_session_admission
+
+ db2, restarted, calls2, row = await _hard_killed_mid_turn(tmp_path, monkeypatch)
+ recover_local_sessions(restarted, schedule=True)
+ await async_wait_until(lambda: len(calls2) == 2, timeout=10)
+ assert db2.get_session('chat')['end_reason'] == 'cli_close'
+ assert db2.get_session('chat')['ended_at'] is not None
+ settled = get_session_admission(db2, admission_id=row['admission_id'])
+ assert (settled['status'], settled['outcome']) == ('terminal', 'interrupted')
+ assert [name for name, _ in calls2] == ['commit', 'finalize'] and _stored(db2) is None
+
+
+async def test_a_viewer_back_before_the_unknown_is_resolved_keeps_the_turn_for_its_own_discard(tmp_path, monkeypatch):
+ """The owner only resolves an unknown turn for a close nobody cancelled: a viewer that attached
+ after the restart cancels the close and decides about the turn itself (Discard)."""
+ from hermes_state_runtime import get_session_admission
+ from gateway.session_viewer_close import rearm_persisted_closes
+
+ db2, restarted, calls2, row = await _hard_killed_mid_turn(tmp_path, monkeypatch)
+ from gateway.session_local_recovery import restore_local_session
+ restore_local_session(restarted, 'chat')
+ await _call(_viewer(restarted, tmp_path), 'session.resume', session_id='chat')
+ assert rearm_persisted_closes(restarted) == [] and restarted.pending_closes == {}
+ from gateway.session_contract import SessionRef
+ restarted._schedule(SessionRef(restarted.profile_id, 'chat'))
+ await asyncio.wait_for(restarted.sessions['chat'].task, 10) # the drain pauses on the unknown turn
+ assert get_session_admission(db2, admission_id=row['admission_id'])['status'] == 'unknown'
+ assert db2.get_session('chat')['ended_at'] is None and calls2 == [] and _stored(db2) is None