The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): a Ctrl+D close survives a hard-killed gateway's unknown turn
Teknium13cc28a5ce42
diff --git a/gateway/session_authority.py b/gateway/session_authority.pyindex bb644ebf92a..a1f441a89f2 100644--- a/gateway/session_authority.py+++ b/gateway/session_authority.py@@ -667,10 +667,16 @@ class SessionAuthority:settlement write failed (its exact result is still in ``pending_results``) commits thatresult instead of discarding it: the answer was produced, only its receipt was lost."""self.authorize(actor, ref, 'session:control')- before = await self.receipt(actor, ref, admission_id)+ await self.receipt(actor, ref, admission_id)+ return await self.resolve_unknown_as_owner(ref, admission_id, generation)++ async def resolve_unknown_as_owner(self, ref, admission_id, generation):+ """``resolve_unknown`` past authorization: also the owner's own call when the last viewer+ closed a session whose turn recovered ``unknown`` (gateway/session_viewer_close)."""captured = self.pending_results.get(admission_id)+ before = get_session_admission(self.db, admission_id=admission_id)prepared = None- if captured is not None and before.status == 'unknown':+ if captured is not None and before is not None and before['status'] == 'unknown':from gateway.session_results import prepare_resultrow = get_session_admission(self.db, admission_id=admission_id)prepared = prepare_result(row, captured['result'].get('final_response') or '', 'completed', captured)diff --git a/gateway/session_local_recovery.py b/gateway/session_local_recovery.pyindex 222ab32a225..6daf7ee2a82 100644--- a/gateway/session_local_recovery.py+++ b/gateway/session_local_recovery.py@@ -178,6 +178,10 @@ def recover_local_sessions(authority, *, schedule=False):logging.getLogger(__name__).warning('Local session %s paused: %s', sid, exc.reason)if schedule:# A last-viewer close the previous owner stopped before completing (#134889 LC-6b).- from gateway.session_viewer_close import rearm_persisted_closes+ from gateway.session_viewer_close import rearm_persisted_closes, settle_viewer_close+ from gateway.session_runtime_workers import track_mutationfor sid in rearm_persisted_closes(authority):authority._schedule(SessionRef(authority.profile_id, sid))+ if any(row['status'] == 'unknown' for row in list_session_admissions(authority.db, session_id=sid)):+ # A hard-killed owner's turn: the drain pauses on it, so the close resolves it.+ track_mutation(authority, settle_viewer_close(authority, sid))diff --git a/gateway/session_viewer_close.py b/gateway/session_viewer_close.pyindex 58edf8dba4a..a41c220e9da 100644--- a/gateway/session_viewer_close.py+++ b/gateway/session_viewer_close.py@@ -160,6 +160,8 @@ async def settle_viewer_close(authority, session_id):if close is None or live is None or live.subscribers:authority.pending_closes.pop(session_id, None)return+ if not await _resolve_abandoned_unknown(authority, session_id, live, close):+ return # left paused (a viewer came back, or the resolution was refused)target = _physical_target(authority, session_id)# Every await below can let a viewer reattach (attach drops the pending close) or another# settler run, so the close is re-checked after each one.@@ -185,6 +187,36 @@ async def settle_viewer_close(authority, session_id):await _run_close_lifecycle(authority, live, target, platform, close.reason)+async def _resolve_abandoned_unknown(authority, session_id, live, close):+ """A turn the previous owner lost (hard kill) recovers ``unknown`` and pauses the FIFO until+ someone Discards it, but the last viewer closed the session: nobody is left to. The viewer's+ close is that decision, so the owner resolves it as Discard does (interrupted, never replayed,+ transcript boundary written) and the drain moves on; a queued successor still runs before the+ close. A viewer back first cancels the close and keeps the decision. Returns whether the+ close may go on to its idle check (no ``unknown`` turn is left)."""+ from hermes_state_runtime import list_session_admissions+ rows = await asyncio.to_thread(list_session_admissions, authority.db, session_id=session_id)+ unknown = [row for row in rows if row['status'] == 'unknown']+ if not unknown:+ return True+ from gateway.session_contract import SessionRef+ ref = SessionRef(authority.profile_id, session_id)+ for row in unknown:+ if not _still_pending(authority, session_id, live, close):+ return False+ try:+ await authority.resolve_unknown_as_owner(ref, row['admission_id'], row['generation'])+ except RuntimeStoreError as exc:+ logger.warning('Unknown turn %s on closed session %s left paused: %s',+ row['admission_id'], session_id, exc.reason)+ return False+ logger.info('Unknown turn %s resolved interrupted: its session %s was closed by its last viewer',+ row['admission_id'], session_id)+ # The resolution also rescheduled the drain; the idle check below and its next idle point+ # are both safe (the close is claimed exactly once).+ return _still_pending(authority, session_id, live, close)++def _still_pending(authority, session_id, live, close):return not live.subscribers and authority.pending_closes.get(session_id) is closediff --git a/tests/gateway/test_viewer_close_lifecycle.py b/tests/gateway/test_viewer_close_lifecycle.pyindex 7910bad6a10..39b25508d2a 100644--- a/tests/gateway/test_viewer_close_lifecycle.py+++ b/tests/gateway/test_viewer_close_lifecycle.py@@ -486,3 +486,60 @@ async def test_a_stale_owner_cannot_delete_a_stored_close(tmp_path, monkeypatch)assert db.get_meta(CLOSE_PREFIX + 'chat') == recordassert forget_persisted_close(authority, 'chat', record) is Trueassert db.get_meta(CLOSE_PREFIX + 'chat') is None+++async def _hard_killed_mid_turn(tmp_path, monkeypatch):+ """Ctrl+D mid-turn, then the owner dies with no shutdown code (SIGKILL): the started admission+ and the stored close are all that is left; the next owner recovers the admission ``unknown``."""+ from hermes_state_runtime import admit_session_input, claim_session_input++ db, authority, _calls = await _owner(tmp_path, monkeypatch)+ viewer, subscription = await _open(db, authority, tmp_path)+ admit_session_input(db, epoch=authority.epoch, principal_id='native', session_id='chat',+ request_id='r1', payload={'text': 'mid-turn'})+ row = claim_session_input(db, epoch=authority.epoch, session_id='chat')+ assert row['status'] == 'started'+ await _call(viewer, 'session.detach', session_id='chat', subscription_id=subscription, end_reason='cli_close')+ assert _stored(db)['reason'] == 'cli_close' and db.get_session('chat')['ended_at'] is None+ db2, restarted, calls2 = await _restarted_owner(tmp_path, monkeypatch, db)+ from hermes_state_runtime import get_session_admission+ assert get_session_admission(db2, admission_id=row['admission_id'])['status'] == 'unknown'+ return db2, restarted, calls2, row+++async def test_a_close_stored_before_a_hard_kill_ends_the_row_past_the_unknown_turn(tmp_path, monkeypatch):+ """#134889 LC-6b, @alt-glitch warning 1 on #140489: after SIGKILL the in-flight turn recovers+ ``unknown``, the drain pauses on ``unknown_execution`` before the idle point, and the stored close+ never applied: the row stayed open forever. Nobody is left to Discard the turn (the viewer closed+ the session), so the owner resolves it as interrupted (never replayed) and the close completes."""+ from gateway.session_local_recovery import recover_local_sessions+ from hermes_state_runtime import get_session_admission++ db2, restarted, calls2, row = await _hard_killed_mid_turn(tmp_path, monkeypatch)+ recover_local_sessions(restarted, schedule=True)+ await async_wait_until(lambda: len(calls2) == 2, timeout=10)+ assert db2.get_session('chat')['end_reason'] == 'cli_close'+ assert db2.get_session('chat')['ended_at'] is not None+ settled = get_session_admission(db2, admission_id=row['admission_id'])+ assert (settled['status'], settled['outcome']) == ('terminal', 'interrupted')+ assert [name for name, _ in calls2] == ['commit', 'finalize'] and _stored(db2) is None+++async def test_a_viewer_back_before_the_unknown_is_resolved_keeps_the_turn_for_its_own_discard(tmp_path, monkeypatch):+ """The owner only resolves an unknown turn for a close nobody cancelled: a viewer that attached+ after the restart cancels the close and decides about the turn itself (Discard)."""+ from hermes_state_runtime import get_session_admission+ from gateway.session_viewer_close import rearm_persisted_closes++ db2, restarted, calls2, row = await _hard_killed_mid_turn(tmp_path, monkeypatch)+ from gateway.session_local_recovery import restore_local_session+ restore_local_session(restarted, 'chat')+ await _call(_viewer(restarted, tmp_path), 'session.resume', session_id='chat')+ assert rearm_persisted_closes(restarted) == [] and restarted.pending_closes == {}+ from gateway.session_contract import SessionRef+ restarted._schedule(SessionRef(restarted.profile_id, 'chat'))+ await asyncio.wait_for(restarted.sessions['chat'].task, 10) # the drain pauses on the unknown turn+ assert get_session_admission(db2, admission_id=row['admission_id'])['status'] == 'unknown'+ assert db2.get_session('chat')['ended_at'] is None and calls2 == [] and _stored(db2) is None