Skip to content

The agent that grows with you

fix(gateway): the restart notice follows the turn's recorded interruption cause

Teknium7360d9659050
diff --git a/gateway/session_settlement_recovery.py b/gateway/session_settlement_recovery.py
index 054ce30d307..b6da9cd2496 100644
--- a/gateway/session_settlement_recovery.py
+++ b/gateway/session_settlement_recovery.py
@@ -99,7 +99,7 @@ def _publish_completion(authority, live, ref, row, settled, response, captured):
except Exception:
logger.exception('Terminal control cleanup failed for admission %s', row['admission_id'])
complete = completion_payload(row, settled, response, captured)
- notice = owner_stop_notice(getattr(authority, 'runner', None), settled['outcome'])
+ notice = owner_stop_notice(captured, settled['outcome'])
if notice and not complete.get('warning'):
# ``warning`` is the slot every surface already renders on message.complete (Ink activity
# line, Desktop toast), so a turn the owner's stop cut short says why (#134889 F1).
@@ -113,16 +113,17 @@ SHUTDOWN_INTERRUPT_NOTICE = ('Gateway shutting down: this turn was interrupted a
'Send your message again once Hermes is back.')
-def owner_stop_notice(runner, outcome):
+def owner_stop_notice(captured, outcome):
"""The named notice for a turn the owner's restart/stop interrupted, or None. With the default
``agent.restart_drain_timeout: 0`` a restart interrupts in-flight turns at once; the admission
settles ``interrupted`` and is never replayed, so the viewer is told the restart did it."""
- # ``_running`` drops only once stop() begins: a user Stop during a restart's after-turn wait
- # (draining, still running) is the user's own interrupt and gets no restart notice.
- if (outcome != 'interrupted' or runner is None or getattr(runner, '_draining', False) is not True
- or getattr(runner, '_running', True) is not False):
- return None
- return RESTART_INTERRUPT_NOTICE if getattr(runner, '_restart_requested', False) is True else SHUTDOWN_INTERRUPT_NOTICE
+ # The cause is the one the turn recorded when it was interrupted (stop() interrupts with these
+ # reasons), never the runner's flags at publication: a user Stop whose settlement was delayed
+ # until after shutdown began is still the user's Stop.
+ from gateway.run import _INTERRUPT_REASON_GATEWAY_RESTART, _INTERRUPT_REASON_GATEWAY_SHUTDOWN
+ cause = ((captured or {}).get('result') or {}).get('interrupt_message') if outcome == 'interrupted' else None
+ return {_INTERRUPT_REASON_GATEWAY_RESTART: RESTART_INTERRUPT_NOTICE,
+ _INTERRUPT_REASON_GATEWAY_SHUTDOWN: SHUTDOWN_INTERRUPT_NOTICE}.get(cause)
def _release_media(authority, row):
diff --git a/tests/gateway/test_restart_interrupt_notice.py b/tests/gateway/test_restart_interrupt_notice.py
index 5bffaff9c91..9cf701759cf 100644
--- a/tests/gateway/test_restart_interrupt_notice.py
+++ b/tests/gateway/test_restart_interrupt_notice.py
@@ -5,6 +5,11 @@ With the 0 s default a restart or stop interrupts in-flight turns at once. The a
case, ``tests/gateway/test_prompt_queued_transfer.py`` owner-restart case), and its
``message.complete`` carries a named notice in ``warning``, the slot Ink and Desktop already render.
"""
+import asyncio
+import sqlite3
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor
from types import SimpleNamespace
import pytest
@@ -33,40 +38,40 @@ class _Stream:
self.frames.append(payload)
-def _publish(runner, outcome, captured=None):
+def _publish(outcome, captured=None):
stream = _Stream()
live = SimpleNamespace(controls=SimpleNamespace(snapshot=lambda *a: None), event_stream=stream)
row = {'admission_id': 'adm-1', 'request_id': 'sub-1'}
- _publish_completion(SimpleNamespace(runner=runner), live, SimpleNamespace(session_id='s'), row,
+ _publish_completion(SimpleNamespace(runner=None), live, SimpleNamespace(session_id='s'), row,
{'outcome': outcome}, 'partial reply', captured)
(frame,) = stream.frames
return frame
-def _runner(*, restart, stopping=True):
- return SimpleNamespace(_draining=True, _running=not stopping, _restart_requested=restart)
+def _cause(message, **extra):
+ return {'result': {'interrupted': True, 'interrupt_message': message, **extra}}
[email protected](('restart', 'notice'), [(True, RESTART_INTERRUPT_NOTICE), (False, SHUTDOWN_INTERRUPT_NOTICE)])
-def test_turn_interrupted_by_the_owner_stop_carries_a_named_notice(restart, notice):
- frame = _publish(_runner(restart=restart), 'interrupted')
[email protected](('cause', 'notice'), [('Gateway restarting', RESTART_INTERRUPT_NOTICE),
+ ('Gateway shutting down', SHUTDOWN_INTERRUPT_NOTICE)])
+def test_turn_interrupted_by_the_owner_stop_carries_a_named_notice(cause, notice):
+ frame = _publish('interrupted', _cause(cause))
assert frame['status'] == 'interrupted' and frame['warning'] == notice
assert 'will not be run again' in notice
EVENTS['message.complete'].payload.model_validate({k: frame[k] for k in ('text', 'status', 'warning')})
def test_other_endings_get_no_restart_notice():
- # Completed during the drain, a user Stop in a restart's after-turn wait (still running),
- # and an ordinary interrupt with no restart all stay as they were.
- assert 'warning' not in _publish(_runner(restart=True), 'completed')
- assert 'warning' not in _publish(_runner(restart=True, stopping=False), 'interrupted')
- assert 'warning' not in _publish(SimpleNamespace(_draining=False, _running=True, _restart_requested=False),
- 'interrupted')
+ # A completed turn, a user Stop (no recorded cause, or its own) and a turn with no result.
+ assert 'warning' not in _publish('completed', _cause('Gateway restarting'))
+ assert 'warning' not in _publish('interrupted', {'result': {'interrupted': True}})
+ assert 'warning' not in _publish('interrupted', _cause('User requested stop'))
+ assert 'warning' not in _publish('interrupted')
def test_a_turn_warning_is_not_overwritten():
- captured = {'result': {'local_viewer': True, 'warning': 'own warning'}}
- assert _publish(_runner(restart=True), 'interrupted', captured)['warning'] == 'own warning'
+ captured = _cause('Gateway restarting', local_viewer=True, warning='own warning')
+ assert _publish('interrupted', captured)['warning'] == 'own warning'
def test_classic_cli_view_prints_the_restart_notice_for_an_interrupted_turn(capsys):
@@ -79,3 +84,64 @@ def test_classic_cli_view_prints_the_restart_notice_for_an_interrupted_turn(caps
view._complete('a2', {'text': 'done', 'status': 'complete', 'warning': 'other warning'})
err = capsys.readouterr().err
assert RESTART_INTERRUPT_NOTICE in err and 'other warning' not in err
+
+
+async def test_user_stop_is_not_relabelled_when_shutdown_starts_during_settlement(tmp_path):
+ """A user Stop whose settlement a competing SQLite writer delays until after shutdown began is
+ still the user's Stop: the notice follows the turn's recorded cause, not the runner's flags."""
+ from gateway.session_authority import LiveSession, SessionAuthority
+ from gateway.session_contract import SessionRef
+ from gateway.session_results import admission_result
+ from gateway.session_settlement_recovery import commit_and_publish
+ from hermes_state import SessionDB
+ from hermes_state_runtime import admit_session_input, begin_runtime_epoch, claim_session_input
+
+ runner = SimpleNamespace(_draining=False, _running=True, _restart_requested=False)
+ with SessionDB(db_path=tmp_path / 'state.db') as db:
+ db.create_session('s', source='test')
+ epoch = begin_runtime_epoch(db, instance_id='owner')
+ authority = SessionAuthority(runner, profile_id='owned', instance_id='owner', db=db, epoch=epoch)
+ live = authority.sessions['s'] = LiveSession(None, 'route')
+ ref = SessionRef('owned', 's')
+ admit_session_input(db, epoch=epoch, principal_id='person', session_id='s', request_id='r',
+ payload={'text': 'please work'})
+ row = claim_session_input(db, epoch=epoch, session_id='s')
+ authority._stamp_claim(ref, live, row)
+ frames = []
+ live.event_stream.observers.add(frames.append)
+ captured = {'result': {'final_response': '', 'messages': [], 'interrupted': True,
+ 'completed': False, 'interrupt_message': 'User requested stop'}}
+ reached = threading.Event()
+ settlement = {}
+ lock = sqlite3.connect(tmp_path / 'state.db')
+ lock.execute('BEGIN IMMEDIATE')
+
+ def trace(frame, event, arg):
+ if event == 'call' and frame.f_code.co_name == 'settle_session_input':
+ reached.set()
+ return trace
+
+ def commit():
+ sys.settrace(trace)
+ try:
+ commit_and_publish(authority, live, ref, row, '', 'interrupted', captured, settlement)
+ finally:
+ sys.settrace(None)
+
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ task = pool.submit(commit)
+ try:
+ assert await asyncio.to_thread(reached.wait, 10), 'settlement not reached'
+ assert not task.done(), 'competing SQLite writer should block settlement'
+ runner._draining = True
+ runner._running = False
+ lock.rollback()
+ await asyncio.wrap_future(task)
+ finally:
+ lock.close()
+ complete = [f['params']['payload'] for f in frames if f['params']['type'] == 'message.complete']
+ persisted = admission_result(db, row['admission_id'])
+ assert len(complete) == 1
+ assert persisted['result']['interrupt_message'] == 'User requested stop'
+ assert 'warning' not in complete[0], complete[0]