Skip to content

The agent that grows with you

fix(gateway): a restart notice replaces this turn's own failed-turn boundary

Tekniumbceea8887b0b
diff --git a/gateway/session_results.py b/gateway/session_results.py
index 3df19a58fed..6eec49c2299 100644
--- a/gateway/session_results.py
+++ b/gateway/session_results.py
@@ -90,7 +90,8 @@ def finish_result(db, *, epoch, row, response, outcome, result=None, prepared=No
# terminal returns above, so the notice is written once.
from gateway.session_settlement_recovery import owner_stop_notice
notice = owner_stop_notice(result, outcome, row)
- write = None if notice is None else (lambda conn, _current: close_discarded_turn(db, conn, row, notice))
+ write = None if notice is None else (lambda conn, _current: close_discarded_turn(
+ db, conn, row, notice, replace_after=row.get('transcript_mark')))
settled = settle_session_input(db, epoch=epoch, admission_id=row['admission_id'],
generation=row['generation'], outcome=outcome, result=stored, _transfers=transfers,
_terminal_write=write)
@@ -107,7 +108,7 @@ def admission_result(db, admission_id):
return load_result(saved[0]) if saved else None
-def close_discarded_turn(db, conn, row, notice=None):
+def close_discarded_turn(db, conn, row, notice=None, *, replace_after=None):
"""Close a discarded ``unknown`` turn the way a failed turn is closed, on the resolution's
own transaction (``resolve_unknown_session_input(_terminal_write=...)``).
@@ -124,14 +125,23 @@ def close_discarded_turn(db, conn, row, notice=None):
gets the same boundary (merged onto the call row, whose calls are then pruned).
``notice`` (an unattended automation turn settled by the owner, GW-10) replaces the hedged copy
- and is written even on a closed tail: nobody resolved it, so the transcript must say why."""
+ and is written even on a closed tail: nobody resolved it, so the transcript must say why.
+
+ ``replace_after`` (the claim's transcript mark, an owner-stop notice): when the tail is a
+ failed-turn boundary written after that mark, it is THIS turn's own (the agent's core closer
+ wrote "Send it again" for the interrupted turn), so the notice replaces its copy instead of
+ stacking a second boundary (#134889 GW-21). An earlier turn's boundary stays."""
import time
from agent.turn_failure_copy import FAILED_TURN_DISPLAY_KIND, PARTIAL_FAILED_TURN_NOTICE
from hermes_state_local_lineage import local_physical_target
from hermes_state_runtime import _canonical_chain
target = _canonical_chain(conn, local_physical_target(conn, row['target_session_id']))[-1]
- tail = conn.execute("SELECT role, tool_calls FROM messages WHERE session_id=? AND active=1 "
+ tail = conn.execute("SELECT role, tool_calls, id, display_kind FROM messages WHERE session_id=? AND active=1 "
"AND role NOT IN ('session_meta','system') ORDER BY id DESC LIMIT 1", (target,)).fetchone()
+ if (notice is not None and type(replace_after) is int and tail is not None and tail[0] == 'assistant'
+ and tail[3] == FAILED_TURN_DISPLAY_KIND and tail[2] > replace_after):
+ conn.execute("UPDATE messages SET content=?, display_metadata=NULL WHERE id=?", (notice, tail[2]))
+ return True
if notice is None and (tail is None or not (
tail[0] == 'user' or (tail[0] == 'assistant' and tail[1] not in (None, '', '[]')))):
return False
diff --git a/hermes_state_runtime.py b/hermes_state_runtime.py
index d8584482166..7ebad76355e 100644
--- a/hermes_state_runtime.py
+++ b/hermes_state_runtime.py
@@ -227,7 +227,12 @@ def claim_session_input(db, *, epoch: int, session_id: str, _guard=None, head=_A
changed = conn.execute("UPDATE session_admissions SET status='started',owner_epoch=?,generation=? WHERE admission_id=? AND status='queued'", (epoch, generation, row['admission_id']))
if changed.rowcount != 1:
raise RuntimeStoreError('stale_generation')
- return _row(_admission(conn, row['admission_id']))
+ claimed = _row(_admission(conn, row['admission_id']))
+ # The transcript high-water mark at the claim (``messages.id`` is AUTOINCREMENT across every
+ # session): a row above it on this conversation was written by this turn, so settlement can
+ # tell this turn's own failed-turn boundary from an earlier turn's (#134889 GW-21).
+ claimed['transcript_mark'] = conn.execute('SELECT COALESCE(MAX(id), 0) FROM messages').fetchone()[0]
+ return claimed
return db._execute_write(write)
diff --git a/tests/gateway/test_restart_interrupt_notice.py b/tests/gateway/test_restart_interrupt_notice.py
index 0e89008e304..4d0cd588195 100644
--- a/tests/gateway/test_restart_interrupt_notice.py
+++ b/tests/gateway/test_restart_interrupt_notice.py
@@ -397,6 +397,72 @@ def test_automation_turn_transcript_notice_is_the_neutral_copy(tmp_path):
assert _notices(db) == [AUTOMATION_INTERRUPTED_NOTICE]
+
+def _core_closed_turn(tmp_path, monkeypatch, *, principal='person', earlier_failed=False):
+ """A claimed admission whose agent was interrupted by the restart before any assistant text:
+ the real AIAgent persists the user row and the real core closer
+ (``agent/conversation_loop.py::_close_durable_failed_turn``) writes its failed-turn boundary.
+ ``earlier_failed``: a previous turn already ended on a failed-turn boundary before this claim."""
+ monkeypatch.setenv('HERMES_HOME', str(tmp_path))
+ from agent.conversation_loop import _close_durable_failed_turn
+ from agent.turn_failure_copy import FAILED_TURN_DISPLAY_KIND, FAILED_TURN_NOTICE
+ from gateway.session_authority import LiveSession, SessionAuthority
+ from gateway.session_contract import SessionRef
+ from hermes_state import SessionDB
+ from hermes_state_runtime import admit_session_input, begin_runtime_epoch, claim_session_input
+ from run_agent import AIAgent
+
+ runner = SimpleNamespace(_draining=False, _running=True, _restart_requested=True)
+ db = SessionDB(db_path=tmp_path / 'state.db')
+ db.create_session('s', source='test')
+ if earlier_failed:
+ db.append_message('s', 'user', 'earlier request')
+ db.append_message('s', 'assistant', FAILED_TURN_NOTICE, display_kind=FAILED_TURN_DISPLAY_KIND)
+ epoch = begin_runtime_epoch(db, instance_id='owner')
+ authority = SessionAuthority(runner, profile_id='owned', instance_id='owner', db=db, epoch=epoch)
+ live = authority.sessions['s'] = LiveSession(None, 'route')
+ ref = SessionRef('owned', 's')
+ payload = {'text': 'please work'}
+ admit_session_input(db, epoch=epoch, principal_id=principal, session_id='s', request_id='r', payload=payload)
+ row = claim_session_input(db, epoch=epoch, session_id='s')
+ authority._stamp_claim(ref, live, row)
+ result = {'final_response': '', 'interrupted': True, 'completed': False, 'interrupt_message': 'Gateway restarting'}
+ if not earlier_failed:
+ agent = AIAgent(session_db=db, session_id='s', model='test-model', provider='openai-compat', api_key='test',
+ base_url='http://127.0.0.1:1/v1', quiet_mode=True, skip_context_files=True, skip_memory=True)
+ messages = [{'role': 'user', 'content': 'please work'}]
+ agent._flush_messages_to_session_db(messages) # the turn-start flush of the accepted user row
+ result['messages'] = messages
+ _close_durable_failed_turn(agent, result)
+ return db, authority, live, ref, row, result
+
+
[email protected]('principal', ['person', 'cron-owner'])
+def test_restart_notice_replaces_the_core_failed_turn_boundary_of_the_same_turn(tmp_path, monkeypatch, principal):
+ """#134889 GW-21 review (GW-A): the agent's own closer already ended this turn with
+ "Send it again"; settlement replaces that boundary with the restart notice in its own write,
+ so the transcript holds one Hermes boundary, and an automation row never keeps resend advice."""
+ from agent.turn_failure_copy import AUTOMATION_INTERRUPTED_NOTICE, FAILED_TURN_NOTICE
+ from gateway.session_settlement_recovery import commit_and_publish
+ db, authority, live, ref, row, result = _core_closed_turn(tmp_path, monkeypatch, principal=principal)
+ with db:
+ assert _notices(db) == [FAILED_TURN_NOTICE] # the real core closer wrote it
+ commit_and_publish(authority, live, ref, row, '', 'interrupted', {'result': result}, {})
+ expected = RESTART_INTERRUPT_NOTICE if principal == 'person' else AUTOMATION_INTERRUPTED_NOTICE
+ assert _notices(db) == [expected]
+ assert [m['role'] for m in db.get_messages('s')] == ['user', 'assistant']
+
+
+def test_an_earlier_turns_failed_boundary_is_kept_and_the_restart_notice_appended(tmp_path, monkeypatch):
+ """A failed-turn boundary written before this claim belongs to an earlier turn: it stays."""
+ from agent.turn_failure_copy import FAILED_TURN_NOTICE
+ from gateway.session_settlement_recovery import commit_and_publish
+ db, authority, live, ref, row, result = _core_closed_turn(tmp_path, monkeypatch, earlier_failed=True)
+ with db:
+ commit_and_publish(authority, live, ref, row, '', 'interrupted', {'result': result}, {})
+ assert _notices(db) == [FAILED_TURN_NOTICE, RESTART_INTERRUPT_NOTICE]
+
+
@pytest.mark.platforms("linux")
@pytest.mark.parametrize('ending', ['spawned', 'cancelled'])
def test_shutdown_stop_before_the_managed_worker_spawned_carries_its_cause(tmp_path, monkeypatch, ending):
@@ -458,6 +524,7 @@ def test_shutdown_stop_before_the_managed_worker_spawned_carries_its_cause(tmp_p
assert result['interrupted'] and result['interrupt_message'] == 'Gateway restarting'
+
def test_a_user_stop_first_keeps_no_shutdown_cause_before_spawn():
"""A user's Stop latched for the pending generation came first: shutdown records no cause."""
from gateway.run_runtime import stop_authority_turns