Skip to content

The agent that grows with you

fix(gateway): a stamped local route keeps its conversation instead of an empty replacement

Tekniumf97d306232b8
diff --git a/gateway/session.py b/gateway/session.py
index 4f4f895c923..45478880371 100644
--- a/gateway/session.py
+++ b/gateway/session.py
@@ -949,8 +949,11 @@ class SessionStore(
checks = None
if not force_new and observed is not None:
sid = observed.session_id
+ # A local route the authority holds by receipt is never stale: a stamped row is
+ # reopened by its next admitted turn, never replaced (#140642).
checks = _RouteChecks(
- sid, self._compression_tip_for_session_id(sid), self._is_session_ended_in_db(sid),
+ sid, self._compression_tip_for_session_id(sid),
+ self._is_session_ended_in_db(sid) and self._local_receipt_target(observed) is None,
self._route_reset_reason(observed),
)
# Phase 2 (lock): apply the decisions to _entries.
diff --git a/gateway/session_lifecycle.py b/gateway/session_lifecycle.py
index 3f13ced6319..66b0404ca2d 100644
--- a/gateway/session_lifecycle.py
+++ b/gateway/session_lifecycle.py
@@ -89,6 +89,33 @@ class SessionLifecycleMixin:
return False
return row is None or row.get("end_reason") is not None
+ def _local_receipt_target(self, entry: Optional[SessionEntry]) -> Optional[str]:
+ """The physical target the authority's receipt names for *entry*'s local route, else None
+ (not a local route, no receipt, or the receipt belongs to another route).
+
+ Such a route moves only through the authority (compression and reset advance the receipt
+ and the route in one transaction), and its first admitted turn reopens a stamped row
+ (``reopen_local_session``). A row stamped ended (a viewer's ``ws_orphan_reap``) is
+ therefore never a stale route: dropping it minted a replacement id the authority refuses
+ (``admission_conflict``) and stranded the conversation (#140642)."""
+ origin = getattr(entry, "origin", None)
+ if origin is None or getattr(origin, "platform", None) is None or origin.platform.value != "local":
+ return None
+ db = self._db_for_key(entry.session_key)
+ if db is None or getattr(db, "_read_ctx", None) is None:
+ return None
+ import sqlite3
+
+ from hermes_state_local import local_receipt
+ from hermes_state_runtime import RuntimeStoreError
+ try:
+ receipt = local_receipt(db, str(origin.chat_id))
+ if receipt.get("route") != entry.session_key:
+ return None
+ return str(receipt["entry"]["session_id"]) or None
+ except (RuntimeStoreError, sqlite3.Error, KeyError, TypeError, AttributeError):
+ return None
+
def _route_reset_reason(self, entry: SessionEntry) -> Optional[str]:
"""Only explicit suspension replaces a routed conversation; time never does."""
return "suspended" if entry.suspended else None
diff --git a/gateway/session_local_recovery.py b/gateway/session_local_recovery.py
index fbba6d473cb..f1f02a43207 100644
--- a/gateway/session_local_recovery.py
+++ b/gateway/session_local_recovery.py
@@ -110,6 +110,28 @@ def owned_receipt(authority, sid):
return receipt, source, route, _checked_entry(authority, receipt, source, route)
+def _stray_route_target(authority, receipt, session_id):
+ """*session_id* is a replacement the SessionStore's stale-route self-heal minted on this local
+ route before #140642: outside the receipt's lineage, never admitted, no transcript. The receipt
+ is the route's truth, so restoring it drops that entry instead of refusing the conversation
+ forever (``admission_conflict`` on every resume). Anything with history or ledger rows still
+ refuses: it is not provably a stray."""
+ if not session_id or session_id in receipt.get('lineage', [receipt['session_id']]):
+ return False
+ with authority.db._read_ctx() as conn:
+ busy = conn.execute(
+ 'SELECT 1 FROM messages WHERE session_id=? UNION ALL SELECT 1 FROM session_admissions '
+ 'WHERE target_session_id=? UNION ALL SELECT 1 FROM worker_executions WHERE session_id=? '
+ 'UNION ALL SELECT 1 FROM sessions WHERE parent_session_id=? LIMIT 1',
+ (session_id, session_id, session_id, session_id)).fetchone()
+ if busy is not None:
+ return False
+ logging.getLogger(__name__).warning(
+ 'Local route for %s named an empty replacement %s outside its lineage; restoring the receipt target',
+ receipt['session_id'], session_id)
+ return True
+
+
def restore_local_session(authority, sid):
from gateway.session_authority import LiveSession
from gateway.session_local import LocalSessionAdapter
@@ -147,7 +169,9 @@ def restore_local_session(authority, sid):
store._ensure_loaded_locked()
current = store._entries.get(route)
if current is not None and current.session_id != target:
- raise RuntimeStoreError('admission_conflict')
+ if not _stray_route_target(authority, receipt, current.session_id):
+ raise RuntimeStoreError('admission_conflict')
+ current = None
# The receipt is the creation copy; a persisted entry for the same target keeps the
# route's explicit ``/yolo`` toggle (an OFF over a ``--yolo`` launch must survive a
# cold restore). An untoggled entry stays None and keeps following the launch policy.
@@ -165,7 +189,8 @@ def restore_local_session(authority, sid):
else:
with store._lock:
current = store._entry_locked(route)
- if current is None or current.session_id not in receipt.get('lineage', [sid]):
+ if current is None or (current.session_id not in receipt.get('lineage', [sid])
+ and not _stray_route_target(authority, receipt, current.session_id)):
raise RuntimeStoreError('admission_conflict')
current.session_id = target
return SessionRef(authority.profile_id, sid)
diff --git a/gateway/session_persistence.py b/gateway/session_persistence.py
index feda98b8378..acc675426d4 100644
--- a/gateway/session_persistence.py
+++ b/gateway/session_persistence.py
@@ -449,6 +449,21 @@ class SessionPersistenceMixin:
"""For a routing entry whose row has ended: ``"prune"``, a replacement entry (repoint), or
None (keep as-is)."""
from gateway.session_lifecycle import _now
+ local_target = self._local_receipt_target(entry)
+ if local_target is not None:
+ # The authority's receipt names the route's target (a compression child after a
+ # split); its first admitted turn reopens a stamped row. A peer-row recovery here
+ # adopted an empty, never-admitted replacement instead (#140642).
+ if local_target == entry.session_id:
+ return None
+ import copy
+ repointed = copy.copy(entry)
+ repointed.session_id = local_target
+ logger.warning(
+ "gateway.session: repointing stale sessions.json entry %r from ended %s "
+ "(end_reason=%r) to its local receipt target %s", key, entry.session_id,
+ row["end_reason"], local_target)
+ return repointed
recovered_entry = None
if entry.origin is not None:
try:
diff --git a/tests/gateway/test_local_route_orphan_reap_stamp.py b/tests/gateway/test_local_route_orphan_reap_stamp.py
new file mode 100644
index 00000000000..b4cd7091659
--- /dev/null
+++ b/tests/gateway/test_local_route_orphan_reap_stamp.py
@@ -0,0 +1,98 @@
+"""A local route the authority owns keeps its conversation when its row is stamped ended (#140642).
+
+A viewer process stamped a Desktop session ``ws_orphan_reap`` while the gateway still ran its turn.
+The SessionStore's stale-route self-heal (#54878) then dropped the local route, minted a fresh
+session id (whose peer record the authority refuses: ``admission_conflict``, after the row was
+already inserted), and the next restart's prune repointed ``sessions.json`` at that empty row, so
+every later resume was refused ``admission_conflict``. A local route belongs to its receipt: only
+the authority advances it (compression, reset), and the first admitted turn reopens a stamped row.
+"""
+from types import SimpleNamespace
+
+import pytest
+
+
+async def _local_session(tmp_path, monkeypatch):
+ from gateway import run
+ from gateway.config import GatewayConfig
+ from gateway.session import SessionStore
+ from gateway.session_authority import initialize_session_authority
+ from gateway.session_local import create_local_session
+
+ monkeypatch.setattr(run, '_load_gateway_config', lambda: {'platform_toolsets': {'cli': []}})
+ store = SessionStore(tmp_path / 'sessions', GatewayConfig())
+ runner = SimpleNamespace(session_store=store, _session_db=store._db, adapters={}, _draining=False,
+ _cached_agent_for=lambda route: None, _adapter_for_source=lambda source: None)
+ authority = await initialize_session_authority(runner, profile_id=str(tmp_path), instance_id='fixture')
+ from gateway.session_controls import AuthorityConnection
+ connection = AuthorityConnection(authority, object(), {'user_id': 'owner'})
+ ref = create_local_session(authority, connection.actor, {'request_id': 'r', 'source': 'gui',
+ 'cwd': str(tmp_path), 'model': 'm', 'toolsets': []})
+ authority.db.append_message(ref.session_id, 'user', 'earlier turn')
+ return store, authority, connection, ref
+
+
+def _rows(db):
+ with db._read_ctx() as conn:
+ return [row[0] for row in conn.execute('SELECT id FROM sessions ORDER BY started_at')]
+
+
+async def test_stamped_local_route_is_not_replaced_by_a_fresh_session(tmp_path, monkeypatch):
+ store, authority, connection, ref = await _local_session(tmp_path, monkeypatch)
+ try:
+ live = authority.sessions[ref.session_id]
+ authority.db.end_session(ref.session_id, 'ws_orphan_reap')
+
+ entry = store.get_or_create_session(live.source)
+
+ assert entry.session_id == ref.session_id
+ assert _rows(authority.db) == [ref.session_id], 'no replacement row may be minted for a local route'
+ # The conversation still resumes through the authority.
+ reply = await connection.dispatch({'id': 1, 'method': 'session.resume',
+ 'params': {'session_id': ref.session_id}})
+ assert reply['result']['messages'][0]['content'] == 'earlier turn', reply
+ finally:
+ await connection.close()
+ authority.db.close()
+
+
+async def test_restart_prune_keeps_a_stamped_local_route_on_its_receipt_target(tmp_path, monkeypatch):
+ store, authority, connection, ref = await _local_session(tmp_path, monkeypatch)
+ try:
+ live = authority.sessions[ref.session_id]
+ authority.db.end_session(ref.session_id, 'ws_orphan_reap')
+ # The empty replacement row a pre-fix self-heal left behind on the same route.
+ authority.db.create_session('stray-replacement', 'local', session_key=live.route,
+ chat_id=live.source.chat_id, chat_type='dm', user_id=live.source.user_id)
+ with store._lock:
+ entry = store._entries[live.route]
+ verdict = store._stale_entry_verdict(live.route, entry, authority.db.get_session(ref.session_id))
+ assert verdict is None, verdict
+ assert store._entries[live.route].session_id == ref.session_id
+ finally:
+ await connection.close()
+ authority.db.close()
+
+
+async def test_resume_heals_a_route_repointed_at_an_empty_stray_row(tmp_path, monkeypatch):
+ """Installs already wedged by the old repoint: the route names an empty row outside the
+ receipt's lineage. The authority's receipt is the truth; the stray entry is dropped."""
+ store, authority, connection, ref = await _local_session(tmp_path, monkeypatch)
+ try:
+ live = authority.sessions.pop(ref.session_id)
+ authority.db.create_session('stray-replacement', 'local', session_key=live.route,
+ chat_id=live.source.chat_id, chat_type='dm', user_id=live.source.user_id)
+ with store._lock:
+ store._entries[live.route].session_id = 'stray-replacement'
+
+ reply = await connection.dispatch({'id': 1, 'method': 'session.resume',
+ 'params': {'session_id': ref.session_id}})
+
+ assert reply['result']['messages'][0]['content'] == 'earlier turn', reply
+ assert store._entries[live.route].session_id == ref.session_id
+ finally:
+ await connection.close()
+ authority.db.close()
diff --git a/tests/tui_gateway/test_reap_respects_canonical_ledger.py b/tests/tui_gateway/test_reap_respects_canonical_ledger.py
new file mode 100644
index 00000000000..e6ebdd88de9
--- /dev/null
+++ b/tests/tui_gateway/test_reap_respects_canonical_ledger.py
@@ -0,0 +1,73 @@
+"""A viewer's reap never stamps a session ended while the gateway's ledger holds its turn (#140642).
+
+A remote viewer of a Desktop session disconnected mid-turn; its process's ws-orphan reaper ran
+``_finalize_session`` and stamped the row ``ws_orphan_reap`` while the owning gateway was still
+running the admitted turn. The owner then read its own live route as stale (#54878).
+"""
+import threading
+from unittest.mock import MagicMock
+
+import pytest
+
+from hermes_state import SessionDB
+from hermes_state_runtime import admit_session_input, begin_runtime_epoch, claim_session_input
+from tui_gateway.server import _teardown_session
+
+
+def _session(tmp_path, monkeypatch, source, session_id):
+ from run_agent import AIAgent
+ home = tmp_path / 'home'
+ home.mkdir()
+ monkeypatch.setenv('HERMES_HOME', str(home))
+ db = SessionDB(db_path=home / 'state.db')
+ db.create_session(session_id, source=source)
+ agent = AIAgent.__new__(AIAgent)
+ agent.session_id = session_id
+ agent._session_db = db
+ agent._owns_session_db = True
+ agent._end_session_on_close = True
+ agent._session_messages = []
+ agent._active_children_lock = threading.Lock()
+ agent._active_children = []
+ agent._memory_manager = None
+ session = {'agent': agent, 'history': [{'role': 'user', 'content': 'x'}], 'history_lock': None,
+ 'session_key': session_id, 'profile_home': str(home), 'source': source}
+ return home / 'state.db', db, session, agent
+
+
[email protected]('claimed', [True, False], ids=['started', 'queued'])
[email protected]('source', ['gui', 'tui'])
+def test_reap_leaves_row_open_while_the_ledger_holds_its_turn(tmp_path, monkeypatch, source, claimed):
+ sid = f'{source}-live'
+ path, db, session, agent = _session(tmp_path, monkeypatch, source, sid)
+ epoch = begin_runtime_epoch(db, instance_id='gateway')
+ admit_session_input(db, epoch=epoch, principal_id='owner', session_id=sid, request_id='r1',
+ payload={'text': 'long turn'})
+ if claimed:
+ claim_session_input(db, epoch=epoch, session_id=sid)
+
+ _teardown_session(session, end_reason='ws_orphan_reap')
+
+ check = SessionDB(db_path=path)
+ try:
+ row = check.get_session(sid)
+ assert (row['ended_at'], row['end_reason']) == (None, None)
+ assert agent._end_session_on_close is False # agent.close() must not end it either
+ finally:
+ check.close()
+
+
+def test_reap_still_ends_an_idle_row(tmp_path, monkeypatch):
+ path, db, session, _agent = _session(tmp_path, monkeypatch, 'tui', 'tui-idle')
+ epoch = begin_runtime_epoch(db, instance_id='gateway')
+ db.create_session('other', source='tui')
+ admit_session_input(db, epoch=epoch, principal_id='owner', session_id='other', request_id='r1',
+ payload={'text': 'another session'})
+
+ _teardown_session(session, end_reason='ws_orphan_reap')
+
+ check = SessionDB(db_path=path)
+ try:
+ assert check.get_session('tui-idle')['end_reason'] == 'ws_orphan_reap'
+ finally:
+ check.close()
diff --git a/tui_gateway/session_lifecycle.py b/tui_gateway/session_lifecycle.py
index 9f6c2110e63..853bd516612 100644
--- a/tui_gateway/session_lifecycle.py
+++ b/tui_gateway/session_lifecycle.py
@@ -143,6 +143,37 @@ _CANONICAL_PENDING_SQL = """SELECT status FROM session_admissions WHERE target_s
UNION ALL SELECT status FROM worker_executions WHERE session_id=? AND status!='terminal' LIMIT 1"""
+def _end_unless_ledger_busy(db, session_id: str, session_key, end_reason: str) -> bool:
+ """End *session_id*'s row unless the gateway's durable ledger still holds work for it; True when
+ the row may end (ended now, or already ended). The check and the stamp share one write txn.
+
+ A viewer process (a dashboard, a second Desktop window) runs this reaper for a session the
+ owning gateway is still executing; stamping it ended mid-turn made the owner treat its own live
+ route as stale (#54878) and replace the conversation (#140642). The ledger names a local session
+ by its logical id (the row's ``chat_id``), so the physical id, the key and that id are checked."""
+ execute_write = getattr(db, "_execute_write", None)
+ if execute_write is None:
+ db.end_session(session_id, end_reason)
+ return True
+
+ def write(conn):
+ row = conn.execute("SELECT chat_id FROM sessions WHERE id=?", (session_id,)).fetchone()
+ ids = {session_id, str(session_key or "")}
+ if row is not None and str(row[0] or "").startswith("local-"):
+ ids.add(str(row[0]))
+ for sid in filter(None, ids):
+ if conn.execute(_CANONICAL_PENDING_SQL, (sid, sid)).fetchone() is not None:
+ return False
+ db._end_and_bump(conn, "UPDATE sessions SET ended_at = ?, end_reason = ? WHERE id = ? AND ended_at IS NULL",
+ (time.time(), end_reason, session_id), session_id, end_reason)
+ return True
+
+ if execute_write(write):
+ return True
+ logger.info("Preserving session %s during %s: the gateway's ledger still holds its turn", session_id, end_reason)
+ return False
+
+
def _canonical_ledger_refusal(session: dict):
"""Refusal when the gateway's durable ledger still holds work for this stored session, else None.
@@ -461,7 +492,9 @@ def _finalize_session(session: dict | None, end_reason: str = "tui_close") -> No
# Automatic Desktop cleanup (ws_orphan_reap, idle_timeout, etc.) reclaims
# runtime but must not end the durable row — the conversation stays open
# and resumable until the user explicitly closes or archives it. #105588
- db.end_session(session_id, end_reason)
+ # A turn the gateway's ledger still holds keeps the row open too (#140642).
+ if not _end_unless_ledger_busy(db, session_id, session_key, end_reason):
+ _tui_owns_lifecycle = False
# ``_teardown_session`` follows with ``agent.close()``, whose ``_finalize_owned_session_row`` ends the row as
# ``agent_close`` through the agent's own handle — every spare decision above would be undone one call later.
if agent is not None and (_desktop_automatic_cleanup or not _tui_owns_lifecycle):