Skip to content

The agent that grows with you

fix(gateway): retarget a Bot Chat's queued DMs at the deferred reset's fresh transcript

Tekniumbc3a6ad7cfda
diff --git a/gateway/session_automation.py b/gateway/session_automation.py
index 47e277627ca..946ada3ca3f 100644
--- a/gateway/session_automation.py
+++ b/gateway/session_automation.py
@@ -533,6 +533,40 @@ async def move_route_wakes(runner, session_key, old_session_id):
ordered=True, after=partial(authority._schedule_admitted, new))
+def _local_owner(authority, session_key):
+ """The LOCAL FIFO serving ``session_key`` (its owner is the receipt id, which a reset keeps)."""
+ for sid, live in list(authority.sessions.items()):
+ if live.route == session_key and getattr(live.source, 'platform', None) == Platform.LOCAL:
+ return SessionRef(authority.profile_id, sid)
+ return None
+
+
+async def _reset_and_retarget_local(authority, runner, session_key, old_session_id, owner):
+ """A LOCAL route (the Bot Chat ``hermes peer dm`` lands in) keeps its FIFO owner across the
+ reset, so nothing moves between FIFOs. Its queued automation rows (a DM that arrived while the
+ exhausted turn ran) still name the pre-reset transcript and are retargeted at the fresh one,
+ in the same ordered worker step as the rotation; left stale, the drain refuses them
+ (``admission_conflict``) and the Bot Chat's FIFO pauses for good, across restarts."""
+ from gateway.session_runtime_workers import tracked_write
+ from hermes_state_runtime import retarget_local_automation
+ live = authority.sessions[owner.session_id]
+
+ def rotate_and_retarget():
+ old_targets = authority.db.get_compression_lineage(old_session_id) or [old_session_id]
+ entry = runner.session_store.reset_session(session_key)
+ if entry is None or entry.session_id in old_targets:
+ return entry, []
+ return entry, retarget_local_automation(
+ authority.db, epoch=authority.epoch, session_id=owner.session_id,
+ principal_id='automation:' + session_key, old_targets=old_targets, new_target=entry.session_id)
+
+ async with live.mutation_lock:
+ entry, moved = await tracked_write(authority, rotate_and_retarget, ordered=True)
+ if moved:
+ authority._publish_pending(owner)
+ return entry
+
+
async def reset_and_rehome_messages(runner, session_key, old_session_id):
"""The compression-exhaustion auto-reset: rotate the route to a fresh session and move the old
owner's still-queued native ``messaging:*`` rows onto it, same order and request ids. Left
@@ -552,6 +586,9 @@ async def reset_and_rehome_messages(runner, session_key, old_session_id):
authority = active_authority(runner)
old = SessionRef(authority.profile_id, authority.logical_owner(old_session_id)) if authority else None
live = authority.sessions.get(old.session_id) if old and old_session_id else None
+ local = _local_owner(authority, session_key) if authority else None
+ if local is not None:
+ return await _reset_and_retarget_local(authority, runner, session_key, old_session_id, local), lambda: None
if live is None:
return await runner.async_session_store.reset_session(session_key), lambda: None
import asyncio
diff --git a/hermes_state_runtime.py b/hermes_state_runtime.py
index ebc56f2e526..c55311acb54 100644
--- a/hermes_state_runtime.py
+++ b/hermes_state_runtime.py
@@ -397,6 +397,43 @@ def rehome_queued_inputs(db, *, epoch: int, old_session_id: str, new_session_id:
return db._execute_write(write)
+def retarget_local_automation(db, *, epoch: int, session_id: str, principal_id: str,
+ old_targets, new_target: str) -> list[str]:
+ """Point a LOCAL owner's still-queued ``principal_id`` automation rows (a Bot Chat's queued
+ ``hermes peer dm`` turns, plugin and completion notices) that name a pre-reset transcript in
+ ``old_targets`` at ``new_target``, in one transaction (the compression-exhaustion auto-reset).
+ A LOCAL FIFO's owner is its receipt id and survives the reset; only the descriptor's ``target``
+ goes stale, and the drain refuses a stale one (``admission_conflict``), pausing the FIFO for
+ good. Admission id, request id and seq are kept; the digest is re-keyed to the new payload.
+ Nothing moves behind an ``unknown`` head. Returns the retargeted admission ids in FIFO order."""
+ for value in (session_id, principal_id, new_target, *old_targets):
+ _text(value)
+ old_targets = set(old_targets)
+
+ def write(conn):
+ _epoch(conn, epoch)
+ _session(conn, new_target)
+ if conn.execute("SELECT 1 FROM session_admissions WHERE target_session_id=? AND status='unknown'",
+ (session_id,)).fetchone():
+ return []
+ moved = []
+ for row in conn.execute("SELECT * FROM session_admissions WHERE target_session_id=? AND status='queued' "
+ "AND principal_id=? ORDER BY seq", (session_id, principal_id)).fetchall():
+ payload = json.loads(row['payload_json'])
+ descriptor = payload.get('local_automation_v1') if isinstance(payload, dict) else None
+ if not isinstance(descriptor, dict) or descriptor.get('target') not in old_targets:
+ continue
+ descriptor['target'] = new_target
+ digest = admission_fingerprint(canonical_target=session_id, payload={'input': payload, 'intent': row['intent']})
+ conn.execute("UPDATE session_admissions SET payload_json=?,payload_digest=? WHERE admission_id=? AND status='queued'",
+ (_json(payload), digest, row['admission_id']))
+ moved.append(row['admission_id'])
+ if moved:
+ conn.execute('UPDATE sessions SET runtime_revision=runtime_revision+1 WHERE id=?', (session_id,))
+ return moved
+ return db._execute_write(write)
+
+
def recover_session_inputs(db, *, epoch: int) -> int:
"""Never replay started work. Live worker adoption is a separate explicit operation."""
def write(conn):
diff --git a/tests/gateway/test_peer_dm_bot_chat_context_overflow.py b/tests/gateway/test_peer_dm_bot_chat_context_overflow.py
index 2d75aec6e09..513f2e618ca 100644
--- a/tests/gateway/test_peer_dm_bot_chat_context_overflow.py
+++ b/tests/gateway/test_peer_dm_bot_chat_context_overflow.py
@@ -155,3 +155,94 @@ def test_peer_dm_overflow_resets_the_local_bot_chat_and_answers_the_follow_up(ov
assert run.follow[0] == 200, (run.follow, run.rows)
assert run.follow[1]['message']['content'].startswith('ACK short follow-up'), run.follow
assert _stored(run, 'short follow-up') == 1, run.rows
+
+
+def _admissions(home):
+ with closing(sqlite3.connect(f'file:{home / "state.db"}?mode=ro', uri=True)) as db:
+ return db.execute('SELECT request_id, status FROM session_admissions ORDER BY rowid').fetchall()
+
+
[email protected](scope='module')
+def racing_run(tmp_path_factory):
+ """A second DM lands in the Bot Chat while the oversized one's turn still runs, then a third DM.
+
+ The deferred reset rotates the route after the oversized turn settles. The racing DM is already
+ queued (a ``local_automation_v1`` row naming the pre-reset target): it must run on the fresh
+ session, not pause the Bot Chat's FIFO (``admission_conflict``) forever."""
+ tmp_path = tmp_path_factory.mktemp('lc23race')
+ root = Path(__file__).resolve().parents[2]
+ home, user = tmp_path / 'state', tmp_path / 'user'
+ home.mkdir(mode=0o700)
+ user.mkdir()
+ model = ThreadingHTTPServer(('127.0.0.1', 0), WindowCappedModel)
+ model.rejected = 0
+ threading.Thread(target=model.serve_forever, daemon=True).start()
+ url = f'http://127.0.0.1:{model.server_port}/v1'
+ api_port = _free_port()
+ (home / 'config.yaml').write_text(json.dumps({
+ 'gateway': {'multiplex_profiles': False},
+ 'model': {'provider': 'custom', 'default': 'fake-model', 'base_url': url, 'context_length': 65536},
+ 'agent': {'api_max_retries': 1},
+ 'approvals': {'mode': 'off'},
+ 'compression': {'protect_last_n': 2},
+ 'memory': {'memory_enabled': False, 'user_profile_enabled': False},
+ 'auxiliary': {'title_generation': {'enabled': False},
+ 'compression': {'provider': 'custom', 'model': 'fake-model', 'base_url': url}},
+ 'platforms': {'api_server': {'enabled': True, 'extra': {'host': '127.0.0.1', 'port': api_port, 'key': KEY}}},
+ 'platform_toolsets': {'cli': [], 'api_server': []}}))
+ env = child_env() | dict(HOME=str(user), USERPROFILE=str(user), HERMES_HOME=str(home), PYTHONPATH=str(root),
+ OPENAI_API_KEY='loopback-only', PYTHONUNBUFFERED='1')
+ big = 'MKBIG ' + 'lorem ipsum dolor sit amet ' * 12000
+ results = {}
+ try:
+ with daemon(root, home, env, barrier=False):
+ status, created = _post(api_port, '/api/sessions', {'title': 'Bot Chat', 'source': 'bot_peer_dm'})
+ assert status == 201, created
+ sid = created['session']['id']
+
+ def send(name, text, delay):
+ threading.Event().wait(delay)
+ try:
+ results[name] = _post(api_port, f'/api/sessions/{sid}/chat', {'message': text}, timeout=45)
+ except OSError as exc: # a wedged FIFO never answers
+ results[name] = (repr(exc), {})
+ threads = [threading.Thread(target=send, args=('big', big, 0)),
+ threading.Thread(target=send, args=('race', 'racing short DM MKRACE', 0.4))]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+ send('third', 'third DM MKTHIRD', 0)
+ admissions = _admissions(home)
+ finally:
+ model.shutdown()
+ log = (home / 'restart.log').read_text(errors='replace')
+ for name in ('gateway.log', 'errors.log'):
+ if (home / 'logs' / name).exists():
+ log += (home / 'logs' / name).read_text(errors='replace')
+ with closing(sqlite3.connect(f'file:{home / "state.db"}?mode=ro', uri=True)) as db:
+ rows = db.execute("SELECT session_id, role, substr(coalesce(content,''),1,24) FROM messages ORDER BY id").fetchall()
+ bot_chat = db.execute('SELECT end_reason FROM sessions WHERE id=?', (sid,)).fetchone()
+ return SimpleNamespace(sid=sid, results=results, admissions=admissions, log=log, rows=rows,
+ rejected=model.rejected, bot_chat_end=bot_chat[0])
+
+
+def test_a_dm_queued_behind_the_overflowing_turn_runs_on_the_fresh_bot_chat(racing_run):
+ run = racing_run
+ assert run.rejected >= 1, 'the first DM never overflowed'
+ assert run.results['big'][0] == 502, run.results['big']
+ assert run.bot_chat_end == 'session_reset', run.rows
+ assert 'admission_conflict' not in run.log, [l for l in run.log.splitlines() if 'admission_conflict' in l][:5]
+ # The racing DM and the later one are both answered, on the fresh session, each stored there once.
+ # (A racing attempt that ran on the old transcript before the reset was owed failed there and kept
+ # its user row, like any failed turn; that row is the old conversation's, not a re-run.)
+ fresh = {s for s, role, text in run.rows if role == 'user' and text.startswith('third DM')}
+ assert len(fresh) == 1 and run.sid not in fresh, run.rows
+ for name, marker in (('race', 'racing short DM'), ('third', 'third DM')):
+ status, body = run.results[name]
+ assert status == 200, (name, run.results[name], run.admissions)
+ assert body['message']['content'].startswith('ACK ' + marker), (name, body)
+ on_fresh = [1 for s, role, text in run.rows if role == 'user' and text.startswith(marker) and s in fresh]
+ assert len(on_fresh) == 1, (name, run.rows)
+ assert not [row for row in run.admissions if row[1] == 'queued'], run.admissions