The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): retarget a Bot Chat's queued DMs at the deferred reset's fresh transcript
Tekniumbc3a6ad7cfda
diff --git a/gateway/session_automation.py b/gateway/session_automation.pyindex 47e277627ca..946ada3ca3f 100644--- a/gateway/session_automation.py+++ b/gateway/session_automation.py@@ -533,6 +533,40 @@ async def move_route_wakes(runner, session_key, old_session_id):ordered=True, after=partial(authority._schedule_admitted, new))+def _local_owner(authority, session_key):+ """The LOCAL FIFO serving ``session_key`` (its owner is the receipt id, which a reset keeps)."""+ for sid, live in list(authority.sessions.items()):+ if live.route == session_key and getattr(live.source, 'platform', None) == Platform.LOCAL:+ return SessionRef(authority.profile_id, sid)+ return None+++async def _reset_and_retarget_local(authority, runner, session_key, old_session_id, owner):+ """A LOCAL route (the Bot Chat ``hermes peer dm`` lands in) keeps its FIFO owner across the+ reset, so nothing moves between FIFOs. Its queued automation rows (a DM that arrived while the+ exhausted turn ran) still name the pre-reset transcript and are retargeted at the fresh one,+ in the same ordered worker step as the rotation; left stale, the drain refuses them+ (``admission_conflict``) and the Bot Chat's FIFO pauses for good, across restarts."""+ from gateway.session_runtime_workers import tracked_write+ from hermes_state_runtime import retarget_local_automation+ live = authority.sessions[owner.session_id]++ def rotate_and_retarget():+ old_targets = authority.db.get_compression_lineage(old_session_id) or [old_session_id]+ entry = runner.session_store.reset_session(session_key)+ if entry is None or entry.session_id in old_targets:+ return entry, []+ return entry, retarget_local_automation(+ authority.db, epoch=authority.epoch, session_id=owner.session_id,+ principal_id='automation:' + session_key, old_targets=old_targets, new_target=entry.session_id)++ async with live.mutation_lock:+ entry, moved = await tracked_write(authority, rotate_and_retarget, ordered=True)+ if moved:+ authority._publish_pending(owner)+ return entry++async def reset_and_rehome_messages(runner, session_key, old_session_id):"""The compression-exhaustion auto-reset: rotate the route to a fresh session and move the oldowner's still-queued native ``messaging:*`` rows onto it, same order and request ids. Left@@ -552,6 +586,9 @@ async def reset_and_rehome_messages(runner, session_key, old_session_id):authority = active_authority(runner)old = SessionRef(authority.profile_id, authority.logical_owner(old_session_id)) if authority else Nonelive = authority.sessions.get(old.session_id) if old and old_session_id else None+ local = _local_owner(authority, session_key) if authority else None+ if local is not None:+ return await _reset_and_retarget_local(authority, runner, session_key, old_session_id, local), lambda: Noneif live is None:return await runner.async_session_store.reset_session(session_key), lambda: Noneimport asynciodiff --git a/hermes_state_runtime.py b/hermes_state_runtime.pyindex ebc56f2e526..c55311acb54 100644--- a/hermes_state_runtime.py+++ b/hermes_state_runtime.py@@ -397,6 +397,43 @@ def rehome_queued_inputs(db, *, epoch: int, old_session_id: str, new_session_id:return db._execute_write(write)+def retarget_local_automation(db, *, epoch: int, session_id: str, principal_id: str,+ old_targets, new_target: str) -> list[str]:+ """Point a LOCAL owner's still-queued ``principal_id`` automation rows (a Bot Chat's queued+ ``hermes peer dm`` turns, plugin and completion notices) that name a pre-reset transcript in+ ``old_targets`` at ``new_target``, in one transaction (the compression-exhaustion auto-reset).+ A LOCAL FIFO's owner is its receipt id and survives the reset; only the descriptor's ``target``+ goes stale, and the drain refuses a stale one (``admission_conflict``), pausing the FIFO for+ good. Admission id, request id and seq are kept; the digest is re-keyed to the new payload.+ Nothing moves behind an ``unknown`` head. Returns the retargeted admission ids in FIFO order."""+ for value in (session_id, principal_id, new_target, *old_targets):+ _text(value)+ old_targets = set(old_targets)++ def write(conn):+ _epoch(conn, epoch)+ _session(conn, new_target)+ if conn.execute("SELECT 1 FROM session_admissions WHERE target_session_id=? AND status='unknown'",+ (session_id,)).fetchone():+ return []+ moved = []+ for row in conn.execute("SELECT * FROM session_admissions WHERE target_session_id=? AND status='queued' "+ "AND principal_id=? ORDER BY seq", (session_id, principal_id)).fetchall():+ payload = json.loads(row['payload_json'])+ descriptor = payload.get('local_automation_v1') if isinstance(payload, dict) else None+ if not isinstance(descriptor, dict) or descriptor.get('target') not in old_targets:+ continue+ descriptor['target'] = new_target+ digest = admission_fingerprint(canonical_target=session_id, payload={'input': payload, 'intent': row['intent']})+ conn.execute("UPDATE session_admissions SET payload_json=?,payload_digest=? WHERE admission_id=? AND status='queued'",+ (_json(payload), digest, row['admission_id']))+ moved.append(row['admission_id'])+ if moved:+ conn.execute('UPDATE sessions SET runtime_revision=runtime_revision+1 WHERE id=?', (session_id,))+ return moved+ return db._execute_write(write)++def recover_session_inputs(db, *, epoch: int) -> int:"""Never replay started work. Live worker adoption is a separate explicit operation."""def write(conn):diff --git a/tests/gateway/test_peer_dm_bot_chat_context_overflow.py b/tests/gateway/test_peer_dm_bot_chat_context_overflow.pyindex 2d75aec6e09..513f2e618ca 100644--- a/tests/gateway/test_peer_dm_bot_chat_context_overflow.py+++ b/tests/gateway/test_peer_dm_bot_chat_context_overflow.py@@ -155,3 +155,94 @@ def test_peer_dm_overflow_resets_the_local_bot_chat_and_answers_the_follow_up(ovassert run.follow[0] == 200, (run.follow, run.rows)assert run.follow[1]['message']['content'].startswith('ACK short follow-up'), run.followassert _stored(run, 'short follow-up') == 1, run.rows+++def _admissions(home):+ with closing(sqlite3.connect(f'file:{home / "state.db"}?mode=ro', uri=True)) as db:+ return db.execute('SELECT request_id, status FROM session_admissions ORDER BY rowid').fetchall()++[email protected](scope='module')+def racing_run(tmp_path_factory):+ """A second DM lands in the Bot Chat while the oversized one's turn still runs, then a third DM.++ The deferred reset rotates the route after the oversized turn settles. The racing DM is already+ queued (a ``local_automation_v1`` row naming the pre-reset target): it must run on the fresh+ session, not pause the Bot Chat's FIFO (``admission_conflict``) forever."""+ tmp_path = tmp_path_factory.mktemp('lc23race')+ root = Path(__file__).resolve().parents[2]+ home, user = tmp_path / 'state', tmp_path / 'user'+ home.mkdir(mode=0o700)+ user.mkdir()+ model = ThreadingHTTPServer(('127.0.0.1', 0), WindowCappedModel)+ model.rejected = 0+ threading.Thread(target=model.serve_forever, daemon=True).start()+ url = f'http://127.0.0.1:{model.server_port}/v1'+ api_port = _free_port()+ (home / 'config.yaml').write_text(json.dumps({+ 'gateway': {'multiplex_profiles': False},+ 'model': {'provider': 'custom', 'default': 'fake-model', 'base_url': url, 'context_length': 65536},+ 'agent': {'api_max_retries': 1},+ 'approvals': {'mode': 'off'},+ 'compression': {'protect_last_n': 2},+ 'memory': {'memory_enabled': False, 'user_profile_enabled': False},+ 'auxiliary': {'title_generation': {'enabled': False},+ 'compression': {'provider': 'custom', 'model': 'fake-model', 'base_url': url}},+ 'platforms': {'api_server': {'enabled': True, 'extra': {'host': '127.0.0.1', 'port': api_port, 'key': KEY}}},+ 'platform_toolsets': {'cli': [], 'api_server': []}}))+ env = child_env() | dict(HOME=str(user), USERPROFILE=str(user), HERMES_HOME=str(home), PYTHONPATH=str(root),+ OPENAI_API_KEY='loopback-only', PYTHONUNBUFFERED='1')+ big = 'MKBIG ' + 'lorem ipsum dolor sit amet ' * 12000+ results = {}+ try:+ with daemon(root, home, env, barrier=False):+ status, created = _post(api_port, '/api/sessions', {'title': 'Bot Chat', 'source': 'bot_peer_dm'})+ assert status == 201, created+ sid = created['session']['id']++ def send(name, text, delay):+ threading.Event().wait(delay)+ try:+ results[name] = _post(api_port, f'/api/sessions/{sid}/chat', {'message': text}, timeout=45)+ except OSError as exc: # a wedged FIFO never answers+ results[name] = (repr(exc), {})+ threads = [threading.Thread(target=send, args=('big', big, 0)),+ threading.Thread(target=send, args=('race', 'racing short DM MKRACE', 0.4))]+ for thread in threads:+ thread.start()+ for thread in threads:+ thread.join()+ send('third', 'third DM MKTHIRD', 0)+ admissions = _admissions(home)+ finally:+ model.shutdown()+ log = (home / 'restart.log').read_text(errors='replace')+ for name in ('gateway.log', 'errors.log'):+ if (home / 'logs' / name).exists():+ log += (home / 'logs' / name).read_text(errors='replace')+ with closing(sqlite3.connect(f'file:{home / "state.db"}?mode=ro', uri=True)) as db:+ rows = db.execute("SELECT session_id, role, substr(coalesce(content,''),1,24) FROM messages ORDER BY id").fetchall()+ bot_chat = db.execute('SELECT end_reason FROM sessions WHERE id=?', (sid,)).fetchone()+ return SimpleNamespace(sid=sid, results=results, admissions=admissions, log=log, rows=rows,+ rejected=model.rejected, bot_chat_end=bot_chat[0])++[email protected]('linux')+def test_a_dm_queued_behind_the_overflowing_turn_runs_on_the_fresh_bot_chat(racing_run):+ run = racing_run+ assert run.rejected >= 1, 'the first DM never overflowed'+ assert run.results['big'][0] == 502, run.results['big']+ assert run.bot_chat_end == 'session_reset', run.rows+ assert 'admission_conflict' not in run.log, [l for l in run.log.splitlines() if 'admission_conflict' in l][:5]+ # The racing DM and the later one are both answered, on the fresh session, each stored there once.+ # (A racing attempt that ran on the old transcript before the reset was owed failed there and kept+ # its user row, like any failed turn; that row is the old conversation's, not a re-run.)+ fresh = {s for s, role, text in run.rows if role == 'user' and text.startswith('third DM')}+ assert len(fresh) == 1 and run.sid not in fresh, run.rows+ for name, marker in (('race', 'racing short DM'), ('third', 'third DM')):+ status, body = run.results[name]+ assert status == 200, (name, run.results[name], run.admissions)+ assert body['message']['content'].startswith('ACK ' + marker), (name, body)+ on_fresh = [1 for s, role, text in run.rows if role == 'user' and text.startswith(marker) and s in fresh]+ assert len(on_fresh) == 1, (name, run.rows)+ assert not [row for row in run.admissions if row[1] == 'queued'], run.admissions