The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): commit a local receipt's compression catch-up off the owner loop
unsupportedpastels146354440098
diff --git a/gateway/session_authority.py b/gateway/session_authority.pyindex a1f441a89f2..4a41f0c4b26 100644--- a/gateway/session_authority.py+++ b/gateway/session_authority.py@@ -1178,6 +1178,10 @@ async def initialize_session_authority(runner, *, profile_id, instance_id, db=Noawait asyncio.to_thread(recover_session_inputs, db, epoch=epoch)await asyncio.to_thread(backfill_legacy_session_owners, db)authority = SessionAuthority(runner, profile_id=profile_id, instance_id=instance_id, db=db, epoch=epoch)+ # A receipt's compression catch-up (#140060) writes: it commits off the loop too, before the+ # authority is published, so the recovery below only reads.+ from gateway.session_local_adoption import adopt_local_compressions+ await adopt_local_compressions(authority)if register:runner.session_authority = authorityfrom gateway.session_cron import bind_ownerdiff --git a/gateway/session_controls.py b/gateway/session_controls.pyindex c437a5e7b34..f54a8a91431 100644--- a/gateway/session_controls.py+++ b/gateway/session_controls.py@@ -243,6 +243,14 @@ class AuthorityConnection:'code': 4001, 'message': 'invalid_params', 'data': {'reason': 'invalid_params', 'fields': problems}}}original_params = dict(params)contract = contract or METHODS.get(method)+ authority = self.authority+ if (isinstance(ref.session_id, str) and ref.session_id and not getattr(authority, 'retiring', False)+ and ref.profile_id == getattr(authority, 'profile_id', None)+ and ref.session_id not in getattr(authority, 'sessions', {ref.session_id: None})):+ # A first read of a session recovery never restored: its compression catch-up (#140060)+ # commits in a worker here, so the handler's synchronous restore only reads.+ from gateway.session_local_adoption import adopt_before_restore+ await adopt_before_restore(authority, ref.session_id)handlers = self.handlers()try:from gateway.session_group_controls import GROUP_METHODS, dispatch_group_controldiff --git a/gateway/session_local_adoption.py b/gateway/session_local_adoption.pynew file mode 100644index 00000000000..8a6bdd63989--- /dev/null+++ b/gateway/session_local_adoption.py@@ -0,0 +1,87 @@+"""A local receipt's compression catch-up (#140060), committed off the owner loop.++``adopt_untracked_compression`` takes the writer lock, up to ``_WRITE_PATIENCE_S`` when contended. On+the owner loop that stalls every profile it serves (streams, controls, admissions), including during+a hot-profile activation. So the async paths that restore a receipt (boot and hot-profile recovery, a+socket's first read) commit the catch-up in a worker first, and the restore that follows only reads.+A restore reached synchronously on the loop hands it to a tracked worker (``schedule_adoption``).+"""+import asyncio+import logging+from functools import partial++from hermes_state_runtime import RuntimeStoreError++logger = logging.getLogger(__name__)+++def on_owner_loop():+ try:+ asyncio.get_running_loop()+ except RuntimeError:+ return False+ return True+++def _due(authority, sid):+ """*sid*'s stored receipt when a catch-up would commit for it on this owner, else None. Read-only."""+ from gateway.session_local_recovery import owned_receipt+ from hermes_state_local_lineage import adoptable_compression+ try:+ receipt = owned_receipt(authority, sid)[0]+ with authority.db._read_ctx() as conn:+ return receipt if adoptable_compression(conn, receipt) is not None else None+ except (KeyError, TypeError, ValueError, AttributeError):+ return None # the restore refuses this receipt and logs the check that failed+++def _adopt(authority, sid, receipt):+ from hermes_state_local_lineage import adopt_untracked_compression+ adopt_untracked_compression(authority.db, epoch=authority.epoch, session_id=sid, expected=receipt)+++async def adopt_before_restore(authority, sid, *, tracked=True):+ """Commit *sid*'s due catch-up in a worker, so the restore that follows only reads.++ ``tracked=False``: an authority still being built, not yet published for retirement to join."""+ receipt = _due(authority, sid)+ if receipt is None:+ return+ write = partial(_adopt, authority, sid, receipt)+ try:+ if tracked:+ from gateway.session_runtime_workers import tracked_write+ await tracked_write(authority, write)+ else:+ await asyncio.to_thread(write)+ except RuntimeStoreError as exc: # the restore that follows refuses the receipt and logs it+ logger.warning('Local session %s compression catch-up failed: %s', sid, exc.reason)+++async def adopt_local_compressions(authority):+ """Boot and hot-profile recovery: every receipt's due catch-up, before ``recover_local_sessions``."""+ from gateway.session_local_recovery import local_receipt_ids+ for sid in local_receipt_ids(authority):+ await adopt_before_restore(authority, sid, tracked=False)+++def schedule_adoption(authority, sid, receipt):+ """A sync restore on the owner loop found *receipt*'s catch-up due: commit it in a tracked worker,+ then publish the session on the loop. One worker per session at a time."""+ from gateway.session_runtime_workers import track_mutation+ pending = getattr(authority, '_local_adoptions', None)+ if pending is None:+ pending = authority._local_adoptions = {}+ if sid in pending and not pending[sid].done():+ return+ logger.info('Local session %s: committing its compression catch-up off the owner loop', sid)++ async def adopt_then_publish():+ try:+ await asyncio.to_thread(_adopt, authority, sid, receipt)+ if not getattr(authority, 'retiring', False):+ from gateway.session_local_recovery import restore_local_session+ restore_local_session(authority, sid)+ except RuntimeStoreError as exc:+ logger.warning('Local session %s paused after its compression catch-up: %s', sid, exc.reason)+ pending[sid] = track_mutation(authority, adopt_then_publish())diff --git a/gateway/session_local_recovery.py b/gateway/session_local_recovery.pyindex db84a84cbb9..fbba6d473cb 100644--- a/gateway/session_local_recovery.py+++ b/gateway/session_local_recovery.py@@ -53,12 +53,26 @@ class _ReceiptRefused(RuntimeStoreError):"""A refused receipt ``restore_local_session`` already logged with its failing check."""+class _CatchUpPending(RuntimeStoreError):+ """The receipt's compression catch-up is committing in a worker; this restore is refused."""++def _with_untracked_compression(authority, sid, receipt):- """*receipt*, after recording compression steps an older build published past its tip (#140060)."""- from hermes_state_local_lineage import adopt_untracked_compression, untracked_compression_tip+ """*receipt*, after recording compression steps an older build published past its tip (#140060).++ The catch-up writes, so it never runs on the owner loop: a contended writer there would stall every+ profile the loop serves. The async restore paths commit it in a worker first+ (``session_local_adoption``); a sync restore on the loop that still finds it hands it to a tracked+ worker, which publishes the session once it commits, and refuses this read.+ """+ from hermes_state_local_lineage import adopt_untracked_compression, adoptable_compressionwith authority.db._read_ctx() as conn:- if untracked_compression_tip(conn, receipt) is None:+ if adoptable_compression(conn, receipt) is None:return receipt+ from gateway.session_local_adoption import on_owner_loop, schedule_adoption+ if on_owner_loop():+ schedule_adoption(authority, sid, receipt)+ raise _CatchUpPending('storage_unavailable')adopt_untracked_compression(authority.db, epoch=authority.epoch, session_id=sid, expected=receipt)return local_receipt(authority.db, sid)@@ -82,23 +96,28 @@ def _checked_entry(authority, receipt, source, route):return entry+def owned_receipt(authority, sid):+ """*sid*'s stored receipt with its LOCAL source, route and entry, once all are this owner's."""+ receipt = local_receipt(authority.db, sid)+ if receipt['profile_id'] != authority.profile_id:+ raise RuntimeStoreError('profile_mismatch')+ chat_id = (receipt['legacy_session_id'] if 'legacy_session_id' in receipt else+ local_identity(receipt['profile_id'], receipt['principal_id'], receipt['request_id']))+ if receipt['session_id'] != sid or sid != chat_id:+ raise ValueError('identity mismatch')+ source = local_source(authority, chat_id, receipt['principal_id'])+ route = authority.runner.session_store._generate_session_key(source)+ return receipt, source, route, _checked_entry(authority, receipt, source, route)++def restore_local_session(authority, sid):from gateway.session_authority import LiveSessionfrom gateway.session_local import LocalSessionAdapterfrom gateway.session_policy import restore_policytry:- receipt = local_receipt(authority.db, sid)- if receipt['profile_id'] != authority.profile_id:- raise RuntimeStoreError('profile_mismatch')- chat_id = (receipt['legacy_session_id'] if 'legacy_session_id' in receipt else- local_identity(receipt['profile_id'], receipt['principal_id'], receipt['request_id']))- if receipt['session_id'] != sid or sid != chat_id:- raise ValueError('identity mismatch')- source = local_source(authority, chat_id, receipt['principal_id'])- store = authority.runner.session_store- route = store._generate_session_key(source)# The catch-up writes, so the receipt as stored must already be this owner's (#140060).- entry = _checked_entry(authority, receipt, source, route)+ receipt, source, route, entry = owned_receipt(authority, sid)+ chat_id, store = source.chat_id, authority.runner.session_storeadopted = _with_untracked_compression(authority, sid, receipt)policy = restore_policy(adopted['policy'])if adopted is not receipt: # re-read after a catch-up: check the copy that will be served@@ -107,6 +126,8 @@ def restore_local_session(authority, sid):from hermes_state_local_lineage import validate_local_lineagewith authority.db._read_ctx() as conn:validate_local_lineage(conn, receipt)+ except _CatchUpPending:+ raiseexcept (KeyError, TypeError, ValueError, AttributeError) as exc:# The client only sees the bounded code: gateway.log names the check that refused (#140099).logging.getLogger(__name__).warning('Local session %s receipt refused: %s', sid, _refused_check(exc))@@ -207,18 +228,21 @@ def reset_local_session(store, old_entry, session_id, now, display_name):return entry-def recover_local_sessions(authority, *, schedule=False):+def local_receipt_ids(authority):with authority.db._read_ctx() as conn:- ids = [row[0][len(POLICY_PREFIX):] for row in conn.execute(+ return [row[0][len(POLICY_PREFIX):] for row in conn.execute('SELECT key FROM state_meta WHERE key LIKE ?', (POLICY_PREFIX + '%',))]- for sid in ids:+++def recover_local_sessions(authority, *, schedule=False):+ for sid in local_receipt_ids(authority):try:ref = restore_local_session(authority, sid)pending = list_session_admissions(authority.db, session_id=sid)if schedule and pending and not any(row['status'] == 'unknown' for row in pending):authority._schedule(ref)- except _ReceiptRefused:- continue # restore_local_session logged the check that refused it+ except (_ReceiptRefused, _CatchUpPending):+ continue # logged: the check that refused it, or the catch-up handed to a workerexcept RuntimeStoreError as exc:logging.getLogger(__name__).warning('Local session %s paused: %s', sid, exc.reason)if schedule:diff --git a/hermes_state_local_lineage.py b/hermes_state_local_lineage.pyindex 395d3d3c26e..93fc303d216 100644--- a/hermes_state_local_lineage.py+++ b/hermes_state_local_lineage.py@@ -119,11 +119,20 @@ def _untracked_steps(conn, tip, lineage):current = batch[-1]-def untracked_compression_tip(conn, receipt):- """The compression child of *receipt*'s tip, when the receipt does not record it yet."""- from hermes_state_compression import _CHAIN_STEP_SQL- step = conn.execute(_CHAIN_STEP_SQL, (receipt['entry']['session_id'],)).fetchone()- return step['id'] if step is not None else None+def adoptable_compression(conn, receipt):+ """``(steps, routed)``: the untracked compression steps *receipt* would adopt and its route's+ entry, else None. Read-only; ``adopt_untracked_compression`` repeats it inside its write."""+ tip = receipt['entry']['session_id']+ children = _untracked_steps(conn, tip, receipt.get('lineage', [receipt['session_id']]))+ if not children:+ return None+ identity = (receipt['route'], receipt['session_id'], receipt['principal_id'])+ for child_id in children:+ child = conn.execute('SELECT session_key,chat_id,user_id FROM sessions WHERE id=?', (child_id,)).fetchone()+ if tuple(child) != identity:+ return None+ routed = _routed_entry(conn, receipt['route'], tip)+ return (children, routed) if routed is not None else Nonedef adopt_untracked_compression(db, *, epoch, session_id, expected):@@ -144,19 +153,12 @@ def adopt_untracked_compression(db, *, epoch, session_id, expected):receipt = json.loads(saved[0])if receipt != expected:return- identity = (receipt['route'], receipt['session_id'], receipt['principal_id'])- lineage = list(receipt.get('lineage', [session_id]))- tip = receipt['entry']['session_id']- children = _untracked_steps(conn, tip, lineage)- for child_id in children or ():- child = conn.execute('SELECT session_key,chat_id,user_id FROM sessions WHERE id=?', (child_id,)).fetchone()- if tuple(child) != identity:- return- routed = _routed_entry(conn, receipt['route'], tip)- if not children or routed is None:+ adoptable = adoptable_compression(conn, receipt)+ if adoptable is None:return+ children, routed = adoptablereceipt['entry'] = dict(receipt['entry'], session_id=children[-1])- receipt['lineage'] = [*lineage, *children]+ receipt['lineage'] = [*receipt.get('lineage', [session_id]), *children]validate_local_lineage(conn, receipt)_commit_local_target(conn, receipt, dict(routed, session_id=children[-1]))db._execute_write(write)diff --git a/tests/gateway/test_local_receipt_untracked_compression.py b/tests/gateway/test_local_receipt_untracked_compression.pyindex a0e79e0d0c2..ed6a588475a 100644--- a/tests/gateway/test_local_receipt_untracked_compression.py+++ b/tests/gateway/test_local_receipt_untracked_compression.py@@ -58,6 +58,20 @@ def _compact_chain_like_0_22_2(db, parent_id, child_ids):db._execute_write(write)+def _unrecovered_owner(tmp_path, instance_id):+ """An owner of the same store whose boot recovery never reached the session: its first read restores."""+ from gateway.config import GatewayConfig+ from gateway.session import SessionStore+ from gateway.session_authority import SessionAuthority+ from hermes_state_runtime import begin_runtime_epoch+ store = SessionStore(tmp_path / 'sessions', GatewayConfig())+ runner = SimpleNamespace(session_store=store, _session_db=store._db, adapters={}, _draining=False,+ _cached_agent_for=lambda route: None, _adapter_for_source=lambda source: None)+ authority = SessionAuthority(runner, profile_id=str(tmp_path), instance_id=instance_id, db=store._db,+ epoch=begin_runtime_epoch(store._db, instance_id=instance_id))+ return authority, AuthorityConnection(authority, object(), {'user_id': 'owner'})++def _receipt(db, sid):from hermes_state_local import local_receiptreturn local_receipt(db, sid)@@ -278,6 +292,42 @@ async def test_a_chain_that_loops_back_past_one_walk_is_not_adopted(tmp_path, mosecond.db.close()+async def test_a_restore_on_the_owner_loop_hands_the_catch_up_to_a_worker(tmp_path, monkeypatch):+ """A restore reached synchronously on the owner loop (a bot, cron or A2A turn, a claim) never takes+ the writer lock there: a contended writer would stall every profile the loop serves. It hands the+ catch-up to a tracked worker and refuses this read; the worker's commit is published on the loop."""+ import asyncio+ import threading+ import hermes_state_local_lineage as lineage+ from gateway.session_local import create_local_session+ from gateway.session_local_recovery import restore_local_session+ from gateway.session_runtime_workers import mutation_tasks+ from hermes_state_runtime import RuntimeStoreError++ first, connection = await _owner(tmp_path, monkeypatch, 'first')+ ref = create_local_session(first, connection.actor, {'request_id': 'r', 'source': 'tui',+ 'cwd': str(tmp_path), 'model': 'm', 'toolsets': []})+ _compact_like_0_22_2(first.db, ref.session_id, 'classic-child')+ await connection.close()+ first.db.close()++ cold, connection = _unrecovered_owner(tmp_path, 'cold')+ threads, adopt = [], lineage.adopt_untracked_compression+ monkeypatch.setattr(lineage, 'adopt_untracked_compression',+ lambda *a, **k: threads.append(threading.get_ident()) or adopt(*a, **k))+ try:+ with pytest.raises(RuntimeStoreError, match='storage_unavailable'):+ restore_local_session(cold, ref.session_id)+ await asyncio.gather(*mutation_tasks(cold))+ assert threads and threading.get_ident() not in threads, 'the catch-up ran on the owner loop'+ assert _receipt(cold.db, ref.session_id)['entry']['session_id'] == 'classic-child'+ assert ref.session_id in cold.sessions, 'the committed catch-up was not published'+ finally:+ await connection.close()+ cold.db.close()++@pytest.mark.asyncioasync def test_a_restore_with_nothing_to_adopt_checks_the_stored_entry_once(tmp_path, monkeypatch):"""The catch-up re-checks the receipt only when it moved it. Every LOCAL turn claim restores, so adiff --git a/tests/gateway/test_owner_loop_writes_class.py b/tests/gateway/test_owner_loop_writes_class.pyindex fb0f58b87ba..85d4838d482 100644--- a/tests/gateway/test_owner_loop_writes_class.py+++ b/tests/gateway/test_owner_loop_writes_class.py@@ -122,6 +122,43 @@ async def _initialize(authority, owner, tmp_path, db):await initialize_session_authority(authority.runner, profile_id='owned', instance_id='second')+async def _initialize_adoption(authority, owner, tmp_path, db):+ """Boot and hot-profile activation (``serve_profile_runtime`` -> ``_build_profile_authority``)+ recover through ``initialize_session_authority`` while the loop serves the other profiles."""+ from hermes_state_local import local_receipt+ from tests.gateway.test_local_receipt_untracked_compression import _compact_like_0_22_2+ ref = _local(authority, owner, tmp_path)+ _compact_like_0_22_2(db, ref.session_id, 'classic-child')+ await _initialize(authority, owner, tmp_path, db)+ assert local_receipt(db, ref.session_id)['entry']['session_id'] == 'classic-child'+++async def _first_read_adoption(authority, owner, tmp_path, db):+ """A cold owner's first read (``session.resume``) of a session its boot recovery never reached."""+ from gateway.config import GatewayConfig+ from gateway.session import SessionStore+ from gateway.session_authority import SessionAuthority+ from gateway.session_controls import AuthorityConnection+ import hermes_state_runtime as rt+ from tests.gateway.test_local_receipt_untracked_compression import _compact_like_0_22_2+ ref = _local(authority, owner, tmp_path)+ _compact_like_0_22_2(db, ref.session_id, 'classic-child')+ store = SessionStore(tmp_path / 'sessions', GatewayConfig())+ runner = SimpleNamespace(**vars(authority.runner))+ runner.session_store, runner._session_db, runner.adapters = store, store._db, {}+ runner._adapter_for_source = lambda source: runner.adapters.get(source.platform)+ cold = SessionAuthority(runner, profile_id='owned', instance_id='cold', db=store._db,+ epoch=rt.begin_runtime_epoch(store._db, instance_id='cold'))+ reader = AuthorityConnection(cold, object(), {'user_id': 'human'})+ try:+ reply = await reader.dispatch({'id': 1, 'method': 'session.resume', 'params': {'session_id': ref.session_id}})+ assert 'error' not in reply, reply+ assert reply['result']['messages'][-1]['content'] == 'after compression into classic-child', reply+ finally:+ await reader.close()+ store.close_all_db_handles()++# (case, module holding the write's name, name, driver): the writer lock is taken just before# the named call runs.CASES = [@@ -132,6 +169,10 @@ CASES = [('create-title-hidden', 'gateway.session_local_title', 'title_new_session', _create_titled_hidden),('native-register', 'gateway.session.SessionStore', 'get_or_create_session', _native_register),('initialize-epoch', 'gateway.session_authority', 'begin_runtime_epoch', _initialize),+ ('initialize-compression-adoption', 'hermes_state_local_lineage', 'adopt_untracked_compression',+ _initialize_adoption),+ ('first-read-compression-adoption', 'hermes_state_local_lineage', 'adopt_untracked_compression',+ _first_read_adoption),]