Skip to content

The agent that grows with you

fix(gateway): commit a local receipt's compression catch-up off the owner loop

unsupportedpastels146354440098
diff --git a/gateway/session_authority.py b/gateway/session_authority.py
index a1f441a89f2..4a41f0c4b26 100644
--- a/gateway/session_authority.py
+++ b/gateway/session_authority.py
@@ -1178,6 +1178,10 @@ async def initialize_session_authority(runner, *, profile_id, instance_id, db=No
await asyncio.to_thread(recover_session_inputs, db, epoch=epoch)
await asyncio.to_thread(backfill_legacy_session_owners, db)
authority = SessionAuthority(runner, profile_id=profile_id, instance_id=instance_id, db=db, epoch=epoch)
+ # A receipt's compression catch-up (#140060) writes: it commits off the loop too, before the
+ # authority is published, so the recovery below only reads.
+ from gateway.session_local_adoption import adopt_local_compressions
+ await adopt_local_compressions(authority)
if register:
runner.session_authority = authority
from gateway.session_cron import bind_owner
diff --git a/gateway/session_controls.py b/gateway/session_controls.py
index c437a5e7b34..f54a8a91431 100644
--- a/gateway/session_controls.py
+++ b/gateway/session_controls.py
@@ -243,6 +243,14 @@ class AuthorityConnection:
'code': 4001, 'message': 'invalid_params', 'data': {'reason': 'invalid_params', 'fields': problems}}}
original_params = dict(params)
contract = contract or METHODS.get(method)
+ authority = self.authority
+ if (isinstance(ref.session_id, str) and ref.session_id and not getattr(authority, 'retiring', False)
+ and ref.profile_id == getattr(authority, 'profile_id', None)
+ and ref.session_id not in getattr(authority, 'sessions', {ref.session_id: None})):
+ # A first read of a session recovery never restored: its compression catch-up (#140060)
+ # commits in a worker here, so the handler's synchronous restore only reads.
+ from gateway.session_local_adoption import adopt_before_restore
+ await adopt_before_restore(authority, ref.session_id)
handlers = self.handlers()
try:
from gateway.session_group_controls import GROUP_METHODS, dispatch_group_control
diff --git a/gateway/session_local_adoption.py b/gateway/session_local_adoption.py
new file mode 100644
index 00000000000..8a6bdd63989
--- /dev/null
+++ b/gateway/session_local_adoption.py
@@ -0,0 +1,87 @@
+"""A local receipt's compression catch-up (#140060), committed off the owner loop.
+
+``adopt_untracked_compression`` takes the writer lock, up to ``_WRITE_PATIENCE_S`` when contended. On
+the owner loop that stalls every profile it serves (streams, controls, admissions), including during
+a hot-profile activation. So the async paths that restore a receipt (boot and hot-profile recovery, a
+socket's first read) commit the catch-up in a worker first, and the restore that follows only reads.
+A restore reached synchronously on the loop hands it to a tracked worker (``schedule_adoption``).
+"""
+import asyncio
+import logging
+from functools import partial
+
+from hermes_state_runtime import RuntimeStoreError
+
+logger = logging.getLogger(__name__)
+
+
+def on_owner_loop():
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ return False
+ return True
+
+
+def _due(authority, sid):
+ """*sid*'s stored receipt when a catch-up would commit for it on this owner, else None. Read-only."""
+ from gateway.session_local_recovery import owned_receipt
+ from hermes_state_local_lineage import adoptable_compression
+ try:
+ receipt = owned_receipt(authority, sid)[0]
+ with authority.db._read_ctx() as conn:
+ return receipt if adoptable_compression(conn, receipt) is not None else None
+ except (KeyError, TypeError, ValueError, AttributeError):
+ return None # the restore refuses this receipt and logs the check that failed
+
+
+def _adopt(authority, sid, receipt):
+ from hermes_state_local_lineage import adopt_untracked_compression
+ adopt_untracked_compression(authority.db, epoch=authority.epoch, session_id=sid, expected=receipt)
+
+
+async def adopt_before_restore(authority, sid, *, tracked=True):
+ """Commit *sid*'s due catch-up in a worker, so the restore that follows only reads.
+
+ ``tracked=False``: an authority still being built, not yet published for retirement to join."""
+ receipt = _due(authority, sid)
+ if receipt is None:
+ return
+ write = partial(_adopt, authority, sid, receipt)
+ try:
+ if tracked:
+ from gateway.session_runtime_workers import tracked_write
+ await tracked_write(authority, write)
+ else:
+ await asyncio.to_thread(write)
+ except RuntimeStoreError as exc: # the restore that follows refuses the receipt and logs it
+ logger.warning('Local session %s compression catch-up failed: %s', sid, exc.reason)
+
+
+async def adopt_local_compressions(authority):
+ """Boot and hot-profile recovery: every receipt's due catch-up, before ``recover_local_sessions``."""
+ from gateway.session_local_recovery import local_receipt_ids
+ for sid in local_receipt_ids(authority):
+ await adopt_before_restore(authority, sid, tracked=False)
+
+
+def schedule_adoption(authority, sid, receipt):
+ """A sync restore on the owner loop found *receipt*'s catch-up due: commit it in a tracked worker,
+ then publish the session on the loop. One worker per session at a time."""
+ from gateway.session_runtime_workers import track_mutation
+ pending = getattr(authority, '_local_adoptions', None)
+ if pending is None:
+ pending = authority._local_adoptions = {}
+ if sid in pending and not pending[sid].done():
+ return
+ logger.info('Local session %s: committing its compression catch-up off the owner loop', sid)
+
+ async def adopt_then_publish():
+ try:
+ await asyncio.to_thread(_adopt, authority, sid, receipt)
+ if not getattr(authority, 'retiring', False):
+ from gateway.session_local_recovery import restore_local_session
+ restore_local_session(authority, sid)
+ except RuntimeStoreError as exc:
+ logger.warning('Local session %s paused after its compression catch-up: %s', sid, exc.reason)
+ pending[sid] = track_mutation(authority, adopt_then_publish())
diff --git a/gateway/session_local_recovery.py b/gateway/session_local_recovery.py
index db84a84cbb9..fbba6d473cb 100644
--- a/gateway/session_local_recovery.py
+++ b/gateway/session_local_recovery.py
@@ -53,12 +53,26 @@ class _ReceiptRefused(RuntimeStoreError):
"""A refused receipt ``restore_local_session`` already logged with its failing check."""
+class _CatchUpPending(RuntimeStoreError):
+ """The receipt's compression catch-up is committing in a worker; this restore is refused."""
+
+
def _with_untracked_compression(authority, sid, receipt):
- """*receipt*, after recording compression steps an older build published past its tip (#140060)."""
- from hermes_state_local_lineage import adopt_untracked_compression, untracked_compression_tip
+ """*receipt*, after recording compression steps an older build published past its tip (#140060).
+
+ The catch-up writes, so it never runs on the owner loop: a contended writer there would stall every
+ profile the loop serves. The async restore paths commit it in a worker first
+ (``session_local_adoption``); a sync restore on the loop that still finds it hands it to a tracked
+ worker, which publishes the session once it commits, and refuses this read.
+ """
+ from hermes_state_local_lineage import adopt_untracked_compression, adoptable_compression
with authority.db._read_ctx() as conn:
- if untracked_compression_tip(conn, receipt) is None:
+ if adoptable_compression(conn, receipt) is None:
return receipt
+ from gateway.session_local_adoption import on_owner_loop, schedule_adoption
+ if on_owner_loop():
+ schedule_adoption(authority, sid, receipt)
+ raise _CatchUpPending('storage_unavailable')
adopt_untracked_compression(authority.db, epoch=authority.epoch, session_id=sid, expected=receipt)
return local_receipt(authority.db, sid)
@@ -82,23 +96,28 @@ def _checked_entry(authority, receipt, source, route):
return entry
+def owned_receipt(authority, sid):
+ """*sid*'s stored receipt with its LOCAL source, route and entry, once all are this owner's."""
+ receipt = local_receipt(authority.db, sid)
+ if receipt['profile_id'] != authority.profile_id:
+ raise RuntimeStoreError('profile_mismatch')
+ chat_id = (receipt['legacy_session_id'] if 'legacy_session_id' in receipt else
+ local_identity(receipt['profile_id'], receipt['principal_id'], receipt['request_id']))
+ if receipt['session_id'] != sid or sid != chat_id:
+ raise ValueError('identity mismatch')
+ source = local_source(authority, chat_id, receipt['principal_id'])
+ route = authority.runner.session_store._generate_session_key(source)
+ return receipt, source, route, _checked_entry(authority, receipt, source, route)
+
+
def restore_local_session(authority, sid):
from gateway.session_authority import LiveSession
from gateway.session_local import LocalSessionAdapter
from gateway.session_policy import restore_policy
try:
- receipt = local_receipt(authority.db, sid)
- if receipt['profile_id'] != authority.profile_id:
- raise RuntimeStoreError('profile_mismatch')
- chat_id = (receipt['legacy_session_id'] if 'legacy_session_id' in receipt else
- local_identity(receipt['profile_id'], receipt['principal_id'], receipt['request_id']))
- if receipt['session_id'] != sid or sid != chat_id:
- raise ValueError('identity mismatch')
- source = local_source(authority, chat_id, receipt['principal_id'])
- store = authority.runner.session_store
- route = store._generate_session_key(source)
# The catch-up writes, so the receipt as stored must already be this owner's (#140060).
- entry = _checked_entry(authority, receipt, source, route)
+ receipt, source, route, entry = owned_receipt(authority, sid)
+ chat_id, store = source.chat_id, authority.runner.session_store
adopted = _with_untracked_compression(authority, sid, receipt)
policy = restore_policy(adopted['policy'])
if adopted is not receipt: # re-read after a catch-up: check the copy that will be served
@@ -107,6 +126,8 @@ def restore_local_session(authority, sid):
from hermes_state_local_lineage import validate_local_lineage
with authority.db._read_ctx() as conn:
validate_local_lineage(conn, receipt)
+ except _CatchUpPending:
+ raise
except (KeyError, TypeError, ValueError, AttributeError) as exc:
# The client only sees the bounded code: gateway.log names the check that refused (#140099).
logging.getLogger(__name__).warning('Local session %s receipt refused: %s', sid, _refused_check(exc))
@@ -207,18 +228,21 @@ def reset_local_session(store, old_entry, session_id, now, display_name):
return entry
-def recover_local_sessions(authority, *, schedule=False):
+def local_receipt_ids(authority):
with authority.db._read_ctx() as conn:
- ids = [row[0][len(POLICY_PREFIX):] for row in conn.execute(
+ return [row[0][len(POLICY_PREFIX):] for row in conn.execute(
'SELECT key FROM state_meta WHERE key LIKE ?', (POLICY_PREFIX + '%',))]
- for sid in ids:
+
+
+def recover_local_sessions(authority, *, schedule=False):
+ for sid in local_receipt_ids(authority):
try:
ref = restore_local_session(authority, sid)
pending = list_session_admissions(authority.db, session_id=sid)
if schedule and pending and not any(row['status'] == 'unknown' for row in pending):
authority._schedule(ref)
- except _ReceiptRefused:
- continue # restore_local_session logged the check that refused it
+ except (_ReceiptRefused, _CatchUpPending):
+ continue # logged: the check that refused it, or the catch-up handed to a worker
except RuntimeStoreError as exc:
logging.getLogger(__name__).warning('Local session %s paused: %s', sid, exc.reason)
if schedule:
diff --git a/hermes_state_local_lineage.py b/hermes_state_local_lineage.py
index 395d3d3c26e..93fc303d216 100644
--- a/hermes_state_local_lineage.py
+++ b/hermes_state_local_lineage.py
@@ -119,11 +119,20 @@ def _untracked_steps(conn, tip, lineage):
current = batch[-1]
-def untracked_compression_tip(conn, receipt):
- """The compression child of *receipt*'s tip, when the receipt does not record it yet."""
- from hermes_state_compression import _CHAIN_STEP_SQL
- step = conn.execute(_CHAIN_STEP_SQL, (receipt['entry']['session_id'],)).fetchone()
- return step['id'] if step is not None else None
+def adoptable_compression(conn, receipt):
+ """``(steps, routed)``: the untracked compression steps *receipt* would adopt and its route's
+ entry, else None. Read-only; ``adopt_untracked_compression`` repeats it inside its write."""
+ tip = receipt['entry']['session_id']
+ children = _untracked_steps(conn, tip, receipt.get('lineage', [receipt['session_id']]))
+ if not children:
+ return None
+ identity = (receipt['route'], receipt['session_id'], receipt['principal_id'])
+ for child_id in children:
+ child = conn.execute('SELECT session_key,chat_id,user_id FROM sessions WHERE id=?', (child_id,)).fetchone()
+ if tuple(child) != identity:
+ return None
+ routed = _routed_entry(conn, receipt['route'], tip)
+ return (children, routed) if routed is not None else None
def adopt_untracked_compression(db, *, epoch, session_id, expected):
@@ -144,19 +153,12 @@ def adopt_untracked_compression(db, *, epoch, session_id, expected):
receipt = json.loads(saved[0])
if receipt != expected:
return
- identity = (receipt['route'], receipt['session_id'], receipt['principal_id'])
- lineage = list(receipt.get('lineage', [session_id]))
- tip = receipt['entry']['session_id']
- children = _untracked_steps(conn, tip, lineage)
- for child_id in children or ():
- child = conn.execute('SELECT session_key,chat_id,user_id FROM sessions WHERE id=?', (child_id,)).fetchone()
- if tuple(child) != identity:
- return
- routed = _routed_entry(conn, receipt['route'], tip)
- if not children or routed is None:
+ adoptable = adoptable_compression(conn, receipt)
+ if adoptable is None:
return
+ children, routed = adoptable
receipt['entry'] = dict(receipt['entry'], session_id=children[-1])
- receipt['lineage'] = [*lineage, *children]
+ receipt['lineage'] = [*receipt.get('lineage', [session_id]), *children]
validate_local_lineage(conn, receipt)
_commit_local_target(conn, receipt, dict(routed, session_id=children[-1]))
db._execute_write(write)
diff --git a/tests/gateway/test_local_receipt_untracked_compression.py b/tests/gateway/test_local_receipt_untracked_compression.py
index a0e79e0d0c2..ed6a588475a 100644
--- a/tests/gateway/test_local_receipt_untracked_compression.py
+++ b/tests/gateway/test_local_receipt_untracked_compression.py
@@ -58,6 +58,20 @@ def _compact_chain_like_0_22_2(db, parent_id, child_ids):
db._execute_write(write)
+def _unrecovered_owner(tmp_path, instance_id):
+ """An owner of the same store whose boot recovery never reached the session: its first read restores."""
+ from gateway.config import GatewayConfig
+ from gateway.session import SessionStore
+ from gateway.session_authority import SessionAuthority
+ from hermes_state_runtime import begin_runtime_epoch
+ store = SessionStore(tmp_path / 'sessions', GatewayConfig())
+ runner = SimpleNamespace(session_store=store, _session_db=store._db, adapters={}, _draining=False,
+ _cached_agent_for=lambda route: None, _adapter_for_source=lambda source: None)
+ authority = SessionAuthority(runner, profile_id=str(tmp_path), instance_id=instance_id, db=store._db,
+ epoch=begin_runtime_epoch(store._db, instance_id=instance_id))
+ return authority, AuthorityConnection(authority, object(), {'user_id': 'owner'})
+
+
def _receipt(db, sid):
from hermes_state_local import local_receipt
return local_receipt(db, sid)
@@ -278,6 +292,42 @@ async def test_a_chain_that_loops_back_past_one_walk_is_not_adopted(tmp_path, mo
second.db.close()
+async def test_a_restore_on_the_owner_loop_hands_the_catch_up_to_a_worker(tmp_path, monkeypatch):
+ """A restore reached synchronously on the owner loop (a bot, cron or A2A turn, a claim) never takes
+ the writer lock there: a contended writer would stall every profile the loop serves. It hands the
+ catch-up to a tracked worker and refuses this read; the worker's commit is published on the loop."""
+ import asyncio
+ import threading
+ import hermes_state_local_lineage as lineage
+ from gateway.session_local import create_local_session
+ from gateway.session_local_recovery import restore_local_session
+ from gateway.session_runtime_workers import mutation_tasks
+ from hermes_state_runtime import RuntimeStoreError
+
+ first, connection = await _owner(tmp_path, monkeypatch, 'first')
+ ref = create_local_session(first, connection.actor, {'request_id': 'r', 'source': 'tui',
+ 'cwd': str(tmp_path), 'model': 'm', 'toolsets': []})
+ _compact_like_0_22_2(first.db, ref.session_id, 'classic-child')
+ await connection.close()
+ first.db.close()
+
+ cold, connection = _unrecovered_owner(tmp_path, 'cold')
+ threads, adopt = [], lineage.adopt_untracked_compression
+ monkeypatch.setattr(lineage, 'adopt_untracked_compression',
+ lambda *a, **k: threads.append(threading.get_ident()) or adopt(*a, **k))
+ try:
+ with pytest.raises(RuntimeStoreError, match='storage_unavailable'):
+ restore_local_session(cold, ref.session_id)
+ await asyncio.gather(*mutation_tasks(cold))
+ assert threads and threading.get_ident() not in threads, 'the catch-up ran on the owner loop'
+ assert _receipt(cold.db, ref.session_id)['entry']['session_id'] == 'classic-child'
+ assert ref.session_id in cold.sessions, 'the committed catch-up was not published'
+ finally:
+ await connection.close()
+ cold.db.close()
+
+
@pytest.mark.asyncio
async def test_a_restore_with_nothing_to_adopt_checks_the_stored_entry_once(tmp_path, monkeypatch):
"""The catch-up re-checks the receipt only when it moved it. Every LOCAL turn claim restores, so a
diff --git a/tests/gateway/test_owner_loop_writes_class.py b/tests/gateway/test_owner_loop_writes_class.py
index fb0f58b87ba..85d4838d482 100644
--- a/tests/gateway/test_owner_loop_writes_class.py
+++ b/tests/gateway/test_owner_loop_writes_class.py
@@ -122,6 +122,43 @@ async def _initialize(authority, owner, tmp_path, db):
await initialize_session_authority(authority.runner, profile_id='owned', instance_id='second')
+async def _initialize_adoption(authority, owner, tmp_path, db):
+ """Boot and hot-profile activation (``serve_profile_runtime`` -> ``_build_profile_authority``)
+ recover through ``initialize_session_authority`` while the loop serves the other profiles."""
+ from hermes_state_local import local_receipt
+ from tests.gateway.test_local_receipt_untracked_compression import _compact_like_0_22_2
+ ref = _local(authority, owner, tmp_path)
+ _compact_like_0_22_2(db, ref.session_id, 'classic-child')
+ await _initialize(authority, owner, tmp_path, db)
+ assert local_receipt(db, ref.session_id)['entry']['session_id'] == 'classic-child'
+
+
+async def _first_read_adoption(authority, owner, tmp_path, db):
+ """A cold owner's first read (``session.resume``) of a session its boot recovery never reached."""
+ from gateway.config import GatewayConfig
+ from gateway.session import SessionStore
+ from gateway.session_authority import SessionAuthority
+ from gateway.session_controls import AuthorityConnection
+ import hermes_state_runtime as rt
+ from tests.gateway.test_local_receipt_untracked_compression import _compact_like_0_22_2
+ ref = _local(authority, owner, tmp_path)
+ _compact_like_0_22_2(db, ref.session_id, 'classic-child')
+ store = SessionStore(tmp_path / 'sessions', GatewayConfig())
+ runner = SimpleNamespace(**vars(authority.runner))
+ runner.session_store, runner._session_db, runner.adapters = store, store._db, {}
+ runner._adapter_for_source = lambda source: runner.adapters.get(source.platform)
+ cold = SessionAuthority(runner, profile_id='owned', instance_id='cold', db=store._db,
+ epoch=rt.begin_runtime_epoch(store._db, instance_id='cold'))
+ reader = AuthorityConnection(cold, object(), {'user_id': 'human'})
+ try:
+ reply = await reader.dispatch({'id': 1, 'method': 'session.resume', 'params': {'session_id': ref.session_id}})
+ assert 'error' not in reply, reply
+ assert reply['result']['messages'][-1]['content'] == 'after compression into classic-child', reply
+ finally:
+ await reader.close()
+ store.close_all_db_handles()
+
+
# (case, module holding the write's name, name, driver): the writer lock is taken just before
# the named call runs.
CASES = [
@@ -132,6 +169,10 @@ CASES = [
('create-title-hidden', 'gateway.session_local_title', 'title_new_session', _create_titled_hidden),
('native-register', 'gateway.session.SessionStore', 'get_or_create_session', _native_register),
('initialize-epoch', 'gateway.session_authority', 'begin_runtime_epoch', _initialize),
+ ('initialize-compression-adoption', 'hermes_state_local_lineage', 'adopt_untracked_compression',
+ _initialize_adoption),
+ ('first-read-compression-adoption', 'hermes_state_local_lineage', 'adopt_untracked_compression',
+ _first_read_adoption),
]