Skip to content

The agent that grows with you

fix(gateway): name the check that refused a local receipt in gateway.log

unsupportedpastels6cf3e718e5b3
diff --git a/gateway/session_local_recovery.py b/gateway/session_local_recovery.py
index 093ba5c3a10..b635ca43cde 100644
--- a/gateway/session_local_recovery.py
+++ b/gateway/session_local_recovery.py
@@ -37,12 +37,24 @@ def local_source(authority, chat_id, user_id):
return source
+def _refused_check(exc):
+ """``<reason or error> at <function>:<line>``: where a receipt check raised, and why, followed
+ through every chained ``__cause__`` (``restore_policy`` and ``local_receipt`` wrap the check
+ that failed into a bare ``storage_unavailable``)."""
+ import traceback
+ frames = traceback.extract_tb(exc.__traceback__)
+ cause = exc.reason if isinstance(exc, RuntimeStoreError) else f'{type(exc).__name__}: {exc}'
+ where = f' at {frames[-1].name}:{frames[-1].lineno}' if frames else ''
+ chained = f', caused by {_refused_check(exc.__cause__)}' if exc.__cause__ is not None else ''
+ return f'{cause}{where}{chained}'
+
+
def restore_local_session(authority, sid):
from gateway.session_authority import LiveSession
from gateway.session_local import LocalSessionAdapter
from gateway.session_policy import restore_policy
- receipt = local_receipt(authority.db, sid)
try:
+ receipt = local_receipt(authority.db, sid)
if receipt['profile_id'] != authority.profile_id:
raise RuntimeStoreError('profile_mismatch')
chat_id = (receipt['legacy_session_id'] if 'legacy_session_id' in receipt else
@@ -72,6 +84,8 @@ def restore_local_session(authority, sid):
with authority.db._read_ctx() as conn:
validate_local_lineage(conn, receipt)
except (KeyError, TypeError, ValueError, AttributeError) as exc:
+ # The client only sees the bounded code: gateway.log names the check that refused (#140099).
+ logging.getLogger(__name__).warning('Local session %s receipt refused: %s', sid, _refused_check(exc))
if isinstance(exc, RuntimeStoreError):
raise
raise RuntimeStoreError('storage_unavailable') from exc
diff --git a/tests/gateway/test_local_receipt_missing_profile.py b/tests/gateway/test_local_receipt_missing_profile.py
index 048c585f6c9..1d26a59fcab 100644
--- a/tests/gateway/test_local_receipt_missing_profile.py
+++ b/tests/gateway/test_local_receipt_missing_profile.py
@@ -5,6 +5,16 @@ import pytest
from tests.gateway.test_session_authorities_multiplex import _reserve_homes, _runner
+# How each variant ages the receipt once its origin lost ``profile`` (``missing`` keeps just that).
+_AGED = {
+ 'missing': lambda receipt, root: None,
+ 'null': lambda receipt, root: receipt['entry']['origin'].update(profile=None),
+ 'foreign': lambda receipt, root: receipt['entry']['origin'].update(profile='beta'),
+ 'route': lambda receipt, root: receipt.update(route=receipt['route'].replace('agent:alpha:', 'agent:main:')),
+ 'home': lambda receipt, root: receipt.update(profile_id=str(root)),
+ 'user': lambda receipt, root: receipt['entry']['origin'].update(user_id='another'),
+}
+
@pytest.mark.asyncio
@pytest.mark.parametrize('variant', ['current', 'missing', 'null', 'foreign', 'route', 'home', 'user'])
@@ -37,14 +47,7 @@ async def test_secondary_receipt_restores_without_losing_its_owner(tmp_path, mon
original = json.loads(json.dumps(receipt))
if variant != 'current':
receipt['entry']['origin'].pop('profile')
- if variant in ('null', 'foreign'):
- receipt['entry']['origin']['profile'] = None if variant == 'null' else 'beta'
- elif variant == 'route':
- receipt['route'] = receipt['route'].replace('agent:alpha:', 'agent:main:')
- elif variant == 'home':
- receipt['profile_id'] = str(root)
- elif variant == 'user':
- receipt['entry']['origin']['user_id'] = 'another'
+ _AGED[variant](receipt, root)
authority.db._execute_write(lambda conn: conn.execute(
'UPDATE state_meta SET value=? WHERE key=?',
(json.dumps(receipt), POLICY_PREFIX + ref.session_id)))
diff --git a/tests/gateway/test_local_recovery_pre_profile_origin.py b/tests/gateway/test_local_recovery_pre_profile_origin.py
new file mode 100644
index 00000000000..5a59cadbd5e
--- /dev/null
+++ b/tests/gateway/test_local_recovery_pre_profile_origin.py
@@ -0,0 +1,182 @@
+"""A served profile's receipts from before origin composition still resume (#140099)."""
+import json
+from types import SimpleNamespace
+
+import pytest
+
+
+def _build_runtime(tmp_path, monkeypatch):
+ """A fresh runner/registry/store over the reserved homes (launch ``default`` + ``secondary``):
+ a cold restart builds a second one over the same on-disk state."""
+ from gateway.config import GatewayConfig
+ from gateway.session import SessionStore
+ from gateway.session_authorities import SessionAuthorities
+ from gateway import run
+ root = tmp_path / '.hermes'
+ root.mkdir(mode=0o700, exist_ok=True)
+ monkeypatch.setenv('HERMES_HOME', str(root))
+ # The autouse isolation fixture pins hermes_state.DEFAULT_DB_PATH to its own fake home once
+ # hermes_state is imported; per-home stores must follow the scoped HERMES_HOME here.
+ import hermes_state
+ monkeypatch.setattr(hermes_state, 'DEFAULT_DB_PATH', hermes_state._IMPORT_DEFAULT_DB_PATH)
+ secondary = root / 'profiles' / 'secondary'
+ secondary.mkdir(parents=True, mode=0o700, exist_ok=True)
+ if not (secondary / 'config.yaml').exists():
+ (secondary / 'config.yaml').write_text('{}')
+ monkeypatch.setattr(run, '_load_gateway_config', lambda: {'platform_toolsets': {'cli': []}})
+ store = SessionStore(tmp_path / 'sessions', GatewayConfig())
+ registry = SessionAuthorities(root, multiplexed=True)
+ runner = SimpleNamespace(session_store=store, _session_db=store._db, adapters={},
+ _draining=False, session_authorities=registry)
+ return runner, registry, root, secondary
+
+
+async def _served_authority(tmp_path, monkeypatch, instance_id, db=None):
+ """Launch home plus one served secondary sharing the runner's store, as a multiplex gateway
+ builds them; returns (served authority, its state db, the secondary home path)."""
+ from gateway.session_authority import initialize_session_authority
+ from hermes_state import SessionDB
+ runner, registry, root, secondary = _build_runtime(tmp_path, monkeypatch)
+ launch = await initialize_session_authority(runner, profile_id=str(root), instance_id=instance_id)
+ registry.add(root, launch, name='default')
+ if db is None:
+ db = SessionDB(db_path=secondary / 'state.db')
+ # Reserve-then-replace, as the multiplex bring-up does: recovery inside initialize must
+ # already see the served slot (and thus the profile name) to rebuild a profiled source.
+ registry.add(secondary, None, name='secondary')
+ served = await initialize_session_authority(runner, profile_id=str(secondary),
+ instance_id=instance_id, register=False, db=db)
+ registry.replace(secondary, served)
+ return served, db, secondary
+
+
+def _age_receipt_origins(db, sid, mutate):
+ """Rewrite every persisted origin copy of *sid*'s receipt through *mutate* (a dict -> dict
+ function), the way an older build's on-disk spelling differs from the recomputed source."""
+ from hermes_state_local import POLICY_PREFIX
+ with db._read_ctx() as conn:
+ row = conn.execute('SELECT value FROM state_meta WHERE key=?', (POLICY_PREFIX + sid,)).fetchone()
+ receipt = json.loads(row[0])
+ receipt['entry']['origin'] = mutate(receipt['entry']['origin'])
+ db._write_sql('UPDATE state_meta SET value=? WHERE key=?', (json.dumps(receipt), POLICY_PREFIX + sid))
+
+ def age_routing(conn):
+ rows = conn.execute('SELECT session_key, entry_json FROM gateway_routing WHERE session_key=?',
+ (receipt['route'],)).fetchall()
+ for key, entry_json in rows:
+ entry = json.loads(entry_json)
+ if entry.get('origin') is not None:
+ entry['origin'] = mutate(entry['origin'])
+ conn.execute('UPDATE gateway_routing SET entry_json=? WHERE session_key=?',
+ (json.dumps(entry), key))
+ db._execute_write(age_routing)
+ with db._read_ctx() as conn:
+ row = conn.execute('SELECT origin_json FROM sessions WHERE id=?', (sid,)).fetchone()
+ db._write_sql('UPDATE sessions SET origin_json=? WHERE id=?', (json.dumps(mutate(json.loads(row[0]))), sid))
+ return receipt
+
+
+async def test_served_receipt_without_origin_profile_resumes(tmp_path, monkeypatch):
+ """Receipts a served profile wrote before ``local_source`` composed ``profile`` into the
+ origin (every persisted copy missing the key) must still cold-resume once the recomputed
+ source carries the profile; the route, row identity and lineage never changed."""
+ from gateway.session_contract import Principal
+ from gateway.session_local import create_local_session
+
+ served, db, secondary = await _served_authority(tmp_path, monkeypatch, 'first')
+ actor = Principal('owner', str(secondary), frozenset({'session:create', 'session:read'}), 'socket')
+ ref = create_local_session(served, actor, {'request_id': 'legacy', 'source': 'gui',
+ 'cwd': str(tmp_path), 'model': 'frozen', 'toolsets': []})
+ _age_receipt_origins(db, ref.session_id, lambda origin: {k: v for k, v in origin.items() if k != 'profile'})
+
+ cold, _, _ = await _served_authority(tmp_path, monkeypatch, 'second', db=db)
+ assert ref.session_id in cold.sessions, 'pre-profile receipt failed to cold-resume'
+ assert cold.sessions[ref.session_id].source.profile == 'secondary'
+
+
+async def test_pre_profile_tolerance_rejects_a_foreign_profile(tmp_path, monkeypatch):
+ """The tolerance is for the missing key only: an origin that names a DIFFERENT profile is a
+ different session's receipt and must keep failing closed, not borrow this authority."""
+ from gateway.session_contract import Principal
+ from gateway.session_local import create_local_session
+ from gateway.session_local_recovery import restore_local_session
+ from hermes_state_runtime import RuntimeStoreError
+
+ served, db, secondary = await _served_authority(tmp_path, monkeypatch, 'first')
+ actor = Principal('owner', str(secondary), frozenset({'session:create', 'session:read'}), 'socket')
+ ref = create_local_session(served, actor, {'request_id': 'foreign', 'source': 'gui',
+ 'cwd': str(tmp_path), 'model': 'frozen', 'toolsets': []})
+ _age_receipt_origins(db, ref.session_id, lambda origin: {**origin, 'profile': 'other-profile'})
+
+ cold, _, _ = await _served_authority(tmp_path, monkeypatch, 'second', db=db)
+ assert ref.session_id not in cold.sessions
+ with pytest.raises(RuntimeStoreError, match='storage_unavailable'):
+ restore_local_session(cold, ref.session_id)
+
+
+def _store_receipt(db, sid, mutate):
+ """Rewrite *sid*'s stored receipt text through *mutate* (a str -> str function)."""
+ from hermes_state_local import POLICY_PREFIX
+ with db._read_ctx() as conn:
+ stored = conn.execute('SELECT value FROM state_meta WHERE key=?', (POLICY_PREFIX + sid,)).fetchone()[0]
+ db._write_sql('UPDATE state_meta SET value=? WHERE key=?', (mutate(stored), POLICY_PREFIX + sid))
+
+
+def _with_receipt_field(path, value):
+ """A receipt-text mutation that sets the nested field at *path* to *value*."""
+ def mutate(stored):
+ receipt = json.loads(stored)
+ node = receipt
+ for key in path[:-1]:
+ node = node[key]
+ node[path[-1]] = value
+ return json.dumps(receipt)
+ return mutate
+
+
+_RECEIPT_DAMAGE = {
+ 'foreign_origin': lambda db, sid: _age_receipt_origins(db, sid, lambda origin: {**origin, 'profile': 'other-profile'}),
+ 'lineage': lambda db, sid: _store_receipt(db, sid, _with_receipt_field(('lineage',), [sid, 'gone'])),
+ # restore_policy wraps this ValueError into a bare storage_unavailable.
+ 'terminal_policy': lambda db, sid: _store_receipt(db, sid, _with_receipt_field(
+ ('policy', 'terminal_json'), json.dumps({'TERMINAL_ENV': 'local', 'TERMINAL_CWD': '/elsewhere'}))),
+ # local_receipt itself refuses undecodable text.
+ 'malformed': lambda db, sid: _store_receipt(db, sid, lambda stored: stored[:-1]),
+}
+
+
[email protected]('damage, clause', [
+ ('foreign_origin', 'entry mismatch'),
+ ('lineage', 'validate_local_lineage'),
+ ('terminal_policy', 'ValueError: terminal policy mismatch'),
+ ('malformed', 'JSONDecodeError'),
+])
+async def test_a_refused_receipt_names_its_failing_check_in_the_log(tmp_path, monkeypatch, caplog, damage, clause):
+ """#140099: the client only ever sees ``storage_unavailable``; gateway.log must name the check that
+ refused the receipt (the wrapped cause, or where an inner check raised), on cold recovery and on
+ every later restore, so a field report pins the layer without offline replication."""
+ import logging
+ from gateway.session_contract import Principal
+ from gateway.session_local import create_local_session
+ from gateway.session_local_recovery import restore_local_session
+ from hermes_state_runtime import RuntimeStoreError
+
+ served, db, secondary = await _served_authority(tmp_path, monkeypatch, 'first')
+ actor = Principal('owner', str(secondary), frozenset({'session:create', 'session:read'}), 'socket')
+ ref = create_local_session(served, actor, {'request_id': damage, 'source': 'gui',
+ 'cwd': str(tmp_path), 'model': 'frozen', 'toolsets': []})
+ _RECEIPT_DAMAGE[damage](db, ref.session_id)
+
+ with caplog.at_level(logging.WARNING, logger='gateway.session_local_recovery'):
+ cold, _, _ = await _served_authority(tmp_path, monkeypatch, 'second', db=db)
+ assert ref.session_id not in cold.sessions
+ recovery = [r.getMessage() for r in caplog.records if ref.session_id in r.getMessage()]
+ assert any(clause in message for message in recovery), recovery
+ caplog.clear()
+ with pytest.raises(RuntimeStoreError, match='storage_unavailable'):
+ restore_local_session(cold, ref.session_id)
+ resumed = [r.getMessage() for r in caplog.records if ref.session_id in r.getMessage()]
+ assert any(clause in message for message in resumed), resumed