The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): name the check that refused a local receipt in gateway.log
unsupportedpastels6cf3e718e5b3
diff --git a/gateway/session_local_recovery.py b/gateway/session_local_recovery.pyindex 093ba5c3a10..b635ca43cde 100644--- a/gateway/session_local_recovery.py+++ b/gateway/session_local_recovery.py@@ -37,12 +37,24 @@ def local_source(authority, chat_id, user_id):return source+def _refused_check(exc):+ """``<reason or error> at <function>:<line>``: where a receipt check raised, and why, followed+ through every chained ``__cause__`` (``restore_policy`` and ``local_receipt`` wrap the check+ that failed into a bare ``storage_unavailable``)."""+ import traceback+ frames = traceback.extract_tb(exc.__traceback__)+ cause = exc.reason if isinstance(exc, RuntimeStoreError) else f'{type(exc).__name__}: {exc}'+ where = f' at {frames[-1].name}:{frames[-1].lineno}' if frames else ''+ chained = f', caused by {_refused_check(exc.__cause__)}' if exc.__cause__ is not None else ''+ return f'{cause}{where}{chained}'++def restore_local_session(authority, sid):from gateway.session_authority import LiveSessionfrom gateway.session_local import LocalSessionAdapterfrom gateway.session_policy import restore_policy- receipt = local_receipt(authority.db, sid)try:+ receipt = local_receipt(authority.db, sid)if receipt['profile_id'] != authority.profile_id:raise RuntimeStoreError('profile_mismatch')chat_id = (receipt['legacy_session_id'] if 'legacy_session_id' in receipt else@@ -72,6 +84,8 @@ def restore_local_session(authority, sid):with authority.db._read_ctx() as conn:validate_local_lineage(conn, receipt)except (KeyError, TypeError, ValueError, AttributeError) as exc:+ # The client only sees the bounded code: gateway.log names the check that refused (#140099).+ logging.getLogger(__name__).warning('Local session %s receipt refused: %s', sid, _refused_check(exc))if isinstance(exc, RuntimeStoreError):raiseraise RuntimeStoreError('storage_unavailable') from excdiff --git a/tests/gateway/test_local_receipt_missing_profile.py b/tests/gateway/test_local_receipt_missing_profile.pyindex 048c585f6c9..1d26a59fcab 100644--- a/tests/gateway/test_local_receipt_missing_profile.py+++ b/tests/gateway/test_local_receipt_missing_profile.py@@ -5,6 +5,16 @@ import pytestfrom tests.gateway.test_session_authorities_multiplex import _reserve_homes, _runner+# How each variant ages the receipt once its origin lost ``profile`` (``missing`` keeps just that).+_AGED = {+ 'missing': lambda receipt, root: None,+ 'null': lambda receipt, root: receipt['entry']['origin'].update(profile=None),+ 'foreign': lambda receipt, root: receipt['entry']['origin'].update(profile='beta'),+ 'route': lambda receipt, root: receipt.update(route=receipt['route'].replace('agent:alpha:', 'agent:main:')),+ 'home': lambda receipt, root: receipt.update(profile_id=str(root)),+ 'user': lambda receipt, root: receipt['entry']['origin'].update(user_id='another'),+}+@pytest.mark.asyncio@pytest.mark.parametrize('variant', ['current', 'missing', 'null', 'foreign', 'route', 'home', 'user'])@@ -37,14 +47,7 @@ async def test_secondary_receipt_restores_without_losing_its_owner(tmp_path, monoriginal = json.loads(json.dumps(receipt))if variant != 'current':receipt['entry']['origin'].pop('profile')- if variant in ('null', 'foreign'):- receipt['entry']['origin']['profile'] = None if variant == 'null' else 'beta'- elif variant == 'route':- receipt['route'] = receipt['route'].replace('agent:alpha:', 'agent:main:')- elif variant == 'home':- receipt['profile_id'] = str(root)- elif variant == 'user':- receipt['entry']['origin']['user_id'] = 'another'+ _AGED[variant](receipt, root)authority.db._execute_write(lambda conn: conn.execute('UPDATE state_meta SET value=? WHERE key=?',(json.dumps(receipt), POLICY_PREFIX + ref.session_id)))diff --git a/tests/gateway/test_local_recovery_pre_profile_origin.py b/tests/gateway/test_local_recovery_pre_profile_origin.pynew file mode 100644index 00000000000..5a59cadbd5e--- /dev/null+++ b/tests/gateway/test_local_recovery_pre_profile_origin.py@@ -0,0 +1,182 @@+"""A served profile's receipts from before origin composition still resume (#140099)."""+import json+from types import SimpleNamespace++import pytest+++def _build_runtime(tmp_path, monkeypatch):+ """A fresh runner/registry/store over the reserved homes (launch ``default`` + ``secondary``):+ a cold restart builds a second one over the same on-disk state."""+ from gateway.config import GatewayConfig+ from gateway.session import SessionStore+ from gateway.session_authorities import SessionAuthorities+ from gateway import run+ root = tmp_path / '.hermes'+ root.mkdir(mode=0o700, exist_ok=True)+ monkeypatch.setenv('HERMES_HOME', str(root))+ # The autouse isolation fixture pins hermes_state.DEFAULT_DB_PATH to its own fake home once+ # hermes_state is imported; per-home stores must follow the scoped HERMES_HOME here.+ import hermes_state+ monkeypatch.setattr(hermes_state, 'DEFAULT_DB_PATH', hermes_state._IMPORT_DEFAULT_DB_PATH)+ secondary = root / 'profiles' / 'secondary'+ secondary.mkdir(parents=True, mode=0o700, exist_ok=True)+ if not (secondary / 'config.yaml').exists():+ (secondary / 'config.yaml').write_text('{}')+ monkeypatch.setattr(run, '_load_gateway_config', lambda: {'platform_toolsets': {'cli': []}})+ store = SessionStore(tmp_path / 'sessions', GatewayConfig())+ registry = SessionAuthorities(root, multiplexed=True)+ runner = SimpleNamespace(session_store=store, _session_db=store._db, adapters={},+ _draining=False, session_authorities=registry)+ return runner, registry, root, secondary+++async def _served_authority(tmp_path, monkeypatch, instance_id, db=None):+ """Launch home plus one served secondary sharing the runner's store, as a multiplex gateway+ builds them; returns (served authority, its state db, the secondary home path)."""+ from gateway.session_authority import initialize_session_authority+ from hermes_state import SessionDB+ runner, registry, root, secondary = _build_runtime(tmp_path, monkeypatch)+ launch = await initialize_session_authority(runner, profile_id=str(root), instance_id=instance_id)+ registry.add(root, launch, name='default')+ if db is None:+ db = SessionDB(db_path=secondary / 'state.db')+ # Reserve-then-replace, as the multiplex bring-up does: recovery inside initialize must+ # already see the served slot (and thus the profile name) to rebuild a profiled source.+ registry.add(secondary, None, name='secondary')+ served = await initialize_session_authority(runner, profile_id=str(secondary),+ instance_id=instance_id, register=False, db=db)+ registry.replace(secondary, served)+ return served, db, secondary+++def _age_receipt_origins(db, sid, mutate):+ """Rewrite every persisted origin copy of *sid*'s receipt through *mutate* (a dict -> dict+ function), the way an older build's on-disk spelling differs from the recomputed source."""+ from hermes_state_local import POLICY_PREFIX+ with db._read_ctx() as conn:+ row = conn.execute('SELECT value FROM state_meta WHERE key=?', (POLICY_PREFIX + sid,)).fetchone()+ receipt = json.loads(row[0])+ receipt['entry']['origin'] = mutate(receipt['entry']['origin'])+ db._write_sql('UPDATE state_meta SET value=? WHERE key=?', (json.dumps(receipt), POLICY_PREFIX + sid))++ def age_routing(conn):+ rows = conn.execute('SELECT session_key, entry_json FROM gateway_routing WHERE session_key=?',+ (receipt['route'],)).fetchall()+ for key, entry_json in rows:+ entry = json.loads(entry_json)+ if entry.get('origin') is not None:+ entry['origin'] = mutate(entry['origin'])+ conn.execute('UPDATE gateway_routing SET entry_json=? WHERE session_key=?',+ (json.dumps(entry), key))+ db._execute_write(age_routing)+ with db._read_ctx() as conn:+ row = conn.execute('SELECT origin_json FROM sessions WHERE id=?', (sid,)).fetchone()+ db._write_sql('UPDATE sessions SET origin_json=? WHERE id=?', (json.dumps(mutate(json.loads(row[0]))), sid))+ return receipt+++async def test_served_receipt_without_origin_profile_resumes(tmp_path, monkeypatch):+ """Receipts a served profile wrote before ``local_source`` composed ``profile`` into the+ origin (every persisted copy missing the key) must still cold-resume once the recomputed+ source carries the profile; the route, row identity and lineage never changed."""+ from gateway.session_contract import Principal+ from gateway.session_local import create_local_session++ served, db, secondary = await _served_authority(tmp_path, monkeypatch, 'first')+ actor = Principal('owner', str(secondary), frozenset({'session:create', 'session:read'}), 'socket')+ ref = create_local_session(served, actor, {'request_id': 'legacy', 'source': 'gui',+ 'cwd': str(tmp_path), 'model': 'frozen', 'toolsets': []})+ _age_receipt_origins(db, ref.session_id, lambda origin: {k: v for k, v in origin.items() if k != 'profile'})++ cold, _, _ = await _served_authority(tmp_path, monkeypatch, 'second', db=db)+ assert ref.session_id in cold.sessions, 'pre-profile receipt failed to cold-resume'+ assert cold.sessions[ref.session_id].source.profile == 'secondary'+++async def test_pre_profile_tolerance_rejects_a_foreign_profile(tmp_path, monkeypatch):+ """The tolerance is for the missing key only: an origin that names a DIFFERENT profile is a+ different session's receipt and must keep failing closed, not borrow this authority."""+ from gateway.session_contract import Principal+ from gateway.session_local import create_local_session+ from gateway.session_local_recovery import restore_local_session+ from hermes_state_runtime import RuntimeStoreError++ served, db, secondary = await _served_authority(tmp_path, monkeypatch, 'first')+ actor = Principal('owner', str(secondary), frozenset({'session:create', 'session:read'}), 'socket')+ ref = create_local_session(served, actor, {'request_id': 'foreign', 'source': 'gui',+ 'cwd': str(tmp_path), 'model': 'frozen', 'toolsets': []})+ _age_receipt_origins(db, ref.session_id, lambda origin: {**origin, 'profile': 'other-profile'})++ cold, _, _ = await _served_authority(tmp_path, monkeypatch, 'second', db=db)+ assert ref.session_id not in cold.sessions+ with pytest.raises(RuntimeStoreError, match='storage_unavailable'):+ restore_local_session(cold, ref.session_id)+++def _store_receipt(db, sid, mutate):+ """Rewrite *sid*'s stored receipt text through *mutate* (a str -> str function)."""+ from hermes_state_local import POLICY_PREFIX+ with db._read_ctx() as conn:+ stored = conn.execute('SELECT value FROM state_meta WHERE key=?', (POLICY_PREFIX + sid,)).fetchone()[0]+ db._write_sql('UPDATE state_meta SET value=? WHERE key=?', (mutate(stored), POLICY_PREFIX + sid))+++def _with_receipt_field(path, value):+ """A receipt-text mutation that sets the nested field at *path* to *value*."""+ def mutate(stored):+ receipt = json.loads(stored)+ node = receipt+ for key in path[:-1]:+ node = node[key]+ node[path[-1]] = value+ return json.dumps(receipt)+ return mutate+++_RECEIPT_DAMAGE = {+ 'foreign_origin': lambda db, sid: _age_receipt_origins(db, sid, lambda origin: {**origin, 'profile': 'other-profile'}),+ 'lineage': lambda db, sid: _store_receipt(db, sid, _with_receipt_field(('lineage',), [sid, 'gone'])),+ # restore_policy wraps this ValueError into a bare storage_unavailable.+ 'terminal_policy': lambda db, sid: _store_receipt(db, sid, _with_receipt_field(+ ('policy', 'terminal_json'), json.dumps({'TERMINAL_ENV': 'local', 'TERMINAL_CWD': '/elsewhere'}))),+ # local_receipt itself refuses undecodable text.+ 'malformed': lambda db, sid: _store_receipt(db, sid, lambda stored: stored[:-1]),+}++[email protected]('damage, clause', [+ ('foreign_origin', 'entry mismatch'),+ ('lineage', 'validate_local_lineage'),+ ('terminal_policy', 'ValueError: terminal policy mismatch'),+ ('malformed', 'JSONDecodeError'),+])+async def test_a_refused_receipt_names_its_failing_check_in_the_log(tmp_path, monkeypatch, caplog, damage, clause):+ """#140099: the client only ever sees ``storage_unavailable``; gateway.log must name the check that+ refused the receipt (the wrapped cause, or where an inner check raised), on cold recovery and on+ every later restore, so a field report pins the layer without offline replication."""+ import logging+ from gateway.session_contract import Principal+ from gateway.session_local import create_local_session+ from gateway.session_local_recovery import restore_local_session+ from hermes_state_runtime import RuntimeStoreError++ served, db, secondary = await _served_authority(tmp_path, monkeypatch, 'first')+ actor = Principal('owner', str(secondary), frozenset({'session:create', 'session:read'}), 'socket')+ ref = create_local_session(served, actor, {'request_id': damage, 'source': 'gui',+ 'cwd': str(tmp_path), 'model': 'frozen', 'toolsets': []})+ _RECEIPT_DAMAGE[damage](db, ref.session_id)++ with caplog.at_level(logging.WARNING, logger='gateway.session_local_recovery'):+ cold, _, _ = await _served_authority(tmp_path, monkeypatch, 'second', db=db)+ assert ref.session_id not in cold.sessions+ recovery = [r.getMessage() for r in caplog.records if ref.session_id in r.getMessage()]+ assert any(clause in message for message in recovery), recovery+ caplog.clear()+ with pytest.raises(RuntimeStoreError, match='storage_unavailable'):+ restore_local_session(cold, ref.session_id)+ resumed = [r.getMessage() for r in caplog.records if ref.session_id in r.getMessage()]+ assert any(clause in message for message in resumed), resumed