The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): say how a parked profile comes back; log what Bot recovery skipped
unsupportedpastels4dc234372b04
diff --git a/gateway/run_runtime.py b/gateway/run_runtime.pyindex cd054f80ef8..f4c6b20028d 100644--- a/gateway/run_runtime.py+++ b/gateway/run_runtime.py@@ -412,12 +412,14 @@ async def _recover_profile_bot_mailbox(runner, authority):# A writer outlived Stop: the lock and the retiring authority stay for shutdown to# settle, but the profile leaves the served set, so later boot never starts it._withdraw_profile_home(runner, home)- park_profile(runner, name, f'startup recovery failed; ownership retained: {exc}')+ park_profile(runner, name, f'startup recovery failed; ownership retained (a control-socket rescan, '+ f'`hermes -p {name} gateway run`, serves it once its writer stops): {exc}')returnregistry.remove(home)_publish_served_set(runner)release_profile_home(runner, home)- park_profile(runner, name, f'startup recovery failed: {exc}')+ park_profile(runner, name, f'startup recovery failed (a control-socket rescan, '+ f'`hermes -p {name} gateway run`, serves it again): {exc}')async def recover_gateway_native_sessions(runner):diff --git a/gateway/session_bot.py b/gateway/session_bot.pyindex 24132fd640f..57f7aa47790 100644--- a/gateway/session_bot.py+++ b/gateway/session_bot.py@@ -283,7 +283,14 @@ def _expired_outcome(authority, record):if record.get('status') in _TERMINAL:return record['status'], record.get('reply', '')session_id = record.get('session_id')- if session_id and (retired_session(authority.db, session_id) or authority.db.get_session(session_id) is None):+ if session_id and retired_session(authority.db, session_id):+ return 'ambiguous', record.get('reply', '')+ if session_id and authority.db.get_session(session_id) is None:+ # Never resent either way, but a row gone without its fence is not a plain delete: store damage,+ # or a marker an older release reclaimed. Logged once: the receipt is final from here on.+ logging.getLogger(__name__).warning(+ 'Bot delivery %s: session %s is gone with no deletion marker; settled as ambiguous',+ record.get('delivery_id'), session_id)return 'ambiguous', record.get('reply', '')raise RuntimeStoreError('storage_unavailable')@@ -560,8 +567,8 @@ async def recover_bot_deliveries(authority):# One receipt the store cannot answer must not stop the owner (and, multiplexed, every# profile with it): log it and recover the rest. The next start reads it again (a# retry already pinned on it is then admitted once).- logging.getLogger(__name__).error('Bot delivery %s not recovered: %s',- record.get('delivery_id'), exc.reason)+ logging.getLogger(__name__).error('Bot delivery %s not recovered: %s (%s)',+ record.get('delivery_id'), exc.reason, path, exc_info=True)continueawait write_receipt(home, record)remember_receipt(authority, home, record)diff --git a/tests/gateway/test_bot_receipt_recovery_logs.py b/tests/gateway/test_bot_receipt_recovery_logs.pynew file mode 100644index 00000000000..50485c6a527--- /dev/null+++ b/tests/gateway/test_bot_receipt_recovery_logs.py@@ -0,0 +1,49 @@+"""Startup Bot receipt recovery says what it could not answer, and where.++A receipt the store cannot answer is logged with its file and traceback (the next start reads it+again). A pending receipt whose session row is gone with no deletion marker is settled as+``ambiguous`` like a deleted chat's, but that is store damage, so it is logged, not silent.+"""+import asyncio+from types import SimpleNamespace++from hermes_state import SessionDB+from hermes_state_runtime import begin_runtime_epoch+from tools.bot_live_delivery import _locked, _read, _write+++def _recover_one(tmp_path, record):+ from gateway.session_bot import recover_bot_deliveries+ db = SessionDB(tmp_path / 'state.db')+ try:+ begin_runtime_epoch(db, instance_id='owner')+ if record['session_id'] == 'live':+ db.create_session('live', source='cli')+ with _locked(tmp_path) as root:+ _write(root / f"{record['delivery_id']}.json", record)+ asyncio.run(recover_bot_deliveries(SimpleNamespace(db=db, waiters={})))+ with _locked(tmp_path) as root:+ return _read(root / f"{record['delivery_id']}.json")+ finally:+ db.close()+++def _record(key, session_id):+ return dict(status='queued', delivery_id=key, profile_home='', session_id=session_id,+ admission_id='missing', principal_id='owner', message='ping')+++def test_an_unanswerable_receipt_is_logged_with_its_file_and_traceback(tmp_path, caplog):+ key = 'c' * 32+ record = dict(_record(key, 'live'), profile_home=str(tmp_path))+ assert _recover_one(tmp_path, record)['status'] == 'queued'+ line = next(r for r in caplog.records if 'not recovered' in r.getMessage())+ assert f'{key}.json' in line.getMessage() and line.exc_info is not None, line.getMessage()+++def test_a_receipt_whose_session_vanished_without_a_marker_is_logged(tmp_path, caplog):+ key = 'd' * 32+ record = dict(_record(key, 'ghost'), profile_home=str(tmp_path))+ assert _recover_one(tmp_path, record)['status'] == 'ambiguous'+ assert any('ghost' in r.getMessage() and 'no deletion marker' in r.getMessage() for r in caplog.records), \+ caplog.textdiff --git a/tests/gateway/test_startup_recovery_parks_profile.py b/tests/gateway/test_startup_recovery_parks_profile.pyindex 9e793d22c76..25039edd98e 100644--- a/tests/gateway/test_startup_recovery_parks_profile.py+++ b/tests/gateway/test_startup_recovery_parks_profile.py@@ -64,6 +64,7 @@ def test_a_secondary_whose_recovery_fails_is_parked_and_the_rest_start(tmp_path,assert runner.session_runtime_descriptor["api_origin"] == "http://127.0.0.1:1"assert recovered == [str(launch), str(homes["beta"]), str(homes["gamma"])]assert "beta" in parked and "leftover receipt" in parked["beta"]+ assert "rescan" in parked["beta"], "the parked reason does not say how the profile comes back"assert runner.session_authorities.for_home(homes["beta"]) is Noneassert not process_ownership.owns(homes["beta"]), "a parked profile keeps its reservation"assert [n for n, _ in runner.config._runtime_profile_homes] == ["gamma"]@@ -91,7 +92,7 @@ def test_a_secondary_whose_retirement_times_out_keeps_its_lock_but_is_never_starprocess_ownership.reserve([homes["beta"]])try:asyncio.run(run_runtime.start_gateway_runtime_api(runner))- assert "ownership retained" in parked["beta"]+ assert "ownership retained" in parked["beta"] and "rescan" in parked["beta"]assert process_ownership.owns(homes["beta"]), "the live writer's home lost its lock"assert runner.session_authorities.for_home(homes["beta"]) is beta and beta.retiringassert beta in all_authorities(runner), "shutdown must still settle the retained authority"