Skip to content

The agent that grows with you

fix(gateway): say how a parked profile comes back; log what Bot recovery skipped

unsupportedpastels4dc234372b04
diff --git a/gateway/run_runtime.py b/gateway/run_runtime.py
index cd054f80ef8..f4c6b20028d 100644
--- a/gateway/run_runtime.py
+++ b/gateway/run_runtime.py
@@ -412,12 +412,14 @@ async def _recover_profile_bot_mailbox(runner, authority):
# A writer outlived Stop: the lock and the retiring authority stay for shutdown to
# settle, but the profile leaves the served set, so later boot never starts it.
_withdraw_profile_home(runner, home)
- park_profile(runner, name, f'startup recovery failed; ownership retained: {exc}')
+ park_profile(runner, name, f'startup recovery failed; ownership retained (a control-socket rescan, '
+ f'`hermes -p {name} gateway run`, serves it once its writer stops): {exc}')
return
registry.remove(home)
_publish_served_set(runner)
release_profile_home(runner, home)
- park_profile(runner, name, f'startup recovery failed: {exc}')
+ park_profile(runner, name, f'startup recovery failed (a control-socket rescan, '
+ f'`hermes -p {name} gateway run`, serves it again): {exc}')
async def recover_gateway_native_sessions(runner):
diff --git a/gateway/session_bot.py b/gateway/session_bot.py
index 24132fd640f..57f7aa47790 100644
--- a/gateway/session_bot.py
+++ b/gateway/session_bot.py
@@ -283,7 +283,14 @@ def _expired_outcome(authority, record):
if record.get('status') in _TERMINAL:
return record['status'], record.get('reply', '')
session_id = record.get('session_id')
- if session_id and (retired_session(authority.db, session_id) or authority.db.get_session(session_id) is None):
+ if session_id and retired_session(authority.db, session_id):
+ return 'ambiguous', record.get('reply', '')
+ if session_id and authority.db.get_session(session_id) is None:
+ # Never resent either way, but a row gone without its fence is not a plain delete: store damage,
+ # or a marker an older release reclaimed. Logged once: the receipt is final from here on.
+ logging.getLogger(__name__).warning(
+ 'Bot delivery %s: session %s is gone with no deletion marker; settled as ambiguous',
+ record.get('delivery_id'), session_id)
return 'ambiguous', record.get('reply', '')
raise RuntimeStoreError('storage_unavailable')
@@ -560,8 +567,8 @@ async def recover_bot_deliveries(authority):
# One receipt the store cannot answer must not stop the owner (and, multiplexed, every
# profile with it): log it and recover the rest. The next start reads it again (a
# retry already pinned on it is then admitted once).
- logging.getLogger(__name__).error('Bot delivery %s not recovered: %s',
- record.get('delivery_id'), exc.reason)
+ logging.getLogger(__name__).error('Bot delivery %s not recovered: %s (%s)',
+ record.get('delivery_id'), exc.reason, path, exc_info=True)
continue
await write_receipt(home, record)
remember_receipt(authority, home, record)
diff --git a/tests/gateway/test_bot_receipt_recovery_logs.py b/tests/gateway/test_bot_receipt_recovery_logs.py
new file mode 100644
index 00000000000..50485c6a527
--- /dev/null
+++ b/tests/gateway/test_bot_receipt_recovery_logs.py
@@ -0,0 +1,49 @@
+"""Startup Bot receipt recovery says what it could not answer, and where.
+
+A receipt the store cannot answer is logged with its file and traceback (the next start reads it
+again). A pending receipt whose session row is gone with no deletion marker is settled as
+``ambiguous`` like a deleted chat's, but that is store damage, so it is logged, not silent.
+"""
+import asyncio
+from types import SimpleNamespace
+
+from hermes_state import SessionDB
+from hermes_state_runtime import begin_runtime_epoch
+from tools.bot_live_delivery import _locked, _read, _write
+
+
+def _recover_one(tmp_path, record):
+ from gateway.session_bot import recover_bot_deliveries
+ db = SessionDB(tmp_path / 'state.db')
+ try:
+ begin_runtime_epoch(db, instance_id='owner')
+ if record['session_id'] == 'live':
+ db.create_session('live', source='cli')
+ with _locked(tmp_path) as root:
+ _write(root / f"{record['delivery_id']}.json", record)
+ asyncio.run(recover_bot_deliveries(SimpleNamespace(db=db, waiters={})))
+ with _locked(tmp_path) as root:
+ return _read(root / f"{record['delivery_id']}.json")
+ finally:
+ db.close()
+
+
+def _record(key, session_id):
+ return dict(status='queued', delivery_id=key, profile_home='', session_id=session_id,
+ admission_id='missing', principal_id='owner', message='ping')
+
+
+def test_an_unanswerable_receipt_is_logged_with_its_file_and_traceback(tmp_path, caplog):
+ key = 'c' * 32
+ record = dict(_record(key, 'live'), profile_home=str(tmp_path))
+ assert _recover_one(tmp_path, record)['status'] == 'queued'
+ line = next(r for r in caplog.records if 'not recovered' in r.getMessage())
+ assert f'{key}.json' in line.getMessage() and line.exc_info is not None, line.getMessage()
+
+
+def test_a_receipt_whose_session_vanished_without_a_marker_is_logged(tmp_path, caplog):
+ key = 'd' * 32
+ record = dict(_record(key, 'ghost'), profile_home=str(tmp_path))
+ assert _recover_one(tmp_path, record)['status'] == 'ambiguous'
+ assert any('ghost' in r.getMessage() and 'no deletion marker' in r.getMessage() for r in caplog.records), \
+ caplog.text
diff --git a/tests/gateway/test_startup_recovery_parks_profile.py b/tests/gateway/test_startup_recovery_parks_profile.py
index 9e793d22c76..25039edd98e 100644
--- a/tests/gateway/test_startup_recovery_parks_profile.py
+++ b/tests/gateway/test_startup_recovery_parks_profile.py
@@ -64,6 +64,7 @@ def test_a_secondary_whose_recovery_fails_is_parked_and_the_rest_start(tmp_path,
assert runner.session_runtime_descriptor["api_origin"] == "http://127.0.0.1:1"
assert recovered == [str(launch), str(homes["beta"]), str(homes["gamma"])]
assert "beta" in parked and "leftover receipt" in parked["beta"]
+ assert "rescan" in parked["beta"], "the parked reason does not say how the profile comes back"
assert runner.session_authorities.for_home(homes["beta"]) is None
assert not process_ownership.owns(homes["beta"]), "a parked profile keeps its reservation"
assert [n for n, _ in runner.config._runtime_profile_homes] == ["gamma"]
@@ -91,7 +92,7 @@ def test_a_secondary_whose_retirement_times_out_keeps_its_lock_but_is_never_star
process_ownership.reserve([homes["beta"]])
try:
asyncio.run(run_runtime.start_gateway_runtime_api(runner))
- assert "ownership retained" in parked["beta"]
+ assert "ownership retained" in parked["beta"] and "rescan" in parked["beta"]
assert process_ownership.owns(homes["beta"]), "the live writer's home lost its lock"
assert runner.session_authorities.for_home(homes["beta"]) is beta and beta.retiring
assert beta in all_authorities(runner), "shutdown must still settle the retained authority"