The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): keep booting after a deleted Bot Chat's tombstones expire
David Dudok de Witbd82a5a5cbff
diff --git a/gateway/platforms/api_server_response_admissions.py b/gateway/platforms/api_server_response_admissions.pyindex b08789e9bad..a2e6d92ea4d 100644--- a/gateway/platforms/api_server_response_admissions.py+++ b/gateway/platforms/api_server_response_admissions.py@@ -45,7 +45,8 @@ def find_admission(adapter, durable_key, request_id):if admission_id:row = get_session_admission(authority.db, admission_id=admission_id)if row is None:- raise RuntimeStoreError('storage_unavailable')+ # The admission's deletion tombstone aged out: the key is spent, never a new request.+ raise RuntimeStoreError('admission_conflict')else:rows = _request_admission_rows(authority.db, request_id)if len(rows) > 1:diff --git a/gateway/session_bot.py b/gateway/session_bot.pyindex 60dfd500111..621c4384b59 100644--- a/gateway/session_bot.py+++ b/gateway/session_bot.py@@ -15,7 +15,7 @@ from gateway.session_contract import Principal, SessionReffrom gateway.config import Platformfrom gateway.platforms.event import MessageEventfrom hermes_state_runtime import RuntimeStoreError, get_session_admission-from tools.bot_live_delivery import _delivery_id, _read+from tools.bot_live_delivery import _TERMINAL, _delivery_id, _readfrom gateway.session_bot_mailbox import (mailbox_lock, read_receipt, write_receipt,scan_receipts, remember_receipt, track_receipt_task, withdraw_pin, refuse_pin, requeue_pin)@@ -249,7 +249,7 @@ def _admission_outcome(authority, admission_id, fallback=None):"""``(status, reply)`` of one admission, read from its committed result, never transcript recency."""row = get_session_admission(authority.db, admission_id=admission_id)if row is None:- raise RuntimeStoreError('storage_unavailable')+ return _expired_outcome(authority, fallback)# A terminal row without a result blob is still definitive when the outcome says the# input never ran (cancelled) or was refused/failed; ambiguous is reserved for the# durable unknown state and for missing evidence (completed/interrupted without result).@@ -272,6 +272,21 @@ def _admission_outcome(authority, admission_id, fallback=None):return status, reply+def _expired_outcome(authority, record):+ """A receipt outlives its admission once a deleted target's tombstone passes the retry horizon+ (``prune_retired_tombstones``). A final receipt keeps the answer it already holds; a pending one+ whose session was deleted (retired, or its row gone) can never learn its outcome and becomes+ ``ambiguous``, never resent. A live session missing its admission is still a storage fault."""+ from hermes_state_mutation_retirement import retired_session+ record = record or {}+ if record.get('status') in _TERMINAL:+ return record['status'], record.get('reply', '')+ session_id = record.get('session_id')+ if session_id and (retired_session(authority.db, session_id) or authority.db.get_session(session_id) is None):+ return 'ambiguous', record.get('reply', '')+ raise RuntimeStoreError('storage_unavailable')++def _retry_admission(authority, record):"""The delivery's one retry admission id, read from the FIFO by its derived identity, so everyreader (an in-memory record, the receipt file, an owner restart) follows the same retry."""diff --git a/hermes_state_mutation_retirement.py b/hermes_state_mutation_retirement.pyindex e3eb636909b..cc0e08d117f 100644--- a/hermes_state_mutation_retirement.py+++ b/hermes_state_mutation_retirement.py@@ -216,10 +216,12 @@ _EXPIRED = "json_valid(value) AND json_extract(value, '$.retired_at') < ?"def prune_retired_tombstones(conn, older_than=RETRY_HORIZON_S):"""Reclaim deletion tombstones retired more than *older_than* seconds ago: the terminal admission- with its identity pointer and compacted result, terminal workers and the retired-session marker.+ with its identity pointer and compacted result, and terminal workers. The retired-session marker+ is the deletion fence itself and is never reclaimed: it holds only ``retired_at``, and without it+ a deleted id could be recreated, re-admitted or reissued as a Bot Chat generation.Tombstones written before ``retired_at`` existed are stamped now and age from this pass."""from hermes_state_terminal import ADMISSION_PREFIX, RESULT_PREFIX, WORKER_PREFIX, identity_key- prefixes = (ADMISSION_PREFIX, WORKER_PREFIX, RETIRED_PREFIX)+ prefixes = (ADMISSION_PREFIX, WORKER_PREFIX)now = time.time()for prefix in prefixes:conn.execute("UPDATE state_meta SET value=json_set(value, '$.retired_at', ?) WHERE key GLOB ? "@@ -230,5 +232,4 @@ def prune_retired_tombstones(conn, older_than=RETRY_HORIZON_S):row = json.loads(raw)conn.execute('DELETE FROM state_meta WHERE key IN (?,?,?)', (key, RESULT_PREFIX + row['admission_id'],identity_key(row['principal_id'], row['target_session_id'], row['request_id'])))- for prefix in prefixes[1:]:- conn.execute(f'DELETE FROM state_meta WHERE key GLOB ? AND {_EXPIRED}', (prefix + '*', cutoff))+ conn.execute(f'DELETE FROM state_meta WHERE key GLOB ? AND {_EXPIRED}', (WORKER_PREFIX + '*', cutoff))diff --git a/tests/gateway/test_bot_receipt_tombstone_expiry.py b/tests/gateway/test_bot_receipt_tombstone_expiry.pynew file mode 100644index 00000000000..fcf9d0cb15e--- /dev/null+++ b/tests/gateway/test_bot_receipt_tombstone_expiry.py@@ -0,0 +1,128 @@+"""A deleted session's admission tombstones age out after the retry horizon; the receipts and+pointers that name them do not. None of them may turn into ``storage_unavailable``: a Bot receipt+re-read at every owner start would otherwise abort the gateway on every boot."""+import asyncio+import time+from types import SimpleNamespace++import pytest+from aiohttp import web+from aiohttp.test_utils import TestClient, TestServer++import hermes_state_runtime as rt+from gateway.session_results import retain_result+from hermes_state import SessionDB+from tools.bot_live_delivery import _locked, _read, _write++KEY = "a" * 32+++def _delete(db, epoch, session_id, *, days_ago):+ snap = db.get_session(session_id)+ at = time.time() - days_ago * 86400+ with pytest.MonkeyPatch.context() as clock:+ clock.setattr(time, "time", lambda: at)+ rt.mutate_runtime_session(db, epoch=epoch, principal_id="human", session_id=session_id,+ request_id="del", operation="delete", payload={},+ expected_revision=snap["runtime_revision"],+ expected_generation=snap["runtime_generation"])+++def _receipt_then_deleted(tmp_path, *, status, days_ago, drop_marker=False):+ db = SessionDB(tmp_path / "state.db")+ epoch = rt.begin_runtime_epoch(db, instance_id="owner")+ db.create_session("botchat", source="cli")+ admission = rt.admit_session_input(db, epoch=epoch, principal_id="owner", session_id="botchat",+ request_id="bot:" + KEY, payload={"text": "ping"})+ row = rt.claim_session_input(db, epoch=epoch, session_id="botchat")+ retain_result(db, epoch=epoch, row=row, result={"result": {"final_response": "pong"}, "usage": {}})+ record = dict(status=status, delivery_id=KEY, profile_home=str(tmp_path), session_id="botchat",+ admission_id=admission["admission_id"], principal_id="owner", message="ping")+ if status == "settled":+ record["reply"] = "pong"+ with _locked(tmp_path) as root:+ _write(root / f"{KEY}.json", record)+ _delete(db, epoch, "botchat", days_ago=days_ago)+ db.prune_sessions(older_than_days=90)+ if drop_marker: # a store whose marker an earlier release's prune already reclaimed+ db._execute_write(lambda conn: conn.execute(+ "DELETE FROM state_meta WHERE key GLOB 'gateway.retired_session.v1.*'"))+ return db+++def _receipt(tmp_path):+ with _locked(tmp_path) as root:+ return _read(root / f"{KEY}.json")++[email protected]("days_ago", [6, 8])+def test_a_settled_receipt_keeps_its_answer_after_its_chat_is_deleted(tmp_path, days_ago):+ from gateway.session_bot import recover_bot_deliveries+ db = _receipt_then_deleted(tmp_path, status="settled", days_ago=days_ago)+ try:+ asyncio.run(recover_bot_deliveries(SimpleNamespace(db=db, waiters={})))+ record = _receipt(tmp_path)+ assert (record["status"], record["reply"]) == ("settled", "pong")+ finally:+ db.close()++[email protected]("status", ["queued", "claimed", "canonical"])[email protected]("drop_marker", [False, True])+def test_a_pending_receipt_behind_an_expired_deleted_chat_becomes_ambiguous(tmp_path, status, drop_marker):+ from gateway.session_bot import recover_bot_deliveries+ db = _receipt_then_deleted(tmp_path, status=status, days_ago=8, drop_marker=drop_marker)+ try:+ asyncio.run(recover_bot_deliveries(SimpleNamespace(db=db, waiters={})))+ assert _receipt(tmp_path)["status"] == "ambiguous"+ finally:+ db.close()+++def test_the_runtime_api_starts_after_a_deleted_bot_chat_expired(tmp_path, monkeypatch):+ import gateway.run_api+ from gateway.run_runtime import start_gateway_runtime_api+ db = _receipt_then_deleted(tmp_path, status="settled", days_ago=8)+ try:+ async def fake_api(runner):+ return SimpleNamespace(api_origin="http://127.0.0.1:1")+ monkeypatch.setattr(gateway.run_api, "start_gateway_api", fake_api)+ authority = SimpleNamespace(db=db, waiters={}, runner=None, profile_id=str(tmp_path))+ runner = SimpleNamespace(session_authority=authority, session_runtime_descriptor={})+ asyncio.run(start_gateway_runtime_api(runner))+ assert runner.session_runtime_descriptor["api_origin"] == "http://127.0.0.1:1"+ finally:+ db.close()++[email protected]("days_ago, expected", [(6, 200), (8, 409)])+async def test_a_responses_key_of_a_deleted_session_never_reruns_and_never_503s(+ api, owner, days_ago, expected):+ calls = []++ async def handle(event):+ from gateway.session_results import execution_result+ calls.append(event.text)+ execution_result.get()["result"] = {"final_response": "answer", "messages": []}+ return "answer"++ owner.runner._handle_message = handle+ app = web.Application()+ app.router.add_post("/v1/responses", api._handle_responses)+ body, headers = {"input": "once-only"}, {"Idempotency-Key": "aged-key"}+ async with TestClient(TestServer(app)) as client:+ assert (await client.post("/v1/responses", json=body, headers=headers)).status == 200+ store = api._current_response_store()+ # The replay body was evicted; the compact key -> admission pointer stays.+ store._conn.execute("DELETE FROM responses WHERE response_id LIKE 'idem:%'")+ store._conn.commit()+ sid = owner.db._read_one("SELECT * FROM session_admissions")["target_session_id"]+ at = time.time() - days_ago * 86400+ with pytest.MonkeyPatch.context() as clock:+ clock.setattr(time, "time", lambda: at)+ assert owner.db.delete_session(sid)+ owner.db.prune_sessions(older_than_days=90)+ retry = await client.post("/v1/responses", json=body, headers=headers)+ assert retry.status == expected, await retry.text()+ assert calls == ["once-only"]diff --git a/tests/hermes_state/test_retired_fence_outlives_horizon.py b/tests/hermes_state/test_retired_fence_outlives_horizon.pynew file mode 100644index 00000000000..e8c3d9923fc--- /dev/null+++ b/tests/hermes_state/test_retired_fence_outlives_horizon.py@@ -0,0 +1,52 @@+"""The retired-session marker is the deletion fence: the prune pass reclaims a deleted session's+tombstones after the retry horizon, but never the marker, so the id stays deleted."""+import time+from types import SimpleNamespace++import pytest++import hermes_state_runtime as rt+from gateway.session_results import retain_result+from hermes_state import SessionDB+++def _delete_8_days_ago(db, epoch, sid):+ snap = db.get_session(sid)+ at = time.time() - 8 * 86400+ with pytest.MonkeyPatch.context() as clock:+ clock.setattr(time, "time", lambda: at)+ rt.mutate_runtime_session(db, epoch=epoch, principal_id="human", session_id=sid, request_id="del",+ operation="delete", payload={}, expected_revision=snap["runtime_revision"],+ expected_generation=snap["runtime_generation"])+++def test_a_deleted_session_is_never_recreated_after_the_horizon(tmp_path):+ from hermes_state_mutation_retirement import RETIRED_PREFIX+ with SessionDB(tmp_path / "state.db") as db:+ epoch = rt.begin_runtime_epoch(db, instance_id="owner")+ db.create_session("s", source="api_server")+ args = dict(principal_id="api", session_id="s", request_id="r1", payload={"text": "charge card"})+ rt.admit_session_input(db, epoch=epoch, **args)+ row = rt.claim_session_input(db, epoch=epoch, session_id="s")+ retain_result(db, epoch=epoch, row=row, result={"result": {"final_response": "done"}, "usage": {}})+ _delete_8_days_ago(db, epoch, "s")+ db.prune_sessions(older_than_days=90)+ assert db.get_meta(RETIRED_PREFIX + "s") is not None+ with pytest.raises(rt.RuntimeStoreError, match="not_found"):+ db.ensure_session("s", source="unknown") # a late backfill+ with pytest.raises(rt.RuntimeStoreError, match="not_found"):+ rt.admit_session_input(db, epoch=epoch, **args) # the old exact retry+ assert db.get_session("s") is None+++def test_a_deleted_bot_chat_generation_is_never_reissued(tmp_path):+ from gateway.session_bot import _bot_chat_creation_id+ from gateway.session_local_recovery import local_identity+ with SessionDB(tmp_path / "state.db") as db:+ epoch = rt.begin_runtime_epoch(db, instance_id="owner")+ authority, actor = SimpleNamespace(profile_id="owned", db=db), SimpleNamespace(subject="owner")+ sid = local_identity("owned", "owner", "bot-chat:owned")+ db.create_session(sid, source="cli")+ _delete_8_days_ago(db, epoch, sid)+ db.prune_sessions(older_than_days=90)+ assert _bot_chat_creation_id(authority, actor) == "bot-chat:owned:1"diff --git a/tests/hermes_state/test_terminal_retirement.py b/tests/hermes_state/test_terminal_retirement.pyindex 55949a04b6b..32c4341c850 100644--- a/tests/hermes_state/test_terminal_retirement.py+++ b/tests/hermes_state/test_terminal_retirement.py@@ -333,7 +333,7 @@ def test_retirement_drops_the_turn_user_row_copy(tmp_path):def test_tombstones_past_the_retry_horizon_are_reclaimed_by_the_prune_pass(tmp_path, monkeypatch):"""Deletion tombstones keep byte-exact retry replies for the 7-day horizon, then the maintenance- prune pass reclaims every row they hold (B2-06)."""+ prune pass reclaims every row they hold but the deletion fence itself (B2-06)."""import timeimport hermes_state_mutation_retirement as retirementnow = time.time()@@ -365,7 +365,8 @@ def test_tombstones_past_the_retry_horizon_are_reclaimed_by_the_prune_pass(tmp_passert len(rows_naming('old', old_id)) == 4db.prune_sessions(older_than_days=90)assert len(rows_naming('young', young_id)) == 4- assert rows_naming('old', old_id) == [], 'tombstones past the horizon were not reclaimed'+ assert rows_naming('old', old_id) == [retirement.RETIRED_PREFIX + 'old'], \+ 'tombstones past the horizon were not reclaimed (the deletion fence stays)'with pytest.raises(rt.RuntimeStoreError, match='not_found'):rt.admit_session_input(db, epoch=epoch, **old)replay = rt.admit_session_input(db, epoch=epoch, **young)