Skip to content

The agent that grows with you

fix(gateway): keep booting after a deleted Bot Chat's tombstones expire

David Dudok de Witbd82a5a5cbff
diff --git a/gateway/platforms/api_server_response_admissions.py b/gateway/platforms/api_server_response_admissions.py
index b08789e9bad..a2e6d92ea4d 100644
--- a/gateway/platforms/api_server_response_admissions.py
+++ b/gateway/platforms/api_server_response_admissions.py
@@ -45,7 +45,8 @@ def find_admission(adapter, durable_key, request_id):
if admission_id:
row = get_session_admission(authority.db, admission_id=admission_id)
if row is None:
- raise RuntimeStoreError('storage_unavailable')
+ # The admission's deletion tombstone aged out: the key is spent, never a new request.
+ raise RuntimeStoreError('admission_conflict')
else:
rows = _request_admission_rows(authority.db, request_id)
if len(rows) > 1:
diff --git a/gateway/session_bot.py b/gateway/session_bot.py
index 60dfd500111..621c4384b59 100644
--- a/gateway/session_bot.py
+++ b/gateway/session_bot.py
@@ -15,7 +15,7 @@ from gateway.session_contract import Principal, SessionRef
from gateway.config import Platform
from gateway.platforms.event import MessageEvent
from hermes_state_runtime import RuntimeStoreError, get_session_admission
-from tools.bot_live_delivery import _delivery_id, _read
+from tools.bot_live_delivery import _TERMINAL, _delivery_id, _read
from gateway.session_bot_mailbox import (mailbox_lock, read_receipt, write_receipt,
scan_receipts, remember_receipt, track_receipt_task, withdraw_pin, refuse_pin, requeue_pin)
@@ -249,7 +249,7 @@ def _admission_outcome(authority, admission_id, fallback=None):
"""``(status, reply)`` of one admission, read from its committed result, never transcript recency."""
row = get_session_admission(authority.db, admission_id=admission_id)
if row is None:
- raise RuntimeStoreError('storage_unavailable')
+ return _expired_outcome(authority, fallback)
# A terminal row without a result blob is still definitive when the outcome says the
# input never ran (cancelled) or was refused/failed; ambiguous is reserved for the
# durable unknown state and for missing evidence (completed/interrupted without result).
@@ -272,6 +272,21 @@ def _admission_outcome(authority, admission_id, fallback=None):
return status, reply
+def _expired_outcome(authority, record):
+ """A receipt outlives its admission once a deleted target's tombstone passes the retry horizon
+ (``prune_retired_tombstones``). A final receipt keeps the answer it already holds; a pending one
+ whose session was deleted (retired, or its row gone) can never learn its outcome and becomes
+ ``ambiguous``, never resent. A live session missing its admission is still a storage fault."""
+ from hermes_state_mutation_retirement import retired_session
+ record = record or {}
+ if record.get('status') in _TERMINAL:
+ return record['status'], record.get('reply', '')
+ session_id = record.get('session_id')
+ if session_id and (retired_session(authority.db, session_id) or authority.db.get_session(session_id) is None):
+ return 'ambiguous', record.get('reply', '')
+ raise RuntimeStoreError('storage_unavailable')
+
+
def _retry_admission(authority, record):
"""The delivery's one retry admission id, read from the FIFO by its derived identity, so every
reader (an in-memory record, the receipt file, an owner restart) follows the same retry."""
diff --git a/hermes_state_mutation_retirement.py b/hermes_state_mutation_retirement.py
index e3eb636909b..cc0e08d117f 100644
--- a/hermes_state_mutation_retirement.py
+++ b/hermes_state_mutation_retirement.py
@@ -216,10 +216,12 @@ _EXPIRED = "json_valid(value) AND json_extract(value, '$.retired_at') < ?"
def prune_retired_tombstones(conn, older_than=RETRY_HORIZON_S):
"""Reclaim deletion tombstones retired more than *older_than* seconds ago: the terminal admission
- with its identity pointer and compacted result, terminal workers and the retired-session marker.
+ with its identity pointer and compacted result, and terminal workers. The retired-session marker
+ is the deletion fence itself and is never reclaimed: it holds only ``retired_at``, and without it
+ a deleted id could be recreated, re-admitted or reissued as a Bot Chat generation.
Tombstones written before ``retired_at`` existed are stamped now and age from this pass."""
from hermes_state_terminal import ADMISSION_PREFIX, RESULT_PREFIX, WORKER_PREFIX, identity_key
- prefixes = (ADMISSION_PREFIX, WORKER_PREFIX, RETIRED_PREFIX)
+ prefixes = (ADMISSION_PREFIX, WORKER_PREFIX)
now = time.time()
for prefix in prefixes:
conn.execute("UPDATE state_meta SET value=json_set(value, '$.retired_at', ?) WHERE key GLOB ? "
@@ -230,5 +232,4 @@ def prune_retired_tombstones(conn, older_than=RETRY_HORIZON_S):
row = json.loads(raw)
conn.execute('DELETE FROM state_meta WHERE key IN (?,?,?)', (key, RESULT_PREFIX + row['admission_id'],
identity_key(row['principal_id'], row['target_session_id'], row['request_id'])))
- for prefix in prefixes[1:]:
- conn.execute(f'DELETE FROM state_meta WHERE key GLOB ? AND {_EXPIRED}', (prefix + '*', cutoff))
+ conn.execute(f'DELETE FROM state_meta WHERE key GLOB ? AND {_EXPIRED}', (WORKER_PREFIX + '*', cutoff))
diff --git a/tests/gateway/test_bot_receipt_tombstone_expiry.py b/tests/gateway/test_bot_receipt_tombstone_expiry.py
new file mode 100644
index 00000000000..fcf9d0cb15e
--- /dev/null
+++ b/tests/gateway/test_bot_receipt_tombstone_expiry.py
@@ -0,0 +1,128 @@
+"""A deleted session's admission tombstones age out after the retry horizon; the receipts and
+pointers that name them do not. None of them may turn into ``storage_unavailable``: a Bot receipt
+re-read at every owner start would otherwise abort the gateway on every boot."""
+import asyncio
+import time
+from types import SimpleNamespace
+
+import pytest
+from aiohttp import web
+from aiohttp.test_utils import TestClient, TestServer
+
+import hermes_state_runtime as rt
+from gateway.session_results import retain_result
+from hermes_state import SessionDB
+from tools.bot_live_delivery import _locked, _read, _write
+
+KEY = "a" * 32
+
+
+def _delete(db, epoch, session_id, *, days_ago):
+ snap = db.get_session(session_id)
+ at = time.time() - days_ago * 86400
+ with pytest.MonkeyPatch.context() as clock:
+ clock.setattr(time, "time", lambda: at)
+ rt.mutate_runtime_session(db, epoch=epoch, principal_id="human", session_id=session_id,
+ request_id="del", operation="delete", payload={},
+ expected_revision=snap["runtime_revision"],
+ expected_generation=snap["runtime_generation"])
+
+
+def _receipt_then_deleted(tmp_path, *, status, days_ago, drop_marker=False):
+ db = SessionDB(tmp_path / "state.db")
+ epoch = rt.begin_runtime_epoch(db, instance_id="owner")
+ db.create_session("botchat", source="cli")
+ admission = rt.admit_session_input(db, epoch=epoch, principal_id="owner", session_id="botchat",
+ request_id="bot:" + KEY, payload={"text": "ping"})
+ row = rt.claim_session_input(db, epoch=epoch, session_id="botchat")
+ retain_result(db, epoch=epoch, row=row, result={"result": {"final_response": "pong"}, "usage": {}})
+ record = dict(status=status, delivery_id=KEY, profile_home=str(tmp_path), session_id="botchat",
+ admission_id=admission["admission_id"], principal_id="owner", message="ping")
+ if status == "settled":
+ record["reply"] = "pong"
+ with _locked(tmp_path) as root:
+ _write(root / f"{KEY}.json", record)
+ _delete(db, epoch, "botchat", days_ago=days_ago)
+ db.prune_sessions(older_than_days=90)
+ if drop_marker: # a store whose marker an earlier release's prune already reclaimed
+ db._execute_write(lambda conn: conn.execute(
+ "DELETE FROM state_meta WHERE key GLOB 'gateway.retired_session.v1.*'"))
+ return db
+
+
+def _receipt(tmp_path):
+ with _locked(tmp_path) as root:
+ return _read(root / f"{KEY}.json")
+
+
[email protected]("days_ago", [6, 8])
+def test_a_settled_receipt_keeps_its_answer_after_its_chat_is_deleted(tmp_path, days_ago):
+ from gateway.session_bot import recover_bot_deliveries
+ db = _receipt_then_deleted(tmp_path, status="settled", days_ago=days_ago)
+ try:
+ asyncio.run(recover_bot_deliveries(SimpleNamespace(db=db, waiters={})))
+ record = _receipt(tmp_path)
+ assert (record["status"], record["reply"]) == ("settled", "pong")
+ finally:
+ db.close()
+
+
[email protected]("status", ["queued", "claimed", "canonical"])
[email protected]("drop_marker", [False, True])
+def test_a_pending_receipt_behind_an_expired_deleted_chat_becomes_ambiguous(tmp_path, status, drop_marker):
+ from gateway.session_bot import recover_bot_deliveries
+ db = _receipt_then_deleted(tmp_path, status=status, days_ago=8, drop_marker=drop_marker)
+ try:
+ asyncio.run(recover_bot_deliveries(SimpleNamespace(db=db, waiters={})))
+ assert _receipt(tmp_path)["status"] == "ambiguous"
+ finally:
+ db.close()
+
+
+def test_the_runtime_api_starts_after_a_deleted_bot_chat_expired(tmp_path, monkeypatch):
+ import gateway.run_api
+ from gateway.run_runtime import start_gateway_runtime_api
+ db = _receipt_then_deleted(tmp_path, status="settled", days_ago=8)
+ try:
+ async def fake_api(runner):
+ return SimpleNamespace(api_origin="http://127.0.0.1:1")
+ monkeypatch.setattr(gateway.run_api, "start_gateway_api", fake_api)
+ authority = SimpleNamespace(db=db, waiters={}, runner=None, profile_id=str(tmp_path))
+ runner = SimpleNamespace(session_authority=authority, session_runtime_descriptor={})
+ asyncio.run(start_gateway_runtime_api(runner))
+ assert runner.session_runtime_descriptor["api_origin"] == "http://127.0.0.1:1"
+ finally:
+ db.close()
+
+
[email protected]("days_ago, expected", [(6, 200), (8, 409)])
+async def test_a_responses_key_of_a_deleted_session_never_reruns_and_never_503s(
+ api, owner, days_ago, expected):
+ calls = []
+
+ async def handle(event):
+ from gateway.session_results import execution_result
+ calls.append(event.text)
+ execution_result.get()["result"] = {"final_response": "answer", "messages": []}
+ return "answer"
+
+ owner.runner._handle_message = handle
+ app = web.Application()
+ app.router.add_post("/v1/responses", api._handle_responses)
+ body, headers = {"input": "once-only"}, {"Idempotency-Key": "aged-key"}
+ async with TestClient(TestServer(app)) as client:
+ assert (await client.post("/v1/responses", json=body, headers=headers)).status == 200
+ store = api._current_response_store()
+ # The replay body was evicted; the compact key -> admission pointer stays.
+ store._conn.execute("DELETE FROM responses WHERE response_id LIKE 'idem:%'")
+ store._conn.commit()
+ sid = owner.db._read_one("SELECT * FROM session_admissions")["target_session_id"]
+ at = time.time() - days_ago * 86400
+ with pytest.MonkeyPatch.context() as clock:
+ clock.setattr(time, "time", lambda: at)
+ assert owner.db.delete_session(sid)
+ owner.db.prune_sessions(older_than_days=90)
+ retry = await client.post("/v1/responses", json=body, headers=headers)
+ assert retry.status == expected, await retry.text()
+ assert calls == ["once-only"]
diff --git a/tests/hermes_state/test_retired_fence_outlives_horizon.py b/tests/hermes_state/test_retired_fence_outlives_horizon.py
new file mode 100644
index 00000000000..e8c3d9923fc
--- /dev/null
+++ b/tests/hermes_state/test_retired_fence_outlives_horizon.py
@@ -0,0 +1,52 @@
+"""The retired-session marker is the deletion fence: the prune pass reclaims a deleted session's
+tombstones after the retry horizon, but never the marker, so the id stays deleted."""
+import time
+from types import SimpleNamespace
+
+import pytest
+
+import hermes_state_runtime as rt
+from gateway.session_results import retain_result
+from hermes_state import SessionDB
+
+
+def _delete_8_days_ago(db, epoch, sid):
+ snap = db.get_session(sid)
+ at = time.time() - 8 * 86400
+ with pytest.MonkeyPatch.context() as clock:
+ clock.setattr(time, "time", lambda: at)
+ rt.mutate_runtime_session(db, epoch=epoch, principal_id="human", session_id=sid, request_id="del",
+ operation="delete", payload={}, expected_revision=snap["runtime_revision"],
+ expected_generation=snap["runtime_generation"])
+
+
+def test_a_deleted_session_is_never_recreated_after_the_horizon(tmp_path):
+ from hermes_state_mutation_retirement import RETIRED_PREFIX
+ with SessionDB(tmp_path / "state.db") as db:
+ epoch = rt.begin_runtime_epoch(db, instance_id="owner")
+ db.create_session("s", source="api_server")
+ args = dict(principal_id="api", session_id="s", request_id="r1", payload={"text": "charge card"})
+ rt.admit_session_input(db, epoch=epoch, **args)
+ row = rt.claim_session_input(db, epoch=epoch, session_id="s")
+ retain_result(db, epoch=epoch, row=row, result={"result": {"final_response": "done"}, "usage": {}})
+ _delete_8_days_ago(db, epoch, "s")
+ db.prune_sessions(older_than_days=90)
+ assert db.get_meta(RETIRED_PREFIX + "s") is not None
+ with pytest.raises(rt.RuntimeStoreError, match="not_found"):
+ db.ensure_session("s", source="unknown") # a late backfill
+ with pytest.raises(rt.RuntimeStoreError, match="not_found"):
+ rt.admit_session_input(db, epoch=epoch, **args) # the old exact retry
+ assert db.get_session("s") is None
+
+
+def test_a_deleted_bot_chat_generation_is_never_reissued(tmp_path):
+ from gateway.session_bot import _bot_chat_creation_id
+ from gateway.session_local_recovery import local_identity
+ with SessionDB(tmp_path / "state.db") as db:
+ epoch = rt.begin_runtime_epoch(db, instance_id="owner")
+ authority, actor = SimpleNamespace(profile_id="owned", db=db), SimpleNamespace(subject="owner")
+ sid = local_identity("owned", "owner", "bot-chat:owned")
+ db.create_session(sid, source="cli")
+ _delete_8_days_ago(db, epoch, sid)
+ db.prune_sessions(older_than_days=90)
+ assert _bot_chat_creation_id(authority, actor) == "bot-chat:owned:1"
diff --git a/tests/hermes_state/test_terminal_retirement.py b/tests/hermes_state/test_terminal_retirement.py
index 55949a04b6b..32c4341c850 100644
--- a/tests/hermes_state/test_terminal_retirement.py
+++ b/tests/hermes_state/test_terminal_retirement.py
@@ -333,7 +333,7 @@ def test_retirement_drops_the_turn_user_row_copy(tmp_path):
def test_tombstones_past_the_retry_horizon_are_reclaimed_by_the_prune_pass(tmp_path, monkeypatch):
"""Deletion tombstones keep byte-exact retry replies for the 7-day horizon, then the maintenance
- prune pass reclaims every row they hold (B2-06)."""
+ prune pass reclaims every row they hold but the deletion fence itself (B2-06)."""
import time
import hermes_state_mutation_retirement as retirement
now = time.time()
@@ -365,7 +365,8 @@ def test_tombstones_past_the_retry_horizon_are_reclaimed_by_the_prune_pass(tmp_p
assert len(rows_naming('old', old_id)) == 4
db.prune_sessions(older_than_days=90)
assert len(rows_naming('young', young_id)) == 4
- assert rows_naming('old', old_id) == [], 'tombstones past the horizon were not reclaimed'
+ assert rows_naming('old', old_id) == [retirement.RETIRED_PREFIX + 'old'], \
+ 'tombstones past the horizon were not reclaimed (the deletion fence stays)'
with pytest.raises(rt.RuntimeStoreError, match='not_found'):
rt.admit_session_input(db, epoch=epoch, **old)
replay = rt.admit_session_input(db, epoch=epoch, **young)