Skip to content

The agent that grows with you

fix(gateway): refuse a chain successor whose walk lost a fence before its bind

unsupportedpastels8efcf4880673
diff --git a/gateway/platforms/api_server_authority_runs.py b/gateway/platforms/api_server_authority_runs.py
index 7be5495d9b4..136e49f6c6a 100644
--- a/gateway/platforms/api_server_authority_runs.py
+++ b/gateway/platforms/api_server_authority_runs.py
@@ -13,8 +13,10 @@ def _authority(adapter):
# Malformed input is 400 and a retryable refusal 503; every other reason (ownership, generation,
-# identity) keeps the documented 409, as /v1/runs clients already branch on it.
-_STORE_ERROR_STATUS = {'invalid_params': 400, 'runtime_draining': 503, 'storage_unavailable': 503}
+# identity) keeps the documented 409, as /v1/runs clients already branch on it. ``chain_moved``: a
+# chain's lookup went stale before its bind (``bind_api_steps``); a retry looks it up again.
+_STORE_ERROR_STATUS = {'invalid_params': 400, 'runtime_draining': 503, 'storage_unavailable': 503,
+ 'chain_moved': 503}
def store_error_response(_openai_error, exc):
diff --git a/gateway/session_api.py b/gateway/session_api.py
index 78f300cbd74..60534208809 100644
--- a/gateway/session_api.py
+++ b/gateway/session_api.py
@@ -156,7 +156,8 @@ def bind_api_steps(authority, session_id, *, hosted_dispatch=None, declared_key=
registered as that session's declared key (and stamped into its binding receipt).
``walked_from`` is the stored session a Responses/Runs chain continued from when the chain moved
- past it to *session_id* (``continued_api_session``): the fences it passed stay for good."""
+ past it to *session_id* (``continued_api_session``): the fences it passed stay for good. If one
+ aged out since that lookup, the bind refuses with the retryable ``chain_moved``."""
authority._require_admission_open()
if not isinstance(session_id, str) or not session_id or _is_path_unsafe(session_id):
raise RuntimeStoreError('invalid_params')
@@ -201,8 +202,10 @@ def bind_api_steps(authority, session_id, *, hosted_dispatch=None, declared_key=
from hermes_state_mutation_retirement import RETIRED_PREFIX, keep_passed_fences
if conn.execute('SELECT 1 FROM state_meta WHERE key=?', (RETIRED_PREFIX + session_id,)).fetchone():
raise RuntimeStoreError('not_found')
- if walked_from:
- keep_passed_fences(conn, partial(api_generation_id, walked_from), session_id)
+ # The walk ran before this write: a sweep marker it passed may have been reclaimed since,
+ # and binding the successor then would split the chain (the next walk stops at the root).
+ if walked_from and not keep_passed_fences(conn, partial(api_generation_id, walked_from), session_id):
+ raise RuntimeStoreError('chain_moved')
saved = conn.execute('SELECT value FROM state_meta WHERE key=?',
(_BINDING_PREFIX + session_id,)).fetchone()
if saved is not None:
diff --git a/hermes_state_mutation_retirement.py b/hermes_state_mutation_retirement.py
index 38143ab4278..9f425ca2086 100644
--- a/hermes_state_mutation_retirement.py
+++ b/hermes_state_mutation_retirement.py
@@ -58,12 +58,16 @@ def keep_passed_fences(conn, derive, session_id):
"""Make permanent the fences a walk (``derive`` as for :func:`first_generation`) passed on its
way to *session_id*, inside the write that binds *session_id*: that successor now holds the
conversation, so a sweep marker aging out would send a later walk back to the passed
- generation. Stops at *session_id* or at the first generation without a marker."""
+ generation. False when a passed generation has lost its marker since the walk (a sweep's was
+ reclaimed): a walk now stops there, so *session_id* must not be bound as its successor."""
generation = 0
- while (sid := derive(generation)) != session_id and conn.execute(
- "UPDATE state_meta SET value=json_remove(value, '$.expires') WHERE key=? AND json_valid(value)",
- (RETIRED_PREFIX + sid,)).rowcount:
+ while (sid := derive(generation)) != session_id:
+ if not conn.execute('SELECT 1 FROM state_meta WHERE key=?', (RETIRED_PREFIX + sid,)).fetchone():
+ return False
+ conn.execute("UPDATE state_meta SET value=json_remove(value, '$.expires') WHERE key=? AND json_valid(value)",
+ (RETIRED_PREFIX + sid,))
generation += 1
+ return True
def has_mutation_receipt(db, principal_id, session_id, request_id):
diff --git a/tests/gateway/test_derived_generation_after_sweep.py b/tests/gateway/test_derived_generation_after_sweep.py
index 28159708c1c..df07222ae69 100644
--- a/tests/gateway/test_derived_generation_after_sweep.py
+++ b/tests/gateway/test_derived_generation_after_sweep.py
@@ -141,3 +141,62 @@ async def test_a_chain_resumed_after_a_sweep_never_returns_to_its_swept_root(api
forked = await client.post(path, json={'input': 'what number?', 'previous_response_id': first}, headers=auth)
assert forked.status in (200, 202), await forked.text()
assert owner.db.get_session(root) is None, 'the continuation went back to the swept root'
+
+
[email protected]('path', ['/v1/responses', '/v1/runs'])
+async def test_a_fence_reclaimed_between_chain_lookup_and_bind_never_splits_the_chain(
+ api, owner, served, path, monkeypatch):
+ """The lookup walks past the swept root to its next generation, then housekeeping reclaims the
+ root's sweep marker before that successor's bind. Binding it would leave the walk unpinned: the
+ next continuation would find the root free and land there (root, successor, root). The bind
+ re-checks the walk in its own write and refuses with a retryable ``chain_moved``; the chain then
+ stays on its root."""
+ from gateway.platforms import api_server_authority_runs as runs
+ from gateway.session_api import api_generation_id
+ app = web.Application()
+ app.router.add_post('/v1/responses', api._handle_responses)
+ app.router.add_post('/v1/runs', api._handle_runs)
+ auth = {'Authorization': 'Bearer ' + chat._API_KEY}
+ store = api._current_response_store()
+ async with TestClient(TestServer(app)) as client:
+ opened = await client.post('/v1/responses', json={'input': 'remember 42'}, headers=auth)
+ assert opened.status == 200, await opened.text()
+ first = (await opened.json())['id']
+ root = store.get(first)['session_id']
+ successor = api_generation_id(root, 1)
+ _sweep(owner, root)
+ lookup = runs.continued_session
+
+ async def reclaimed_after_lookup(*args, **kwargs):
+ found = await lookup(*args, **kwargs)
+ _past_the_horizon(owner)
+ return found
+ monkeypatch.setattr(runs, 'continued_session', reclaimed_after_lookup)
+ raced = await client.post(path, json={'input': 'and then?', 'previous_response_id': first}, headers=auth)
+ assert raced.status == 503, await raced.text()
+ assert (await raced.json())['error']['code'] == 'chain_moved'
+ assert raced.headers.get('Retry-After') == '1'
+ assert owner.db.get_session(successor) is None, 'the successor was bound without pinning its walk'
+ monkeypatch.setattr(runs, 'continued_session', lookup)
+ for turn in ('what number?', 'still there?'):
+ again = await client.post(path, json={'input': turn, 'previous_response_id': first}, headers=auth)
+ assert again.status in (200, 202), await again.text()
+ assert owner.db.get_session(root) is not None, 'the retried continuation left the root'
+ assert owner.db.get_session(successor) is None, 'the chain split across the root and its successor'
+
+
+def test_keep_passed_fences_reports_a_walk_it_could_not_pin(owner):
+ """``keep_passed_fences`` pins every fence a walk passed and says whether the walk still holds:
+ a passed generation without a marker means the walk would now stop there."""
+ import json
+ from functools import partial
+ from gateway.session_api import api_generation_id
+ from hermes_state_mutation_retirement import RETIRED_PREFIX, keep_passed_fences, retire_routes
+ derive = partial(api_generation_id, 'root-x')
+ passed = ['root-x', derive(1)]
+ owner.db._execute_write(lambda conn: retire_routes(conn, passed, expiring=passed))
+ assert owner.db._execute_write(lambda conn: keep_passed_fences(conn, derive, derive(2))) is True
+ assert all('expires' not in json.loads(owner.db.get_meta(RETIRED_PREFIX + sid)) for sid in passed)
+ owner.db._execute_write(lambda conn: conn.execute('DELETE FROM state_meta WHERE key=?', (RETIRED_PREFIX + passed[1],)))
+ assert owner.db._execute_write(lambda conn: keep_passed_fences(conn, derive, derive(2))) is False