Skip to content

The agent that grows with you

fix(api_server): Responses admission never waits on the store lock on the event loop

Teknium811701cca531
diff --git a/gateway/platforms/api_server_response_admissions.py b/gateway/platforms/api_server_response_admissions.py
index b26587e705a..b8d35fb64bc 100644
--- a/gateway/platforms/api_server_response_admissions.py
+++ b/gateway/platforms/api_server_response_admissions.py
@@ -24,17 +24,17 @@ def _request_admission_rows(db, request_id):
return rows
-def find_admission(adapter, durable_key, request_id):
+def _lookup_admission(authority, store, durable_key, request_id):
"""Scope comes from authenticated profile/key identity, never a caller-supplied session id.
+ -> ``(authority, ref, row)`` or None. Store and state.db reads only (plus the pointer repair):
+ run it under ``store.atomically`` in a worker, never on the loop, whose wait on the store lock
+ would last for a concurrent response's whole persistence.
The compact pointer survives response-cache eviction and transcript retirement. A process
loss between the authority commit and pointer commit is repaired from that exact request id.
"""
- from gateway.session_authorities import require_active_authority
from gateway.session_contract import SessionRef
from hermes_state_runtime import get_session_admission, _row
- authority = require_active_authority(adapter.gateway_runner, lambda: adapter._ensure_session_db())
- store = adapter._current_response_store()
admission_id = store.request_admission(durable_key[0])
if admission_id:
row = get_session_admission(authority.db, admission_id=admission_id)
@@ -55,13 +55,32 @@ def find_admission(adapter, durable_key, request_id):
# because its caller reused the same key. Existing completed cache records still replay.
raise RuntimeStoreError('admission_conflict')
store.bind_request_admission(durable_key[0], row['admission_id'])
- ref = SessionRef(authority.profile_id, row['target_session_id'])
- if row['status'] in {'queued', 'started'}:
+ return authority, SessionRef(authority.profile_id, row['target_session_id']), row
+
+
+def _restore_admitted(found):
+ """The owner-loop half of a found admission: a live one's session is restored and checked."""
+ if found is not None and found[2]['status'] in {'queued', 'started'}:
from gateway.session_api import restore_api_session
from gateway.session_api_turn import check_api_turn
+ authority, ref, row = found
restore_api_session(authority, ref.session_id)
check_api_turn(authority, ref, row['payload'])
- return authority, ref, row
+ return found
+
+
+def _lookup_call(adapter, durable_key, request_id):
+ """-> ``(store, call)``: ``store.atomically(*call)`` is one ``_lookup_admission`` unit."""
+ from gateway.session_authorities import require_active_authority
+ authority = require_active_authority(adapter.gateway_runner, lambda: adapter._ensure_session_db())
+ store = adapter._current_response_store()
+ return store, (_lookup_admission, authority, store, durable_key, request_id)
+
+
+async def find_admission(adapter, durable_key, request_id):
+ """``_lookup_admission`` off the loop, then the session restore on it."""
+ store, call = _lookup_call(adapter, durable_key, request_id)
+ return _restore_admitted(await store.off_loop(*call))
def admitted_context(admitted, *, replay=False):
@@ -79,12 +98,13 @@ def admitted_context(admitted, *, replay=False):
def _admit_steps(adapter, durable_key, request_id, binding, run_kwargs, model):
"""The final lookup and the admission are one API admission drive (``run_steps_off_loop``
- serializes them per authority): simultaneous observers converge on one admission."""
+ serializes them per authority): simultaneous observers converge on one admission. Every
+ store call is a yielded step, so a worker runs it: only the session restore stays on the loop."""
from gateway.session_api_turn import admit_api_steps
- store = adapter._current_response_store()
- if not store.bind_request_key(durable_key[0], binding, model=model):
+ store, lookup = _lookup_call(adapter, durable_key, request_id)
+ if not (yield partial(store.bind_request_key, durable_key[0], binding, model=model)):
raise RuntimeStoreError('admission_conflict')
- admitted = find_admission(adapter, durable_key, request_id)
+ admitted = _restore_admitted((yield partial(store.atomically, *lookup)))
if admitted is not None:
return (*admitted, True)
try:
@@ -94,10 +114,10 @@ def _admit_steps(adapter, durable_key, request_id, binding, run_kwargs, model):
except Exception:
# A definite pre-admission refusal does not reserve a poisoned identity. If storage
# cannot prove absence, leave the binding intact for later recovery.
- if find_admission(adapter, durable_key, request_id) is None:
- store.forget_unadmitted_request(durable_key[0])
+ if _restore_admitted((yield partial(store.atomically, *lookup))) is None:
+ yield partial(store.forget_unadmitted_request, durable_key[0])
raise
- store.bind_request_admission(durable_key[0], admitted[2]['admission_id'])
+ yield partial(store.bind_request_admission, durable_key[0], admitted[2]['admission_id'])
return (*admitted, False)
@@ -147,7 +167,7 @@ async def prepare_context(adapter, request, body, gateway_session_key, durable_k
from gateway.platforms.api_server_openai_routes import _parse_responses_input, _parse_conversation_history
request_id = f'responses:{scope}:{key}' if durable_key else None
if durable_key:
- admitted = find_admission(adapter, durable_key, request_id)
+ admitted = await find_admission(adapter, durable_key, request_id)
if admitted is not None:
return admitted_context(admitted, replay=True), None
previous = body.get('previous_response_id')
diff --git a/tests/gateway/test_api_responses_store_off_loop.py b/tests/gateway/test_api_responses_store_off_loop.py
index 6a0df376b1f..47427fa3047 100644
--- a/tests/gateway/test_api_responses_store_off_loop.py
+++ b/tests/gateway/test_api_responses_store_off_loop.py
@@ -88,3 +88,40 @@ async def test_store_false_keeps_no_snapshot_and_no_conversation_pointer(api, ow
assert fetched.status == 404, 'store:false response was retained'
assert api._response_store.get_conversation('ephemeral') is None
+
+async def test_keyed_admission_and_retry_never_take_the_store_lock_on_the_loop(api, owner):
+ """The store lock is held by workers for a whole persistence unit; a loop-side wait on it
+ freezes every other session for as long as that write takes."""
+ from gateway.session_results import execution_result
+
+ async def handle(event):
+ execution_result.get()['result'] = {'final_response': 'ok', 'messages': []}
+ return 'ok'
+
+ owner.runner._handle_message = handle
+ api._api_key = _API_KEY
+ store = api._current_response_store()
+ lock_threads = []
+
+ class _SpyLock:
+ def __init__(self, inner):
+ self._inner = inner
+
+ def __enter__(self):
+ lock_threads.append(threading.get_ident())
+ return self._inner.__enter__()
+
+ def __exit__(self, *exc):
+ return self._inner.__exit__(*exc)
+
+ store._lock = _SpyLock(store._lock)
+ app = web.Application()
+ app.router.add_post('/v1/responses', api._handle_responses)
+ async with TestClient(TestServer(app)) as client:
+ for _ in range(2): # a fresh keyed admission, then its exact retry
+ posted = await client.post('/v1/responses', json={'input': 'hi'},
+ headers={**_AUTH, 'Idempotency-Key': 'lock-probe'})
+ assert posted.status == 200, await posted.text()
+ assert lock_threads, 'the store was never used'
+ assert threading.get_ident() not in lock_threads, 'the event loop waited on the ResponseStore lock'