Skip to content

The agent that grows with you

fix(gateway): answer a storage error in a moved chain's pre-bind with a 503

unsupportedpastels6f46e2b98cd9
diff --git a/gateway/platforms/api_server_response_admissions.py b/gateway/platforms/api_server_response_admissions.py
index 47a9bd8c7ad..0c7a539307d 100644
--- a/gateway/platforms/api_server_response_admissions.py
+++ b/gateway/platforms/api_server_response_admissions.py
@@ -1,6 +1,7 @@
"""Responses retries observe their original admission before resolving mutable conversation names."""
from functools import partial
import hashlib
+import logging
import uuid
from hermes_state_runtime import RuntimeStoreError
@@ -127,8 +128,17 @@ async def _bind_successor(adapter, session_id, run_kwargs):
from gateway.session_authorities import active_authority
authority = active_authority(adapter.gateway_runner)
if authority is not None:
- await run_steps_off_loop(authority, bind_api_steps(authority, session_id, walked_from=run_kwargs['walked_from'],
- caller_key=run_kwargs['gateway_session_key']))
+ try:
+ await run_steps_off_loop(authority, bind_api_steps(
+ authority, session_id, walked_from=run_kwargs['walked_from'],
+ caller_key=run_kwargs['gateway_session_key']))
+ except RuntimeStoreError:
+ raise
+ except Exception as exc:
+ # A busy writer or I/O error: the caller maps store refusals only, and anything else
+ # would reach the client as aiohttp's bare 500 instead of the retryable 503.
+ logging.getLogger(__name__).warning('pre-stream bind of %s failed', session_id, exc_info=True)
+ raise RuntimeStoreError('storage_unavailable') from exc
async def prepare_context(adapter, request, body, gateway_session_key, durable_key, scope, key):
diff --git a/tests/gateway/test_derived_generation_after_sweep.py b/tests/gateway/test_derived_generation_after_sweep.py
index 205dc63de31..2338cefad15 100644
--- a/tests/gateway/test_derived_generation_after_sweep.py
+++ b/tests/gateway/test_derived_generation_after_sweep.py
@@ -228,6 +228,38 @@ async def test_a_streamed_continuation_that_races_a_reclaim_gets_a_retryable_503
assert owner.db.get_session(successor) is None, 'the chain split across the root and its successor'
[email protected]('stream', [False, True], ids=['plain', 'streamed'])
+async def test_a_storage_error_in_a_moved_chains_pre_bind_is_a_retryable_503(api, owner, served, stream, monkeypatch):
+ """A chained request without an Idempotency-Key binds the successor its chain moved to before
+ any response starts. A failure there that is not a store refusal (a busy SQLite writer) is
+ ``storage_unavailable``, the JSON 503 with ``Retry-After`` every other store error before the
+ response gets, not aiohttp's bare 500."""
+ import sqlite3
+ from gateway import session_api
+ app = web.Application()
+ app.router.add_post('/v1/responses', api._handle_responses)
+ auth = {'Authorization': 'Bearer ' + chat._API_KEY}
+ store = api._current_response_store()
+ async with TestClient(TestServer(app)) as client:
+ opened = await client.post('/v1/responses', json={'input': 'remember 42'}, headers=auth)
+ assert opened.status == 200, await opened.text()
+ first = (await opened.json())['id']
+ _sweep(owner, store.get(first)['session_id'])
+ bind = session_api.bind_api_steps
+
+ def locked(authority, session_id, **kwargs):
+ if kwargs.get('walked_from') is None:
+ return (yield from bind(authority, session_id, **kwargs))
+ raise sqlite3.OperationalError('database is locked')
+ monkeypatch.setattr(session_api, 'bind_api_steps', locked)
+ body = {'input': 'and then?', 'previous_response_id': first, 'stream': stream}
+ raced = await client.post('/v1/responses', json=body, headers=auth)
+ assert raced.status == 503, await raced.text()
+ assert (await raced.json())['error']['code'] == 'storage_unavailable'
+ assert raced.headers.get('Retry-After') == '1'
+
+
def test_keep_passed_fences_reports_a_walk_it_could_not_pin(owner):
"""``keep_passed_fences`` pins every fence a walk passed and says whether the walk still holds:
a passed generation without a marker means the walk would now stop there."""