Skip to content

The agent that grows with you

fix(api_server): Responses store reads and writes run off the event loop

Tekniumf58662741cee
diff --git a/gateway/platforms/api_server_openai_routes.py b/gateway/platforms/api_server_openai_routes.py
index 05e2d7b6482..81665ffaa05 100644
--- a/gateway/platforms/api_server_openai_routes.py
+++ b/gateway/platforms/api_server_openai_routes.py
@@ -107,13 +107,32 @@ def _responses_fingerprint_keys(body):
return _RESPONSES_FINGERPRINT_KEYS + (("truncation",) if body.get("truncation") is not None else ())
-def _response_identity(store, durable_key):
+async def _response_identity(store, durable_key):
if durable_key is not None:
from gateway.platforms.api_server_response_identity import durable_response_id
- return durable_response_id(durable_key[0]), store.request_created_at(durable_key[0])
+ return durable_response_id(durable_key[0]), await store.off_loop(store.request_created_at, durable_key[0])
return f'resp_{uuid.uuid4().hex[:28]}', int(time.time())
+def _matching_replay(store, key, binding):
+ """-> ``(binding matches, stored replay record or None)``, read as one store unit."""
+ if not store.request_key_matches(key, binding):
+ return False, None
+ return True, store.get(key)
+
+
+def _store_nonstream_response(store, snapshot, replay_record, conversation, terminal_replay):
+ """Non-streaming /v1/responses persistence as one store unit: body, conversation pointer, replay record."""
+ if snapshot is not None:
+ response_id, data = snapshot
+ advance = conversation and (not terminal_replay or store.get_conversation(conversation) in (None, response_id))
+ store.put(response_id, data)
+ if advance:
+ store.set_conversation(conversation, response_id)
+ if replay_record is not None:
+ store.put(*replay_record)
+
+
def _response_status(result):
if result.get('interrupted'):
return 'cancelled'
@@ -333,8 +352,8 @@ class _ResponsesStream:
return next_item_id(self.response_id, self._identity_counts, kind,
call_id=call_id, commentary=commentary)
- def _output_index(self, item_id):
- index = (self.response_store.output_index(self.durable_key[0], item_id)
+ async def _output_index(self, item_id):
+ index = (await self.response_store.off_loop(self.response_store.output_index, self.durable_key[0], item_id)
if self.durable_key else self.output_index)
self.output_index = max(self.output_index, index + 1)
return index
@@ -356,9 +375,13 @@ class _ResponsesStream:
def _history_with_user(self) -> list[dict[str, Any]]:
return list(self.conversation_history) + [{"role": "user", "content": self.user_message}]
- def persist_snapshot(self, response_env: dict[str, Any], *, history=None, session_id=None):
- if not self.store:
- return
+ async def persist_snapshot(self, response_env: dict[str, Any], *, history=None, session_id=None):
+ if self.store:
+ await self.response_store.off_loop(self._persist_snapshot, response_env, history, session_id)
+
+ def _persist_snapshot(self, response_env, history, session_id):
+ """One store unit (worker thread, store lock held): the settled check, the body and the
+ conversation pointer can not interleave with another observer's."""
if self.durable_key and _settled(self.response_store.get(self.durable_key[0])):
return # A concurrent observer already committed the exact terminal envelope.
advance_conversation = self.conversation and (not self.terminal_replay
@@ -371,7 +394,7 @@ class _ResponsesStream:
if advance_conversation:
self.response_store.set_conversation(self.conversation, self.response_id)
- def persist_incomplete_if_needed(self) -> None:
+ async def persist_incomplete_if_needed(self) -> None:
"""Persist an ``incomplete`` snapshot when no terminal one was written (disconnect /
cancel paths), so GET /v1/responses/{id} and ``previous_response_id`` chaining survive."""
if not self.store or self.terminal_snapshot_persisted:
@@ -382,21 +405,22 @@ class _ResponsesStream:
if text:
items.append(_message_item(text))
history.append({"role": "assistant", "content": text})
- self.persist_snapshot(self.terminal_envelope("incomplete", items), history=history)
+ await self.persist_snapshot(self.terminal_envelope("incomplete", items), history=history)
async def emit_created(self) -> None:
env = self.envelope("in_progress")
env["output"] = []
- await self.write_event("response.created", {"type": "response.created", "response": env})
if not self.terminal_replay:
- self.persist_snapshot(env)
+ # Stored before the id goes out: a client that reads it back at once must find it.
+ await self.persist_snapshot(env)
+ await self.write_event("response.created", {"type": "response.created", "response": env})
async def _open_message_item(self) -> None:
"""Emit output_item.added for the assistant message on the first text delta."""
if self.message_opened:
return
self.message_opened = True
- self.message_output_index = self._output_index(self.message_item_id)
+ self.message_output_index = await self._output_index(self.message_item_id)
await self.write_event("response.output_item.added", {
"type": "response.output_item.added", "output_index": self.message_output_index,
"item": {"id": self.message_item_id, "type": "message", "status": "in_progress",
@@ -420,7 +444,7 @@ class _ResponsesStream:
if self.reasoning_item is None:
item = {"id": self._item_id('reasoning'), "type": "reasoning", "status": "in_progress",
"summary": []}
- self.reasoning_item = {"item": item, "output_index": self._output_index(item['id']), "parts": []}
+ self.reasoning_item = {"item": item, "output_index": await self._output_index(item['id']), "parts": []}
await self.write_event("response.output_item.added", {
"type": "response.output_item.added",
"output_index": self.reasoning_item["output_index"], "item": item})
@@ -458,7 +482,7 @@ class _ResponsesStream:
await self.close_reasoning_item()
item = {"id": self._item_id('message', commentary=True), "status": "completed", "phase": "commentary",
**_message_item(text)}
- idx = self._output_index(item['id'])
+ idx = await self._output_index(item['id'])
self.emitted_items.append({"phase": "commentary", **_message_item(text)})
for event in ("response.output_item.added", "response.output_item.done"):
await self.write_event(event, {"type": event, "output_index": idx, "item": item})
@@ -473,7 +497,7 @@ class _ResponsesStream:
name = payload.get("name", "")
item = {"id": self._item_id('function_call', call_id=call_id), "type": "function_call",
"status": "in_progress", "name": name, "call_id": call_id, "arguments": arguments_str}
- idx = self._output_index(item['id'])
+ idx = await self._output_index(item['id'])
self.pending_tool_calls.append({
"call_id": call_id, "name": name, "arguments": arguments_str, "item_id": item["id"],
"output_index": idx})
@@ -500,7 +524,7 @@ class _ResponsesStream:
output_parts = [{"type": "input_text", "text": result_str}]
output_item = {"id": self._item_id('function_call_output', call_id=call_id), "type": "function_call_output",
"call_id": pending["call_id"], "output": output_parts, "status": "completed"}
- idx = self._output_index(output_item['id'])
+ idx = await self._output_index(output_item['id'])
self.emitted_items.append(
{"type": "function_call_output", "call_id": pending["call_id"], "output": output_parts})
for event in ("response.output_item.added", "response.output_item.done"):
@@ -622,7 +646,7 @@ class _ResponsesStream:
history = self._history_with_user()
history.append(
{"role": "assistant", "content": self.final_response_text or redact(self.agent_error)})
- self.persist_snapshot(env, history=history)
+ await self.persist_snapshot(env, history=history)
self.terminal_snapshot_persisted = True
await self.write_event("response.failed", {"type": "response.failed", "response": env})
@@ -633,7 +657,7 @@ class _ResponsesStream:
if self.durable_key:
start = self.adapter._response_messages_turn_start_index(self.conversation_history, self.user_message, result)
output = self.adapter._extract_output_items(result, start_index=start, response_id=self.response_id)
- output = self.response_store.order_output(self.durable_key[0], output)
+ output = await self.response_store.off_loop(self.response_store.order_output, self.durable_key[0], output)
env = self.terminal_envelope(status, output,
error=self._api._redact_api_error_text(result.get('error') or 'The admitted turn failed.')
if status == 'failed' else None)
@@ -643,19 +667,19 @@ class _ResponsesStream:
# Transcript substitution for result["_compressed"] happens in the history builder; only
# a compression-rotated session_id is propagated so chaining resumes the child session.
sid = result.get("session_id") if isinstance(result, dict) else None
- self.persist_snapshot(
- env, history=full_history, session_id=sid if isinstance(sid, str) and sid else None)
- self.terminal_snapshot_persisted = True
- if self.durable_key:
+ record = None
+ if self.durable_key and not self.terminal_replay:
+ # Only the publisher recorded the deltas; a replay observer winning the
+ # INSERT OR IGNORE would leave every later replay without them. Built here, on the
+ # loop, so the worker never reads headers or events the loop may still append to.
headers = {k: v for k, v in self.response.headers.items() if k.startswith('X-Hermes-')}
terminal_event = {'type': 'response.' + status, 'response': env, 'sequence_number': self.sequence_number}
- if not self.terminal_replay:
- # Only the publisher recorded the deltas; a replay observer winning the
- # INSERT OR IGNORE would leave every later replay without them.
- self.response_store.put(self.durable_key[0], {
- 'fingerprint': self.durable_key[1], 'response': env, 'headers': headers,
- 'events': [*self.recorded_events, ('response.' + status, terminal_event)]})
- stored = self.response_store.get(self.durable_key[0])
+ record = {'fingerprint': self.durable_key[1], 'response': env, 'headers': headers,
+ 'events': [*self.recorded_events, ('response.' + status, terminal_event)]}
+ stored = await self.response_store.off_loop(
+ self._commit_terminal, env, full_history, sid if isinstance(sid, str) and sid else None, record)
+ self.terminal_snapshot_persisted = True
+ if self.durable_key:
if _settled(stored):
env = stored['response']
status = env['status']
@@ -663,6 +687,16 @@ class _ResponsesStream:
'response.' + status, {'type': 'response.' + status, 'response': env})
return env
+ def _commit_terminal(self, env, full_history, session_id, record):
+ """The terminal snapshot plus, when keyed, the replay record and its read-back: one store
+ unit, so no other observer of the same key interleaves between them. -> stored record."""
+ self._persist_snapshot(env, full_history, session_id)
+ if not self.durable_key:
+ return None
+ if record is not None:
+ self.response_store.put(self.durable_key[0], record)
+ return self.response_store.get(self.durable_key[0])
+
async def emit_crash(self, exc: BaseException) -> None:
error = self._api._redact_api_error_text(exc, limit=500)
env = self.terminal_envelope("failed", list(self.emitted_items), error=error)
@@ -1244,12 +1278,12 @@ class OpenAICompatRoutesMixin:
else:
await st.emit_completed()
except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError, OSError):
- st.persist_incomplete_if_needed()
+ await st.persist_incomplete_if_needed()
await _abandon_agent_task(agent_ref, agent_task, "SSE client disconnected")
logger.info("SSE client disconnected; interrupted agent task %s", response_id)
except asyncio.CancelledError:
# Server-side cancellation (shutdown, timeout): persist incomplete, then re-raise.
- st.persist_incomplete_if_needed()
+ await st.persist_incomplete_if_needed()
await _abandon_agent_task(
agent_ref, agent_task, "SSE task cancelled",
reap_source="api_server_sse_cancelled", await_cancel=False)
@@ -1259,7 +1293,7 @@ class OpenAICompatRoutesMixin:
# Unhandled agent error (BadRequestError, AuthenticationError, ...): emit
# response.failed and end the stream cleanly (no TransferEncodingError).
import traceback as _tb
- st.persist_incomplete_if_needed()
+ await st.persist_incomplete_if_needed()
st.agent_error = _redact_api_error_text(_tb.format_exc())
with suppress(Exception):
await st.emit_crash(exc)
@@ -1304,10 +1338,10 @@ class OpenAICompatRoutesMixin:
from gateway.platforms.api_server_response_admissions import request_binding
response_store = self._current_response_store()
binding = request_binding(self, request, durable_key)
- if not response_store.request_key_matches(durable_key[0], binding):
+ matches, replay = await response_store.off_loop(_matching_replay, response_store, durable_key[0], binding)
+ if not matches:
return durable_key, idempotency_scope, idempotency_key, web.json_response(
_openai_error('admission_conflict', code='admission_conflict'), status=409)
- replay = response_store.get(durable_key[0])
if replay is not None:
if replay.get('fingerprint') != durable_key[1]:
return durable_key, idempotency_scope, idempotency_key, web.json_response(
@@ -1368,10 +1402,11 @@ class OpenAICompatRoutesMixin:
session_id, user_message = context['session_id'], context['user_message']
conversation_history, instructions = context['conversation_history'], context['instructions']
run_kwargs, run_agent = context['run_kwargs'], context['run_agent']
- response_id, created_at = _response_identity(self._current_response_store(), durable_key)
+ response_store = self._current_response_store()
+ response_id, created_at = await _response_identity(response_store, durable_key)
response_model = body.get('model', self._model_name)
if durable_key:
- response_model = self._current_response_store().request_model(durable_key[0], response_model)
+ response_model = await response_store.off_loop(response_store.request_model, durable_key[0], response_model)
if stream:
_stream_q = ThreadSafeAsyncQueue()
@@ -1415,7 +1450,8 @@ class OpenAICompatRoutesMixin:
if err is not None:
return err
if durable_key is not None:
- replay = self._current_response_store().get(durable_key[0])
+ response_store = self._current_response_store()
+ replay = await response_store.off_loop(response_store.get, durable_key[0])
if _settled(replay):
return web.json_response(replay['response'], headers=replay.get('headers') or {})
result, usage = outcome
@@ -1440,26 +1476,24 @@ class OpenAICompatRoutesMixin:
"output": self._extract_output_items(result, start_index=output_start_index,
**({'response_id': response_id} if durable_key else {})),
"usage": _responses_usage_payload(usage)}
+ response_store = self._current_response_store() # at use time: the store can be swapped mid-turn
if durable_key:
- response_data['output'] = self._current_response_store().order_output(durable_key[0], response_data['output'])
+ response_data['output'] = await response_store.off_loop(
+ response_store.order_output, durable_key[0], response_data['output'])
if response_data['status'] == 'failed':
response_data['error'] = {'code': 'agent_error', 'message': _redact_api_error_text(
result.get('error') or 'The admitted turn failed.')}
- if store:
- response_store = self._current_response_store()
- advance_conversation = conversation and (not context.get('terminal_replay')
- or response_store.get_conversation(conversation) in (None, response_id))
- response_store.put(response_id, {
- "response": response_data, "conversation_history": full_history,
- "instructions": instructions, "session_id": _effective_session_id})
- if advance_conversation:
- response_store.set_conversation(conversation, response_id)
+ snapshot = (response_id, {"response": response_data, "conversation_history": full_history,
+ "instructions": instructions, "session_id": _effective_session_id}) if store else None
response_headers = {"X-Hermes-Session-Id": _effective_session_id}
if gateway_session_key:
response_headers["X-Hermes-Session-Key"] = gateway_session_key
- if durable_key is not None:
- self._current_response_store().put(durable_key[0], {
- 'fingerprint': durable_key[1], 'response': response_data, 'headers': response_headers})
+ replay_record = (durable_key[0], {'fingerprint': durable_key[1], 'response': response_data,
+ 'headers': response_headers}) if durable_key is not None else None
+ if snapshot is not None or replay_record is not None:
+ # Awaited before answering, so the returned id (and its replay record) read back at once.
+ await response_store.off_loop(_store_nonstream_response, response_store, snapshot, replay_record,
+ conversation, bool(context.get('terminal_replay')))
return web.json_response(response_data, headers=response_headers)
async def _handle_get_response(self, request: web.Request) -> web.Response:
@@ -1469,7 +1503,8 @@ class OpenAICompatRoutesMixin:
if auth_err:
return auth_err
response_id = request.match_info["response_id"]
- stored = self._current_response_store().get(response_id)
+ store = self._current_response_store()
+ stored = await store.off_loop(store.get, response_id)
if stored is None:
return _error_response(f"Response not found: {response_id}", 404)
return web.json_response(stored["response"])
@@ -1481,7 +1516,8 @@ class OpenAICompatRoutesMixin:
if auth_err:
return auth_err
response_id = request.match_info["response_id"]
- if not self._current_response_store().delete(response_id):
+ store = self._current_response_store()
+ if not await store.off_loop(store.delete, response_id):
return _error_response(f"Response not found: {response_id}", 404)
return web.json_response({"id": response_id, "object": "response", "deleted": True})
diff --git a/gateway/platforms/api_server_response_admissions.py b/gateway/platforms/api_server_response_admissions.py
index 6ed96e611bd..b26587e705a 100644
--- a/gateway/platforms/api_server_response_admissions.py
+++ b/gateway/platforms/api_server_response_admissions.py
@@ -151,8 +151,9 @@ async def prepare_context(adapter, request, body, gateway_session_key, durable_k
if admitted is not None:
return admitted_context(admitted, replay=True), None
previous = body.get('previous_response_id')
+ response_store = adapter._current_response_store()
if body.get('conversation'):
- previous = adapter._current_response_store().get_conversation(body['conversation'])
+ previous = await response_store.off_loop(response_store.get_conversation, body['conversation'])
messages, error = _parse_responses_input(body['input'])
if error is not None:
return None, error
@@ -164,7 +165,7 @@ async def prepare_context(adapter, request, body, gateway_session_key, durable_k
stored_session_id = chain_root = None
instructions = body.get('instructions')
if not history and previous:
- stored = adapter._current_response_store().get(previous)
+ stored = await response_store.off_loop(response_store.get, previous)
if stored is None:
return None, _error_response(f'Previous response not found: {previous}', 404)
history = list(stored.get('conversation_history', []))
diff --git a/gateway/platforms/api_server_response_store.py b/gateway/platforms/api_server_response_store.py
index 1408b5cfbc2..a0c83b080eb 100644
--- a/gateway/platforms/api_server_response_store.py
+++ b/gateway/platforms/api_server_response_store.py
@@ -1,8 +1,11 @@
"""SQLite-backed store for Responses API state (``previous_response_id`` chaining, named conversations)."""
+import asyncio
+import functools
import json
import logging
import sqlite3
+import threading
import time
from contextlib import closing, suppress
from pathlib import Path
@@ -18,6 +21,16 @@ IDENTITY_RETENTION_FACTOR = 10
_RECORD = "json_extract(CASE WHEN json_valid(data) THEN data END, '$.response"
+def _locked(method):
+ """Hold the store's lock for the call: one sqlite3 connection, used from the loop's worker threads
+ and the admission steps, must never interleave two callers' statements or commits."""
+ @functools.wraps(method)
+ def _call(self, *args, **kwargs):
+ with self._lock:
+ return method(self, *args, **kwargs)
+ return _call
+
+
class ResponseStore:
"""SQLite-backed LRU store for Responses API state (full conversation history per response
for ``previous_response_id`` chaining). Persists across restarts; in-memory fallback.
@@ -38,6 +51,7 @@ class ResponseStore:
def __init__(self, max_size: int = MAX_STORED_RESPONSES, db_path: Optional[str] = None,
max_identities: Optional[int] = None):
+ self._lock = threading.RLock()
self._max_size = max_size
self._max_identities = max_identities
if db_path is None:
@@ -84,6 +98,18 @@ class ResponseStore:
# Conversation history lives here: owner-only perms, once at init (not per commit).
self._tighten_file_permissions()
+ def atomically(self, fn, /, *args, **kwargs):
+ """Run ``fn`` (a sequence of this store's calls) as one unit no other caller interleaves."""
+ with self._lock:
+ return fn(*args, **kwargs)
+
+ async def off_loop(self, fn, /, *args, **kwargs):
+ """``atomically`` on a worker thread. Every read or write an event-loop handler makes goes
+ through here: ``put`` serialises the whole chained conversation_history and commits, which on
+ the loop stalled every other request for as long as it took (#142641). Awaited, so a handler
+ still sees its own write before it answers: a response id it returns is readable at once."""
+ return await asyncio.to_thread(self.atomically, fn, *args, **kwargs)
+
def _tighten_file_permissions(self) -> None:
"""Force owner-only permissions on the DB and SQLite sidecars."""
if not self._db_path:
@@ -95,6 +121,7 @@ class ResponseStore:
except OSError:
logger.debug("Failed to restrict response store permissions for %s", candidate, exc_info=True)
+ @_locked
def get(self, response_id: str) -> Optional[dict[str, Any]]:
"""Retrieve a stored response by ID (updates access time for LRU)."""
row = self._conn.execute(
@@ -113,6 +140,7 @@ class ResponseStore:
self._conn.commit()
return None
+ @_locked
def put(self, response_id: str, data: dict[str, Any]) -> None:
"""Store a response, evicting the oldest if at capacity."""
insert = 'INSERT OR IGNORE' if response_id.startswith('idem:') else 'INSERT OR REPLACE'
@@ -211,6 +239,7 @@ class ResponseStore:
self._conn.execute(f'DELETE FROM {table} WHERE request_key NOT IN (SELECT request_key FROM response_keys)')
self._conn.commit()
+ @_locked
def bind_request_key(self, key: str, fingerprint: str, *, model=None) -> bool:
"""Reserve one immutable digest before any canonical admission can run."""
with self._conn:
@@ -220,6 +249,7 @@ class ResponseStore:
row = self._conn.execute('SELECT fingerprint FROM response_keys WHERE request_key=?', (key,)).fetchone()
return row[0] == fingerprint
+ @_locked
def request_model(self, key: str, fallback):
"""The accepted wire label survives route changes and response-body LRU eviction."""
with self._conn:
@@ -230,6 +260,7 @@ class ResponseStore:
row = self._conn.execute('SELECT model_json FROM response_keys WHERE request_key=?', (key,)).fetchone()
return json.loads(row[0]) if row is not None else fallback
+ @_locked
def output_index(self, key: str, item_id: str) -> int:
"""Allocate before live publication; retries and parallel observers share this order.
@@ -244,21 +275,26 @@ class ResponseStore:
(key, item_id)).fetchone()
return row[0]
+ @_locked
def order_output(self, key: str, items):
return sorted(items, key=lambda item: self.output_index(key, item['id']))
+ @_locked
def request_created_at(self, key: str) -> Optional[int]:
row = self._conn.execute('SELECT created_at FROM response_keys WHERE request_key=?', (key,)).fetchone()
return row[0] if row else None
+ @_locked
def request_key_matches(self, key: str, fingerprint: str) -> bool:
row = self._conn.execute('SELECT fingerprint FROM response_keys WHERE request_key=?', (key,)).fetchone()
return row is None or row[0] == fingerprint
+ @_locked
def request_admission(self, key: str) -> Optional[str]:
row = self._conn.execute('SELECT admission_id FROM response_admissions WHERE request_key=?', (key,)).fetchone()
return row[0] if row else None
+ @_locked
def bind_request_admission(self, key: str, admission_id: str) -> None:
from hermes_state_runtime import RuntimeStoreError
with self._conn:
@@ -266,11 +302,13 @@ class ResponseStore:
if self.request_admission(key) != admission_id:
raise RuntimeStoreError('admission_conflict')
+ @_locked
def forget_unadmitted_request(self, key: str) -> None:
with self._conn:
self._conn.execute('DELETE FROM response_keys WHERE request_key=? AND NOT EXISTS '
'(SELECT 1 FROM response_admissions WHERE request_key=?)', (key, key))
+ @_locked
def delete(self, response_id: str) -> bool:
"""Remove a response, conversation mappings to it and its settled Idempotency-Key
identity (replay record included). True if anything was found and deleted."""
@@ -281,21 +319,25 @@ class ResponseStore:
self._conn.commit()
return cursor.rowcount > 0 or forgotten > 0
+ @_locked
def get_conversation(self, name: str) -> Optional[str]:
"""Get the latest response_id for a conversation name."""
row = self._conn.execute("SELECT response_id FROM conversations WHERE name = ?", (name,)).fetchone()
return row[0] if row else None
+ @_locked
def set_conversation(self, name: str, response_id: str) -> None:
"""Map a conversation name to its latest response_id."""
self._conn.execute("INSERT OR REPLACE INTO conversations (name, response_id) VALUES (?, ?)", (name, response_id))
self._conn.commit()
+ @_locked
def close(self) -> None:
"""Close the database connection."""
with suppress(Exception):
self._conn.close()
+ @_locked
def __len__(self) -> int:
row = self._conn.execute("SELECT COUNT(*) FROM responses").fetchone()
return row[0] if row else 0
diff --git a/gateway/platforms/api_server_runs.py b/gateway/platforms/api_server_runs.py
index 5196c45322e..a7e0f7a576e 100644
--- a/gateway/platforms/api_server_runs.py
+++ b/gateway/platforms/api_server_runs.py
@@ -507,8 +507,9 @@ async def _resolve_chained_history(self, body: dict, raw_input: Any, *, _openai_
that cannot answer is the retryable store verdict (``continued_session``). The last item is the
stored session the chain moved past (``walked_from``), else None."""
from gateway.platforms.api_server_authority_runs import continued_session
- history, instructions, chain_root, error = _resolve_conversation_history(
- self, body, raw_input, _openai_error=_openai_error)
+ # Off the loop: a previous_response_id chain reads (and json-decodes) the stored history.
+ history, instructions, chain_root, error = await asyncio.to_thread(
+ _resolve_conversation_history, self, body, raw_input, _openai_error=_openai_error)
stored_session_id = chain_root
if error is None:
stored_session_id, error = await continued_session(self, chain_root)
diff --git a/tests/gateway/test_api_responses_store_off_loop.py b/tests/gateway/test_api_responses_store_off_loop.py
new file mode 100644
index 00000000000..ab7a8db2e62
--- /dev/null
+++ b/tests/gateway/test_api_responses_store_off_loop.py
@@ -0,0 +1,59 @@
+"""Responses persistence runs off the event loop and stays readable at once (#142641).
+
+``ResponseStore.put`` serialises the whole chained conversation_history and commits it; on the
+gateway's loop that stalled every other request (the Desktop WebSocket included) for as long as
+the write took. Moving it to a worker must not loosen ordering: the id a request returns is
+readable by the very next request.
+"""
+import json
+import threading
+
+import pytest
+from aiohttp import web
+from aiohttp.test_utils import TestClient, TestServer
+
+_API_KEY = 'test-key'
+_AUTH = {'Authorization': 'Bearer ' + _API_KEY}
+
+
[email protected]('stream', [False, True])
+async def test_response_store_writes_leave_the_loop_and_read_back_at_once(api, owner, monkeypatch, stream):
+ from gateway.platforms.api_server_response_store import ResponseStore
+ from gateway.session_results import execution_result
+
+ async def handle(event):
+ execution_result.get()['result'] = {'final_response': 'ok', 'messages': []}
+ return 'ok'
+
+ owner.runner._handle_message = handle
+ api._api_key = _API_KEY
+ loop_thread = threading.get_ident()
+ put_threads = []
+ real_put = ResponseStore.put
+
+ def _put(self, response_id, data):
+ put_threads.append(threading.get_ident())
+ return real_put(self, response_id, data)
+
+ monkeypatch.setattr(ResponseStore, 'put', _put)
+ app = web.Application()
+ app.router.add_post('/v1/responses', api._handle_responses)
+ app.router.add_get('/v1/responses/{response_id}', api._handle_get_response)
+ async with TestClient(TestServer(app)) as client:
+ for n, key in enumerate(('first', 'second')):
+ body = {'input': f'turn {n}', 'conversation': 'project', 'stream': stream}
+ posted = await client.post('/v1/responses', json=body, headers={**_AUTH, 'Idempotency-Key': key})
+ text = await posted.text()
+ assert posted.status == 200, text
+ if stream:
+ done = [json.loads(line[6:]) for line in text.splitlines()
+ if line.startswith('data: ') and '"response.completed"' in line]
+ response_id = done[-1]['response']['id']
+ else:
+ response_id = json.loads(text)['id']
+ fetched = await client.get('/v1/responses/' + response_id, headers=_AUTH)
+ assert fetched.status == 200, f'response {response_id} not readable right after it was returned'
+ assert (await fetched.json())['status'] == 'completed'
+ assert put_threads, 'nothing was persisted'
+ assert loop_thread not in put_threads, 'ResponseStore.put ran on the event loop thread'