Skip to content

The agent that grows with you

fix(gateway): close Responses stores off the loop on adapter disconnect

Teknium681a692b3413
diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py
index 182e68c3572..88e4fd9e41d 100644
--- a/gateway/platforms/api_server.py
+++ b/gateway/platforms/api_server.py
@@ -4077,7 +4077,7 @@ class APIServerAdapter(OpenAICompatRoutesMixin, BasePlatformAdapter):
routed.clear()
for store in stores:
try:
- store.close()
+ await asyncio.to_thread(store.close) # a persistence worker may hold its lock
except Exception:
logger.debug("Failed to close response store for %s", self.name, exc_info=True)
with suppress(Exception):
diff --git a/tests/gateway/test_api_responses_store_off_loop.py b/tests/gateway/test_api_responses_store_off_loop.py
index 47427fa3047..18153274bce 100644
--- a/tests/gateway/test_api_responses_store_off_loop.py
+++ b/tests/gateway/test_api_responses_store_off_loop.py
@@ -5,6 +5,7 @@ gateway's loop that stalled every other request (the Desktop WebSocket included)
the write took. Moving it to a worker must not loosen ordering: the id a request returns is
readable by the very next request.
"""
+import asyncio
import json
import threading
@@ -125,3 +126,30 @@ async def test_keyed_admission_and_retry_never_take_the_store_lock_on_the_loop(a
assert posted.status == 200, await posted.text()
assert lock_threads, 'the store was never used'
assert threading.get_ident() not in lock_threads, 'the event loop waited on the ResponseStore lock'
+
+
+async def test_disconnect_never_waits_on_a_worker_held_store_lock_on_the_loop(api):
+ """A persistence worker keeps running after its handler is cancelled; adapter teardown that
+ took the store lock on the loop froze every profile's loop (and its own cleanup timeout)."""
+ store = api._current_response_store()
+ held, release, released = threading.Event(), threading.Event(), threading.Event()
+
+ def _hold_store():
+ held.set()
+ release.wait(3) # safety deadline: on the bug the loop is stuck until this expires
+ released.set()
+
+ worker = asyncio.create_task(store.off_loop(_hold_store))
+ assert await asyncio.to_thread(held.wait, 5)
+ teardown = asyncio.create_task(api.disconnect())
+ loop_ran_while_held = asyncio.get_running_loop().create_future()
+ asyncio.get_running_loop().call_soon(lambda: loop_ran_while_held.set_result(not released.is_set()))
+ try:
+ assert await loop_ran_while_held, 'disconnect waited on the worker-held ResponseStore lock on the event loop'
+ finally:
+ release.set()
+ await worker
+ await teardown
+ with pytest.raises(Exception):
+ store._conn.execute('SELECT 1') # the connection was still closed