Skip to content

The agent that grows with you

fix(gateway): open every profile's state.db off the event loop at boot

Tekniumcedcf7a0b649
diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.py
index dcfc1c81205..b0b779ba927 100644
--- a/gateway/run_bootstrap.py
+++ b/gateway/run_bootstrap.py
@@ -815,8 +815,12 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =
from gateway.run_startup import recover_left_core_at_gateway_start
await asyncio.to_thread(recover_left_core_at_gateway_start) # before the runner loads platform config
- runner = GatewayRunner(resolved_config)
- from gateway.run_runtime import initialize_gateway_runtime
+ from gateway.run_runtime import initialize_gateway_runtime, pin_launch_store, unpin_launch_store
+ _launch_store_pin = await asyncio.to_thread(pin_launch_store)
+ try:
+ runner = GatewayRunner(resolved_config)
+ finally:
+ unpin_launch_store(_launch_store_pin)
await initialize_gateway_runtime(runner)
# Multiplex: swap the launch-home file handlers for per-profile routers so each profile's records
# land in its own logs/. Must run after the runner resolved (possibly None) config and setup_logging.
diff --git a/gateway/run_runtime.py b/gateway/run_runtime.py
index c1a71b60d4d..329fd974efe 100644
--- a/gateway/run_runtime.py
+++ b/gateway/run_runtime.py
@@ -41,8 +41,8 @@ async def _build_profile_authority(runner, name, home, *, register):
instance_id = runner.session_runtime_descriptor['instance_id']
# Each home's store resolves through the runner's scope-following handle cache, exactly
# the handle every later scoped read of that profile uses (one writer per state.db).
+ db = await asyncio.to_thread(open_profile_store, runner, home)
with _profile_runtime_scope(home, hydrate_secrets=False):
- db = getattr(runner._session_db, '_db', runner._session_db)
if db is None:
raise RuntimeError(_store_open_failure(runner, home))
if (await asyncio.to_thread(Path(db.db_path).resolve)).parent != home:
@@ -58,6 +58,46 @@ async def _build_profile_authority(runner, name, home, *, register):
return authority
+def open_profile_store(runner, home):
+ """*home*'s SessionDB through the runner's scoped handle cache, opened in the CALLING thread
+ under that profile's scope (bound here: a worker thread inherits no scope it was not given).
+
+ Callers run it in a worker: a first open runs the schema / FTS setup, about a second per store
+ on a loaded host, and run on the loop it froze every connected client for the whole
+ multi-profile boot. Later opens of the same path are cache hits."""
+ from gateway.run import _profile_runtime_scope
+ with _profile_runtime_scope(home, hydrate_secrets=False):
+ db = runner._session_db
+ return getattr(db, '_db', db)
+
+
+async def _prewarm_profile_stores(runner, homes):
+ """Open every secondary's store concurrently off the loop before the authorities are built in
+ order. A failed open is not reported here: the cache records it and the authority build that
+ follows parks that profile with the recorded reason."""
+ await asyncio.gather(*(asyncio.to_thread(open_profile_store, runner, home) for _name, home in homes),
+ return_exceptions=True)
+
+
+def pin_launch_store():
+ """Open the launch profile's state.db in the calling (worker) thread and hold one registry
+ reference, so the synchronous ``SessionStore`` open inside ``GatewayRunner()`` on the loop is a
+ registry hit instead of the schema setup. None when the open fails: the store's own open then
+ reports and backs off exactly as before."""
+ from hermes_state_registry import acquire
+ try:
+ return acquire()
+ except Exception:
+ logger.debug('launch state.db prewarm failed; the store open reports it', exc_info=True)
+ return None
+
+
+def unpin_launch_store(db):
+ if db is not None:
+ from hermes_state_registry import release
+ release(db)
+
+
def _store_open_failure(runner, home):
"""Why *home*'s ``state.db`` has no handle: the scoped handle caches answer None for a failed open
(and while they back off), so the park reason named an ownership mismatch over the real cause, a
@@ -150,6 +190,7 @@ async def initialize_gateway_runtime(runner):
runner.session_runtime_descriptor = descriptor
registry = SessionAuthorities(homes[0][1], multiplexed=getattr(runner.config, 'multiplex_profiles', False))
runner.session_authorities = registry
+ await _prewarm_profile_stores(runner, homes[1:])
for index, (name, home) in enumerate(homes):
try:
await _build_profile_authority(runner, name, home, register=index == 0)
diff --git a/tests/gateway/test_profile_store_open_off_loop.py b/tests/gateway/test_profile_store_open_off_loop.py
new file mode 100644
index 00000000000..672cd4f82d5
--- /dev/null
+++ b/tests/gateway/test_profile_store_open_off_loop.py
@@ -0,0 +1,48 @@
+"""A multiplexed boot opens each secondary profile's state.db off the event loop, under its own scope.
+
+A first open runs the schema / FTS setup (about a second per store on a loaded host). Run on the
+loop, an 11-profile boot froze every connected client for ~10 s.
+"""
+import threading
+
+import pytest
+
+from tests.gateway.test_session_authorities_multiplex import _reserve_homes, _runner
+
+
+async def test_secondary_store_opens_run_off_the_loop_in_their_own_scope(tmp_path, monkeypatch):
+ import hermes_state_registry
+ from gateway.run_runtime import initialize_gateway_runtime
+ from gateway.runtime_ownership import process_ownership
+ from hermes_constants import get_hermes_home
+
+ root, homes = _reserve_homes(tmp_path, monkeypatch)
+ process_ownership.reserve([home for _, home in homes])
+ opens = []
+ real_open = hermes_state_registry._open_session_db
+
+ def spy(path):
+ opens.append((path.parent.resolve(), threading.get_ident(), get_hermes_home().resolve()))
+ return real_open(path)
+
+ monkeypatch.setattr(hermes_state_registry, "_open_session_db", spy)
+ loop_thread = threading.get_ident()
+ runner = None
+ try:
+ runner = _runner(root, homes) # opens the launch store (start_gateway prewarms it off-loop)
+ opens.clear()
+ await initialize_gateway_runtime(runner)
+ secondaries = {home.resolve() for _name, home in homes[1:]}
+ opened = {parent for parent, _tid, _scope in opens}
+ assert secondaries <= opened
+ for parent, tid, scope in opens:
+ assert tid != loop_thread, f"{parent}/state.db was opened on the event loop"
+ assert scope == parent, "the worker opened a store outside that profile's scope"
+ for _name, home in homes:
+ assert runner.session_authorities.for_home(home).db.db_path.parent.resolve() == home.resolve()
+ finally:
+ for authority in list(getattr(runner, 'session_authorities', []) or []):
+ authority.db.close()
+ for _, home in homes:
+ process_ownership.release(home)