The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): open every profile's state.db off the event loop at boot
Tekniumcedcf7a0b649
diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.pyindex dcfc1c81205..b0b779ba927 100644--- a/gateway/run_bootstrap.py+++ b/gateway/run_bootstrap.py@@ -815,8 +815,12 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =from gateway.run_startup import recover_left_core_at_gateway_startawait asyncio.to_thread(recover_left_core_at_gateway_start) # before the runner loads platform config- runner = GatewayRunner(resolved_config)- from gateway.run_runtime import initialize_gateway_runtime+ from gateway.run_runtime import initialize_gateway_runtime, pin_launch_store, unpin_launch_store+ _launch_store_pin = await asyncio.to_thread(pin_launch_store)+ try:+ runner = GatewayRunner(resolved_config)+ finally:+ unpin_launch_store(_launch_store_pin)await initialize_gateway_runtime(runner)# Multiplex: swap the launch-home file handlers for per-profile routers so each profile's records# land in its own logs/. Must run after the runner resolved (possibly None) config and setup_logging.diff --git a/gateway/run_runtime.py b/gateway/run_runtime.pyindex c1a71b60d4d..329fd974efe 100644--- a/gateway/run_runtime.py+++ b/gateway/run_runtime.py@@ -41,8 +41,8 @@ async def _build_profile_authority(runner, name, home, *, register):instance_id = runner.session_runtime_descriptor['instance_id']# Each home's store resolves through the runner's scope-following handle cache, exactly# the handle every later scoped read of that profile uses (one writer per state.db).+ db = await asyncio.to_thread(open_profile_store, runner, home)with _profile_runtime_scope(home, hydrate_secrets=False):- db = getattr(runner._session_db, '_db', runner._session_db)if db is None:raise RuntimeError(_store_open_failure(runner, home))if (await asyncio.to_thread(Path(db.db_path).resolve)).parent != home:@@ -58,6 +58,46 @@ async def _build_profile_authority(runner, name, home, *, register):return authority+def open_profile_store(runner, home):+ """*home*'s SessionDB through the runner's scoped handle cache, opened in the CALLING thread+ under that profile's scope (bound here: a worker thread inherits no scope it was not given).++ Callers run it in a worker: a first open runs the schema / FTS setup, about a second per store+ on a loaded host, and run on the loop it froze every connected client for the whole+ multi-profile boot. Later opens of the same path are cache hits."""+ from gateway.run import _profile_runtime_scope+ with _profile_runtime_scope(home, hydrate_secrets=False):+ db = runner._session_db+ return getattr(db, '_db', db)+++async def _prewarm_profile_stores(runner, homes):+ """Open every secondary's store concurrently off the loop before the authorities are built in+ order. A failed open is not reported here: the cache records it and the authority build that+ follows parks that profile with the recorded reason."""+ await asyncio.gather(*(asyncio.to_thread(open_profile_store, runner, home) for _name, home in homes),+ return_exceptions=True)+++def pin_launch_store():+ """Open the launch profile's state.db in the calling (worker) thread and hold one registry+ reference, so the synchronous ``SessionStore`` open inside ``GatewayRunner()`` on the loop is a+ registry hit instead of the schema setup. None when the open fails: the store's own open then+ reports and backs off exactly as before."""+ from hermes_state_registry import acquire+ try:+ return acquire()+ except Exception:+ logger.debug('launch state.db prewarm failed; the store open reports it', exc_info=True)+ return None+++def unpin_launch_store(db):+ if db is not None:+ from hermes_state_registry import release+ release(db)++def _store_open_failure(runner, home):"""Why *home*'s ``state.db`` has no handle: the scoped handle caches answer None for a failed open(and while they back off), so the park reason named an ownership mismatch over the real cause, a@@ -150,6 +190,7 @@ async def initialize_gateway_runtime(runner):runner.session_runtime_descriptor = descriptorregistry = SessionAuthorities(homes[0][1], multiplexed=getattr(runner.config, 'multiplex_profiles', False))runner.session_authorities = registry+ await _prewarm_profile_stores(runner, homes[1:])for index, (name, home) in enumerate(homes):try:await _build_profile_authority(runner, name, home, register=index == 0)diff --git a/tests/gateway/test_profile_store_open_off_loop.py b/tests/gateway/test_profile_store_open_off_loop.pynew file mode 100644index 00000000000..672cd4f82d5--- /dev/null+++ b/tests/gateway/test_profile_store_open_off_loop.py@@ -0,0 +1,48 @@+"""A multiplexed boot opens each secondary profile's state.db off the event loop, under its own scope.++A first open runs the schema / FTS setup (about a second per store on a loaded host). Run on the+loop, an 11-profile boot froze every connected client for ~10 s.+"""+import threading++import pytest++from tests.gateway.test_session_authorities_multiplex import _reserve_homes, _runner+++async def test_secondary_store_opens_run_off_the_loop_in_their_own_scope(tmp_path, monkeypatch):+ import hermes_state_registry+ from gateway.run_runtime import initialize_gateway_runtime+ from gateway.runtime_ownership import process_ownership+ from hermes_constants import get_hermes_home++ root, homes = _reserve_homes(tmp_path, monkeypatch)+ process_ownership.reserve([home for _, home in homes])+ opens = []+ real_open = hermes_state_registry._open_session_db++ def spy(path):+ opens.append((path.parent.resolve(), threading.get_ident(), get_hermes_home().resolve()))+ return real_open(path)++ monkeypatch.setattr(hermes_state_registry, "_open_session_db", spy)+ loop_thread = threading.get_ident()+ runner = None+ try:+ runner = _runner(root, homes) # opens the launch store (start_gateway prewarms it off-loop)+ opens.clear()+ await initialize_gateway_runtime(runner)+ secondaries = {home.resolve() for _name, home in homes[1:]}+ opened = {parent for parent, _tid, _scope in opens}+ assert secondaries <= opened+ for parent, tid, scope in opens:+ assert tid != loop_thread, f"{parent}/state.db was opened on the event loop"+ assert scope == parent, "the worker opened a store outside that profile's scope"+ for _name, home in homes:+ assert runner.session_authorities.for_home(home).db.db_path.parent.resolve() == home.resolve()+ finally:+ for authority in list(getattr(runner, 'session_authorities', []) or []):+ authority.db.close()+ for _, home in homes:+ process_ownership.release(home)