The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(memory): key the shared prefetch registry per Hermes home and self-clear it
kshitijk4poor4b29e6102bbc
diff --git a/agent/memory_manager.py b/agent/memory_manager.pyindex a84dc9f05d6..784e358ff6d 100644--- a/agent/memory_manager.py+++ b/agent/memory_manager.py@@ -24,6 +24,7 @@ from agent.memory_caller_scope import bound_scope, caller_scope, claim_instance,from agent.memory_context import StreamingContextScrubber, build_memory_context_block, sanitize_contextfrom agent.memory_provider import MemoryProvider, PRE_COMPRESS_CHECKPOINT_API_VERSION, ctx_bound, spawn_context_threadfrom agent.skill_commands import extract_user_instruction_from_skill_message+from hermes_constants import hermes_home_keyfrom tools.hook_output_spill import get_spill_config, spill_if_oversizedfrom tools.registry import tool_error@@ -226,12 +227,15 @@ class MemoryManager:swallows per-provider exceptions."""- # Process-wide registry of in-flight external prefetch threads, keyed by provider- # name. A new MemoryManager is built per agent init, so a per-instance registry- # would let every fresh manager spawn one more stuck thread against the same- # provider (#134392); shared, a stuck call suppresses new spawns from any manager- # until it returns.- _EXTERNAL_PREFETCH_THREADS: dict[str, threading.Thread] = {}+ # Process-wide registry of in-flight external prefetch threads, keyed by+ # (Hermes home, provider name). A new MemoryManager is built per agent init, so a+ # per-instance registry would let every fresh manager spawn one more stuck thread+ # against the same provider (#134392); shared, a stuck call suppresses new spawns+ # from any manager until it returns. The home is part of the key because one+ # gateway process can serve several profiles, each with its own config and+ # credentials for a same-named provider: one profile's hung backend must not+ # silence another's recall.+ _EXTERNAL_PREFETCH_THREADS: dict[tuple[str, str], threading.Thread] = {}_EXTERNAL_PREFETCH_LOCK = threading.Lock()def __init__(self, *, external_prefetch_timeout: Optional[float] = None) -> None:@@ -244,6 +248,8 @@ class MemoryManager:if timeout <= 0:raise ValueError("external_prefetch_timeout must be positive")self._external_prefetch_timeout = timeout+ # Captured at build time, inside the profile scope the agent is built for.+ self._home_key = hermes_home_key()# Single-worker background executor for end-of-turn sync/prefetch, created lazily so# the builtin-only path spawns no threads; one worker serializes a provider's writes.self._sync_executor: Optional[ThreadPoolExecutor] = None@@ -387,19 +393,27 @@ class MemoryManager:result_box: dict[str, Any] = {}+ key = (self._home_key, provider.name)+def _run() -> None:try:result_box["value"] = provider.prefetch(query, session_id=session_id) or ""except Exception as exc: # pragma: no cover - re-raised by callerresult_box["error"] = exc+ finally:+ # A timed-out call clears its own entry when it finally returns, so the+ # process-wide registry never pins a finished thread.+ with self._EXTERNAL_PREFETCH_LOCK:+ if self._EXTERNAL_PREFETCH_THREADS.get(key) is thread:+ self._EXTERNAL_PREFETCH_THREADS.pop(key, None)thread = spawn_context_thread(_run, name=f"memory-prefetch-{provider.name}")with self._EXTERNAL_PREFETCH_LOCK:- existing = self._EXTERNAL_PREFETCH_THREADS.get(provider.name)+ existing = self._EXTERNAL_PREFETCH_THREADS.get(key)if existing is not None and existing.is_alive():logger.debug("Memory provider '%s' prefetch is still running; skipping this turn", provider.name)return ""- self._EXTERNAL_PREFETCH_THREADS[provider.name] = thread+ self._EXTERNAL_PREFETCH_THREADS[key] = threadthread.start()thread.join(self._external_prefetch_timeout)@@ -411,8 +425,8 @@ class MemoryManager:return ""with self._EXTERNAL_PREFETCH_LOCK:- if self._EXTERNAL_PREFETCH_THREADS.get(provider.name) is thread:- self._EXTERNAL_PREFETCH_THREADS.pop(provider.name, None)+ if self._EXTERNAL_PREFETCH_THREADS.get(key) is thread:+ self._EXTERNAL_PREFETCH_THREADS.pop(key, None)if "error" in result_box:raise result_box["error"]result = result_box.get("value", "")diff --git a/tests/agent/test_memory_provider.py b/tests/agent/test_memory_provider.pyindex b4480bd7da4..e19d8e737bb 100644--- a/tests/agent/test_memory_provider.py+++ b/tests/agent/test_memory_provider.py@@ -482,8 +482,8 @@ class TestMemoryManager:deadline = time.monotonic() + 1.0while (- external.name in mgr._EXTERNAL_PREFETCH_THREADS- and mgr._EXTERNAL_PREFETCH_THREADS[external.name].is_alive()+ (mgr._home_key, external.name) in mgr._EXTERNAL_PREFETCH_THREADS+ and mgr._EXTERNAL_PREFETCH_THREADS[(mgr._home_key, external.name)].is_alive()and time.monotonic() < deadline):time.sleep(0.01)@@ -492,7 +492,7 @@ class TestMemoryManager:assert result == "builtin memory\n\nlate external memory"assert external.prefetch_queries == ["query", "query 3"]- assert external.name not in mgr._EXTERNAL_PREFETCH_THREADS+ assert (mgr._home_key, external.name) not in mgr._EXTERNAL_PREFETCH_THREADSdef test_stuck_prefetch_thread_is_shared_across_managers(self):"""#134392: a new MemoryManager is built per agent init, so the in-flight@@ -512,9 +512,49 @@ class TestMemoryManager:]assert len(live) == 1assert external.prefetch_queries == ["query"]++ # Once the stuck call returns, the shared registry clears and a fresh+ # manager prefetches again.+ external.release.set()+ live[0].join(timeout=2.0)+ assert not live[0].is_alive()+ assert (mgr._home_key, external.name) not in MemoryManager._EXTERNAL_PREFETCH_THREADS+ mgr = MemoryManager(external_prefetch_timeout=1.0)+ mgr.add_provider(external)+ mgr._prefetch_provider(external, "after", session_id="s-after")+ assert external.prefetch_queries == ["query", "after"]finally:external.release.set()+ def test_stuck_prefetch_registry_is_scoped_per_hermes_home(self, tmp_path):+ """Two profiles served by one process may configure a same-named provider+ against different backends; a hung call in one must not suppress the other.+ """+ from hermes_constants import reset_hermes_home_override, set_hermes_home_override++ stuck = BlockingPrefetchProvider("per-home-stuck")+ other = FakeMemoryProvider("per-home-stuck")+ other._prefetch_result = "profile b memory"+ try:+ token = set_hermes_home_override(tmp_path / "profile-a")+ try:+ mgr_a = MemoryManager(external_prefetch_timeout=0.01)+ finally:+ reset_hermes_home_override(token)+ mgr_a.add_provider(stuck)+ assert mgr_a._prefetch_provider(stuck, "q", session_id="a") == ""++ token = set_hermes_home_override(tmp_path / "profile-b")+ try:+ mgr_b = MemoryManager(external_prefetch_timeout=1.0)+ finally:+ reset_hermes_home_override(token)+ mgr_b.add_provider(other)+ assert mgr_b._home_key != mgr_a._home_key+ assert mgr_b._prefetch_provider(other, "q", session_id="b") == "profile b memory"+ finally:+ stuck.release.set()+class TestPluginMemoryDiscovery:"""Memory providers are discovered from plugins/memory/ directory."""