Skip to content

The agent that grows with you

fix(memory): key the shared prefetch registry per Hermes home and self-clear it

kshitijk4poor4b29e6102bbc
diff --git a/agent/memory_manager.py b/agent/memory_manager.py
index a84dc9f05d6..784e358ff6d 100644
--- a/agent/memory_manager.py
+++ b/agent/memory_manager.py
@@ -24,6 +24,7 @@ from agent.memory_caller_scope import bound_scope, caller_scope, claim_instance,
from agent.memory_context import StreamingContextScrubber, build_memory_context_block, sanitize_context
from agent.memory_provider import MemoryProvider, PRE_COMPRESS_CHECKPOINT_API_VERSION, ctx_bound, spawn_context_thread
from agent.skill_commands import extract_user_instruction_from_skill_message
+from hermes_constants import hermes_home_key
from tools.hook_output_spill import get_spill_config, spill_if_oversized
from tools.registry import tool_error
@@ -226,12 +227,15 @@ class MemoryManager:
swallows per-provider exceptions.
"""
- # Process-wide registry of in-flight external prefetch threads, keyed by provider
- # name. A new MemoryManager is built per agent init, so a per-instance registry
- # would let every fresh manager spawn one more stuck thread against the same
- # provider (#134392); shared, a stuck call suppresses new spawns from any manager
- # until it returns.
- _EXTERNAL_PREFETCH_THREADS: dict[str, threading.Thread] = {}
+ # Process-wide registry of in-flight external prefetch threads, keyed by
+ # (Hermes home, provider name). A new MemoryManager is built per agent init, so a
+ # per-instance registry would let every fresh manager spawn one more stuck thread
+ # against the same provider (#134392); shared, a stuck call suppresses new spawns
+ # from any manager until it returns. The home is part of the key because one
+ # gateway process can serve several profiles, each with its own config and
+ # credentials for a same-named provider: one profile's hung backend must not
+ # silence another's recall.
+ _EXTERNAL_PREFETCH_THREADS: dict[tuple[str, str], threading.Thread] = {}
_EXTERNAL_PREFETCH_LOCK = threading.Lock()
def __init__(self, *, external_prefetch_timeout: Optional[float] = None) -> None:
@@ -244,6 +248,8 @@ class MemoryManager:
if timeout <= 0:
raise ValueError("external_prefetch_timeout must be positive")
self._external_prefetch_timeout = timeout
+ # Captured at build time, inside the profile scope the agent is built for.
+ self._home_key = hermes_home_key()
# Single-worker background executor for end-of-turn sync/prefetch, created lazily so
# the builtin-only path spawns no threads; one worker serializes a provider's writes.
self._sync_executor: Optional[ThreadPoolExecutor] = None
@@ -387,19 +393,27 @@ class MemoryManager:
result_box: dict[str, Any] = {}
+ key = (self._home_key, provider.name)
+
def _run() -> None:
try:
result_box["value"] = provider.prefetch(query, session_id=session_id) or ""
except Exception as exc: # pragma: no cover - re-raised by caller
result_box["error"] = exc
+ finally:
+ # A timed-out call clears its own entry when it finally returns, so the
+ # process-wide registry never pins a finished thread.
+ with self._EXTERNAL_PREFETCH_LOCK:
+ if self._EXTERNAL_PREFETCH_THREADS.get(key) is thread:
+ self._EXTERNAL_PREFETCH_THREADS.pop(key, None)
thread = spawn_context_thread(_run, name=f"memory-prefetch-{provider.name}")
with self._EXTERNAL_PREFETCH_LOCK:
- existing = self._EXTERNAL_PREFETCH_THREADS.get(provider.name)
+ existing = self._EXTERNAL_PREFETCH_THREADS.get(key)
if existing is not None and existing.is_alive():
logger.debug("Memory provider '%s' prefetch is still running; skipping this turn", provider.name)
return ""
- self._EXTERNAL_PREFETCH_THREADS[provider.name] = thread
+ self._EXTERNAL_PREFETCH_THREADS[key] = thread
thread.start()
thread.join(self._external_prefetch_timeout)
@@ -411,8 +425,8 @@ class MemoryManager:
return ""
with self._EXTERNAL_PREFETCH_LOCK:
- if self._EXTERNAL_PREFETCH_THREADS.get(provider.name) is thread:
- self._EXTERNAL_PREFETCH_THREADS.pop(provider.name, None)
+ if self._EXTERNAL_PREFETCH_THREADS.get(key) is thread:
+ self._EXTERNAL_PREFETCH_THREADS.pop(key, None)
if "error" in result_box:
raise result_box["error"]
result = result_box.get("value", "")
diff --git a/tests/agent/test_memory_provider.py b/tests/agent/test_memory_provider.py
index b4480bd7da4..e19d8e737bb 100644
--- a/tests/agent/test_memory_provider.py
+++ b/tests/agent/test_memory_provider.py
@@ -482,8 +482,8 @@ class TestMemoryManager:
deadline = time.monotonic() + 1.0
while (
- external.name in mgr._EXTERNAL_PREFETCH_THREADS
- and mgr._EXTERNAL_PREFETCH_THREADS[external.name].is_alive()
+ (mgr._home_key, external.name) in mgr._EXTERNAL_PREFETCH_THREADS
+ and mgr._EXTERNAL_PREFETCH_THREADS[(mgr._home_key, external.name)].is_alive()
and time.monotonic() < deadline
):
time.sleep(0.01)
@@ -492,7 +492,7 @@ class TestMemoryManager:
assert result == "builtin memory\n\nlate external memory"
assert external.prefetch_queries == ["query", "query 3"]
- assert external.name not in mgr._EXTERNAL_PREFETCH_THREADS
+ assert (mgr._home_key, external.name) not in mgr._EXTERNAL_PREFETCH_THREADS
def test_stuck_prefetch_thread_is_shared_across_managers(self):
"""#134392: a new MemoryManager is built per agent init, so the in-flight
@@ -512,9 +512,49 @@ class TestMemoryManager:
]
assert len(live) == 1
assert external.prefetch_queries == ["query"]
+
+ # Once the stuck call returns, the shared registry clears and a fresh
+ # manager prefetches again.
+ external.release.set()
+ live[0].join(timeout=2.0)
+ assert not live[0].is_alive()
+ assert (mgr._home_key, external.name) not in MemoryManager._EXTERNAL_PREFETCH_THREADS
+ mgr = MemoryManager(external_prefetch_timeout=1.0)
+ mgr.add_provider(external)
+ mgr._prefetch_provider(external, "after", session_id="s-after")
+ assert external.prefetch_queries == ["query", "after"]
finally:
external.release.set()
+ def test_stuck_prefetch_registry_is_scoped_per_hermes_home(self, tmp_path):
+ """Two profiles served by one process may configure a same-named provider
+ against different backends; a hung call in one must not suppress the other.
+ """
+ from hermes_constants import reset_hermes_home_override, set_hermes_home_override
+
+ stuck = BlockingPrefetchProvider("per-home-stuck")
+ other = FakeMemoryProvider("per-home-stuck")
+ other._prefetch_result = "profile b memory"
+ try:
+ token = set_hermes_home_override(tmp_path / "profile-a")
+ try:
+ mgr_a = MemoryManager(external_prefetch_timeout=0.01)
+ finally:
+ reset_hermes_home_override(token)
+ mgr_a.add_provider(stuck)
+ assert mgr_a._prefetch_provider(stuck, "q", session_id="a") == ""
+
+ token = set_hermes_home_override(tmp_path / "profile-b")
+ try:
+ mgr_b = MemoryManager(external_prefetch_timeout=1.0)
+ finally:
+ reset_hermes_home_override(token)
+ mgr_b.add_provider(other)
+ assert mgr_b._home_key != mgr_a._home_key
+ assert mgr_b._prefetch_provider(other, "q", session_id="b") == "profile b memory"
+ finally:
+ stuck.release.set()
+
class TestPluginMemoryDiscovery:
"""Memory providers are discovered from plugins/memory/ directory."""