Skip to content

The agent that grows with you

fix(memory): skip background calls while one is stuck against the same provider

kshitijk4poord3e7573ba09a
diff --git a/agent/memory_manager.py b/agent/memory_manager.py
index 784e358ff6d..1015a95b2cc 100644
--- a/agent/memory_manager.py
+++ b/agent/memory_manager.py
@@ -12,6 +12,7 @@ import json
import logging
import re
import threading
+import time
from concurrent.futures import Future, ThreadPoolExecutor, wait
from functools import partial
from typing import Any, Callable, Dict, List, Optional
@@ -37,6 +38,10 @@ _LEGACY_PRE_COMPRESS_API_VERSION = 1
# blocks interpreter exit.
_SYNC_DRAIN_TIMEOUT_S = 5.0
_EXTERNAL_PREFETCH_TIMEOUT_S = 8.0
+# A background provider call (sync_turn / queue_prefetch) running longer than this is treated as
+# stuck: later calls for the same provider are skipped until it returns. Well above the shutdown
+# drain bound and the prefetch timeout so a slow-but-healthy backend is never skipped.
+_BACKGROUND_CALL_STALL_S = 30.0
# -- Signature introspection (providers are duck-typed; call shapes vary) -----
@@ -237,6 +242,14 @@ class MemoryManager:
# silence another's recall.
_EXTERNAL_PREFETCH_THREADS: dict[tuple[str, str], threading.Thread] = {}
_EXTERNAL_PREFETCH_LOCK = threading.Lock()
+ # Same idea for background writes: start times of in-flight sync_turn / queue_prefetch calls per
+ # (Hermes home, provider name). Python cannot stop a thread, so a call that never returns pins its
+ # mem-sync worker for the process lifetime; managers are rebuilt per agent, so without a shared
+ # view every fresh manager would start one more call against the hung backend and pin one more
+ # thread. Calls are skipped while any call for the key is stuck past _BACKGROUND_CALL_STALL_S.
+ _BACKGROUND_CALLS: dict[tuple[str, str], dict[object, float]] = {}
+ _BACKGROUND_SKIPS: dict[tuple[str, str], int] = {}
+ _BACKGROUND_CALLS_LOCK = threading.Lock()
def __init__(self, *, external_prefetch_timeout: Optional[float] = None) -> None:
self._providers: list[MemoryProvider] = []
@@ -460,7 +473,8 @@ class MemoryManager:
return
clean_query = _redact_for_provider(clean_query)
self._submit_background(lambda: self._each_provider(
- "queue_prefetch failed (non-fatal)", lambda p: p.queue_prefetch(clean_query, session_id=session_id),
+ "queue_prefetch failed (non-fatal)",
+ lambda p: self._call_unless_stalled(p, lambda: p.queue_prefetch(clean_query, session_id=session_id)),
providers=providers,
), kind="prefetch")
@@ -496,7 +510,8 @@ class MemoryManager:
for keyword, value in optional_kwargs.items():
if value is not None and self._provider_sync_accepts(provider, keyword):
kwargs[keyword] = value
- provider.sync_turn(clean_user_content, assistant_content, **kwargs)
+ self._call_unless_stalled(
+ provider, lambda: provider.sync_turn(clean_user_content, assistant_content, **kwargs))
self._submit_background(
lambda: self._each_provider("sync_turn failed", _sync, level=logging.WARNING, providers=providers)
@@ -541,6 +556,36 @@ class MemoryManager:
except Exception as e: # pragma: no cover - fn guards internally
logger.debug("Inline memory background task failed: %s", e)
+ def _call_unless_stalled(self, provider: MemoryProvider, call: Callable[[], Any]) -> None:
+ """Run a background provider call unless an earlier one for the same provider (any manager in
+ this process) is stuck; skipping costs that turn's write/prefetch but keeps the thread count
+ bounded. The builtin provider is local and never gated."""
+ if provider.name == "builtin":
+ call()
+ return
+ key = (self._home_key, provider.name)
+ token = object()
+ now = time.monotonic()
+ with self._BACKGROUND_CALLS_LOCK:
+ calls = self._BACKGROUND_CALLS.setdefault(key, {})
+ if any(now - started > _BACKGROUND_CALL_STALL_S for started in calls.values()):
+ skips = self._BACKGROUND_SKIPS[key] = self._BACKGROUND_SKIPS.get(key, 0) + 1
+ if skips == 1 or skips % 100 == 0:
+ logger.warning(
+ "Memory provider '%s' has a background call stuck for over %.0fs; skipping its "
+ "sync/prefetch until it returns (%d skipped)", provider.name, _BACKGROUND_CALL_STALL_S, skips,
+ )
+ return
+ calls[token] = now
+ try:
+ call()
+ finally:
+ with self._BACKGROUND_CALLS_LOCK:
+ calls.pop(token, None)
+ if not calls and self._BACKGROUND_CALLS.get(key) is calls:
+ self._BACKGROUND_CALLS.pop(key, None)
+ self._BACKGROUND_SKIPS.pop(key, None)
+
def _forget_background_future(self, future: Future) -> None:
with self._sync_executor_lock:
self._background_futures.pop(future, None)
diff --git a/tests/agent/test_memory_stalled_background_call.py b/tests/agent/test_memory_stalled_background_call.py
new file mode 100644
index 00000000000..6dea0378b87
--- /dev/null
+++ b/tests/agent/test_memory_stalled_background_call.py
@@ -0,0 +1,157 @@
+"""A background provider call that never returns pins its mem-sync worker thread forever (Python cannot
+stop a thread). MemoryManagers are rebuilt per agent, so without a process-wide view every fresh manager
+started one more call against the hung backend and pinned one more thread. Calls for a provider are now
+skipped while an earlier call for it is stuck past the stall threshold."""
+import threading
+import time
+
+import pytest
+
+import agent.memory_manager as mm
+from agent.memory_manager import MemoryManager
+from agent.memory_provider import MemoryProvider
+
+
+class _HangingProvider(MemoryProvider):
+ def __init__(self, name: str = "hangs"):
+ self._name = name
+ self.release = threading.Event()
+ self.started = threading.Event()
+ self.sync_calls = 0
+ self.prefetch_calls = 0
+ self.hang_next = True
+
+ @property
+ def name(self) -> str:
+ return self._name
+
+ def initialize(self, session_id: str = "", **kwargs) -> None:
+ pass
+
+ def is_available(self) -> bool:
+ return True
+
+ def system_prompt_block(self) -> str:
+ return ""
+
+ def prefetch(self, query, *, session_id: str = "") -> str:
+ return ""
+
+ def get_tool_schemas(self):
+ return []
+
+ def queue_prefetch(self, query, *, session_id: str = "") -> None:
+ self.prefetch_calls += 1
+
+ def sync_turn(self, user_content, assistant_content, *, session_id: str = "") -> None:
+ self.sync_calls += 1
+ if self.hang_next:
+ self.hang_next = False
+ self.started.set()
+ self.release.wait(10)
+
+
[email protected](autouse=True)
+def _short_stall(monkeypatch):
+ monkeypatch.setattr(mm, "_BACKGROUND_CALL_STALL_S", 0.0)
+ MemoryManager._BACKGROUND_CALLS.clear()
+ MemoryManager._BACKGROUND_SKIPS.clear()
+ yield
+ MemoryManager._BACKGROUND_CALLS.clear()
+ MemoryManager._BACKGROUND_SKIPS.clear()
+
+
+def _manager(provider):
+ mgr = MemoryManager()
+ mgr._providers.append(provider)
+ mgr._has_external = True
+ return mgr
+
+
+def _mem_sync_threads():
+ return [t for t in threading.enumerate() if t.name.startswith("mem-sync")]
+
+
+def test_stuck_call_stops_fresh_managers_from_pinning_more_threads():
+ provider = _HangingProvider()
+ first = _manager(provider)
+ first.sync_all("hello", "hi")
+ assert provider.started.wait(5)
+ before = len(_mem_sync_threads())
+
+ for _ in range(5):
+ fresh = _manager(provider)
+ fresh.sync_all("again", "reply")
+ fresh.queue_prefetch_all("query")
+ assert fresh.flush_pending(timeout=5) # returns: nothing blocked behind the hung call
+ fresh.shutdown_all()
+
+ assert provider.sync_calls == 1 and provider.prefetch_calls == 0
+ assert MemoryManager._BACKGROUND_SKIPS[(first._home_key, provider.name)] == 10
+ deadline = time.monotonic() + 5 # idle workers of shut-down executors exit asynchronously
+ while len(_mem_sync_threads()) > before and time.monotonic() < deadline:
+ time.sleep(0.02)
+ assert len(_mem_sync_threads()) <= before
+
+ provider.release.set()
+ assert first.flush_pending(timeout=5)
+ assert (first._home_key, provider.name) not in MemoryManager._BACKGROUND_CALLS
+ first.shutdown_all()
+
+
+def test_calls_resume_once_the_stuck_call_returns():
+ provider = _HangingProvider()
+ mgr = _manager(provider)
+ mgr.sync_all("hello", "hi")
+ assert provider.started.wait(5)
+ provider.release.set()
+ assert mgr.flush_pending(timeout=5)
+
+ mgr.sync_all("next", "turn")
+ mgr.queue_prefetch_all("query")
+ assert mgr.flush_pending(timeout=5)
+ assert provider.sync_calls == 2 and provider.prefetch_calls == 1
+ mgr.shutdown_all()
+
+
+def test_stall_is_scoped_per_provider_and_home(monkeypatch):
+ hung = _HangingProvider("hangs")
+ mgr = _manager(hung)
+ mgr.sync_all("hello", "hi")
+ assert hung.started.wait(5)
+
+ other = _HangingProvider("healthy")
+ other.hang_next = False
+ other_mgr = _manager(other)
+ other_mgr.sync_all("a", "b")
+ assert other_mgr.flush_pending(timeout=5)
+ assert other.sync_calls == 1
+
+ monkeypatch.setattr(mm, "hermes_home_key", lambda: "/other/profile")
+ same_name_other_home = _HangingProvider("hangs")
+ same_name_other_home.hang_next = False
+ profile_mgr = _manager(same_name_other_home)
+ profile_mgr.sync_all("a", "b")
+ assert profile_mgr.flush_pending(timeout=5)
+ assert same_name_other_home.sync_calls == 1
+
+ hung.release.set()
+ for m in (mgr, other_mgr, profile_mgr):
+ m.shutdown_all()
+
+
+def test_slow_call_below_threshold_is_not_skipped(monkeypatch):
+ monkeypatch.setattr(mm, "_BACKGROUND_CALL_STALL_S", 60.0)
+ provider = _HangingProvider()
+ first = _manager(provider)
+ first.sync_all("hello", "hi")
+ assert provider.started.wait(5)
+
+ second = _manager(provider)
+ second.sync_all("again", "reply")
+ assert second.flush_pending(timeout=5)
+ assert provider.sync_calls == 2
+
+ provider.release.set()
+ for m in (first, second):
+ m.shutdown_all()