The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(memory): skip background calls while one is stuck against the same provider
kshitijk4poord3e7573ba09a
diff --git a/agent/memory_manager.py b/agent/memory_manager.pyindex 784e358ff6d..1015a95b2cc 100644--- a/agent/memory_manager.py+++ b/agent/memory_manager.py@@ -12,6 +12,7 @@ import jsonimport loggingimport reimport threading+import timefrom concurrent.futures import Future, ThreadPoolExecutor, waitfrom functools import partialfrom typing import Any, Callable, Dict, List, Optional@@ -37,6 +38,10 @@ _LEGACY_PRE_COMPRESS_API_VERSION = 1# blocks interpreter exit._SYNC_DRAIN_TIMEOUT_S = 5.0_EXTERNAL_PREFETCH_TIMEOUT_S = 8.0+# A background provider call (sync_turn / queue_prefetch) running longer than this is treated as+# stuck: later calls for the same provider are skipped until it returns. Well above the shutdown+# drain bound and the prefetch timeout so a slow-but-healthy backend is never skipped.+_BACKGROUND_CALL_STALL_S = 30.0# -- Signature introspection (providers are duck-typed; call shapes vary) -----@@ -237,6 +242,14 @@ class MemoryManager:# silence another's recall._EXTERNAL_PREFETCH_THREADS: dict[tuple[str, str], threading.Thread] = {}_EXTERNAL_PREFETCH_LOCK = threading.Lock()+ # Same idea for background writes: start times of in-flight sync_turn / queue_prefetch calls per+ # (Hermes home, provider name). Python cannot stop a thread, so a call that never returns pins its+ # mem-sync worker for the process lifetime; managers are rebuilt per agent, so without a shared+ # view every fresh manager would start one more call against the hung backend and pin one more+ # thread. Calls are skipped while any call for the key is stuck past _BACKGROUND_CALL_STALL_S.+ _BACKGROUND_CALLS: dict[tuple[str, str], dict[object, float]] = {}+ _BACKGROUND_SKIPS: dict[tuple[str, str], int] = {}+ _BACKGROUND_CALLS_LOCK = threading.Lock()def __init__(self, *, external_prefetch_timeout: Optional[float] = None) -> None:self._providers: list[MemoryProvider] = []@@ -460,7 +473,8 @@ class MemoryManager:returnclean_query = _redact_for_provider(clean_query)self._submit_background(lambda: self._each_provider(- "queue_prefetch failed (non-fatal)", lambda p: p.queue_prefetch(clean_query, session_id=session_id),+ "queue_prefetch failed (non-fatal)",+ lambda p: self._call_unless_stalled(p, lambda: p.queue_prefetch(clean_query, session_id=session_id)),providers=providers,), kind="prefetch")@@ -496,7 +510,8 @@ class MemoryManager:for keyword, value in optional_kwargs.items():if value is not None and self._provider_sync_accepts(provider, keyword):kwargs[keyword] = value- provider.sync_turn(clean_user_content, assistant_content, **kwargs)+ self._call_unless_stalled(+ provider, lambda: provider.sync_turn(clean_user_content, assistant_content, **kwargs))self._submit_background(lambda: self._each_provider("sync_turn failed", _sync, level=logging.WARNING, providers=providers)@@ -541,6 +556,36 @@ class MemoryManager:except Exception as e: # pragma: no cover - fn guards internallylogger.debug("Inline memory background task failed: %s", e)+ def _call_unless_stalled(self, provider: MemoryProvider, call: Callable[[], Any]) -> None:+ """Run a background provider call unless an earlier one for the same provider (any manager in+ this process) is stuck; skipping costs that turn's write/prefetch but keeps the thread count+ bounded. The builtin provider is local and never gated."""+ if provider.name == "builtin":+ call()+ return+ key = (self._home_key, provider.name)+ token = object()+ now = time.monotonic()+ with self._BACKGROUND_CALLS_LOCK:+ calls = self._BACKGROUND_CALLS.setdefault(key, {})+ if any(now - started > _BACKGROUND_CALL_STALL_S for started in calls.values()):+ skips = self._BACKGROUND_SKIPS[key] = self._BACKGROUND_SKIPS.get(key, 0) + 1+ if skips == 1 or skips % 100 == 0:+ logger.warning(+ "Memory provider '%s' has a background call stuck for over %.0fs; skipping its "+ "sync/prefetch until it returns (%d skipped)", provider.name, _BACKGROUND_CALL_STALL_S, skips,+ )+ return+ calls[token] = now+ try:+ call()+ finally:+ with self._BACKGROUND_CALLS_LOCK:+ calls.pop(token, None)+ if not calls and self._BACKGROUND_CALLS.get(key) is calls:+ self._BACKGROUND_CALLS.pop(key, None)+ self._BACKGROUND_SKIPS.pop(key, None)+def _forget_background_future(self, future: Future) -> None:with self._sync_executor_lock:self._background_futures.pop(future, None)diff --git a/tests/agent/test_memory_stalled_background_call.py b/tests/agent/test_memory_stalled_background_call.pynew file mode 100644index 00000000000..6dea0378b87--- /dev/null+++ b/tests/agent/test_memory_stalled_background_call.py@@ -0,0 +1,157 @@+"""A background provider call that never returns pins its mem-sync worker thread forever (Python cannot+stop a thread). MemoryManagers are rebuilt per agent, so without a process-wide view every fresh manager+started one more call against the hung backend and pinned one more thread. Calls for a provider are now+skipped while an earlier call for it is stuck past the stall threshold."""+import threading+import time++import pytest++import agent.memory_manager as mm+from agent.memory_manager import MemoryManager+from agent.memory_provider import MemoryProvider+++class _HangingProvider(MemoryProvider):+ def __init__(self, name: str = "hangs"):+ self._name = name+ self.release = threading.Event()+ self.started = threading.Event()+ self.sync_calls = 0+ self.prefetch_calls = 0+ self.hang_next = True++ @property+ def name(self) -> str:+ return self._name++ def initialize(self, session_id: str = "", **kwargs) -> None:+ pass++ def is_available(self) -> bool:+ return True++ def system_prompt_block(self) -> str:+ return ""++ def prefetch(self, query, *, session_id: str = "") -> str:+ return ""++ def get_tool_schemas(self):+ return []++ def queue_prefetch(self, query, *, session_id: str = "") -> None:+ self.prefetch_calls += 1++ def sync_turn(self, user_content, assistant_content, *, session_id: str = "") -> None:+ self.sync_calls += 1+ if self.hang_next:+ self.hang_next = False+ self.started.set()+ self.release.wait(10)++[email protected](autouse=True)+def _short_stall(monkeypatch):+ monkeypatch.setattr(mm, "_BACKGROUND_CALL_STALL_S", 0.0)+ MemoryManager._BACKGROUND_CALLS.clear()+ MemoryManager._BACKGROUND_SKIPS.clear()+ yield+ MemoryManager._BACKGROUND_CALLS.clear()+ MemoryManager._BACKGROUND_SKIPS.clear()+++def _manager(provider):+ mgr = MemoryManager()+ mgr._providers.append(provider)+ mgr._has_external = True+ return mgr+++def _mem_sync_threads():+ return [t for t in threading.enumerate() if t.name.startswith("mem-sync")]+++def test_stuck_call_stops_fresh_managers_from_pinning_more_threads():+ provider = _HangingProvider()+ first = _manager(provider)+ first.sync_all("hello", "hi")+ assert provider.started.wait(5)+ before = len(_mem_sync_threads())++ for _ in range(5):+ fresh = _manager(provider)+ fresh.sync_all("again", "reply")+ fresh.queue_prefetch_all("query")+ assert fresh.flush_pending(timeout=5) # returns: nothing blocked behind the hung call+ fresh.shutdown_all()++ assert provider.sync_calls == 1 and provider.prefetch_calls == 0+ assert MemoryManager._BACKGROUND_SKIPS[(first._home_key, provider.name)] == 10+ deadline = time.monotonic() + 5 # idle workers of shut-down executors exit asynchronously+ while len(_mem_sync_threads()) > before and time.monotonic() < deadline:+ time.sleep(0.02)+ assert len(_mem_sync_threads()) <= before++ provider.release.set()+ assert first.flush_pending(timeout=5)+ assert (first._home_key, provider.name) not in MemoryManager._BACKGROUND_CALLS+ first.shutdown_all()+++def test_calls_resume_once_the_stuck_call_returns():+ provider = _HangingProvider()+ mgr = _manager(provider)+ mgr.sync_all("hello", "hi")+ assert provider.started.wait(5)+ provider.release.set()+ assert mgr.flush_pending(timeout=5)++ mgr.sync_all("next", "turn")+ mgr.queue_prefetch_all("query")+ assert mgr.flush_pending(timeout=5)+ assert provider.sync_calls == 2 and provider.prefetch_calls == 1+ mgr.shutdown_all()+++def test_stall_is_scoped_per_provider_and_home(monkeypatch):+ hung = _HangingProvider("hangs")+ mgr = _manager(hung)+ mgr.sync_all("hello", "hi")+ assert hung.started.wait(5)++ other = _HangingProvider("healthy")+ other.hang_next = False+ other_mgr = _manager(other)+ other_mgr.sync_all("a", "b")+ assert other_mgr.flush_pending(timeout=5)+ assert other.sync_calls == 1++ monkeypatch.setattr(mm, "hermes_home_key", lambda: "/other/profile")+ same_name_other_home = _HangingProvider("hangs")+ same_name_other_home.hang_next = False+ profile_mgr = _manager(same_name_other_home)+ profile_mgr.sync_all("a", "b")+ assert profile_mgr.flush_pending(timeout=5)+ assert same_name_other_home.sync_calls == 1++ hung.release.set()+ for m in (mgr, other_mgr, profile_mgr):+ m.shutdown_all()+++def test_slow_call_below_threshold_is_not_skipped(monkeypatch):+ monkeypatch.setattr(mm, "_BACKGROUND_CALL_STALL_S", 60.0)+ provider = _HangingProvider()+ first = _manager(provider)+ first.sync_all("hello", "hi")+ assert provider.started.wait(5)++ second = _manager(provider)+ second.sync_all("again", "reply")+ assert second.flush_pending(timeout=5)+ assert provider.sync_calls == 2++ provider.release.set()+ for m in (first, second):+ m.shutdown_all()