The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(memory): orphans from past outages stop counting; half-open probe with a hard cap
kshitijk4poor880a59fe3bcb
diff --git a/agent/memory_manager.py b/agent/memory_manager.pyindex 131f43584c8..26bef8714d5 100644--- a/agent/memory_manager.py+++ b/agent/memory_manager.py@@ -44,11 +44,17 @@ _EXTERNAL_PREFETCH_TIMEOUT_S = 8.0# stuck. Well above the shutdown drain bound and the prefetch timeout so a slow-but-healthy backend# is never counted._BACKGROUND_CALL_STALL_S = 30.0-# Overdue calls per (home, provider) at which new calls stop being admitted. Python cannot stop a-# thread, so each stuck call pins one until it returns; tolerating more than one keeps a recovered-# backend (one orphaned socket, a healthy replacement connection) usable. Calls not yet overdue are-# not counted (healthy concurrent sessions must run), so a burst inside one stall window can add more.+# Admission for external provider calls per (home, provider); see _StuckCallGate. Python cannot stop a+# thread, so every stuck call pins one until it returns. New calls stop once this many calls that started+# after the provider's last successful return are overdue; older orphans belong to an outage it has+# since recovered from._MAX_STUCK_CALLS = 4+# While the gate is closed, one call per interval goes through, so a recovered backend proves itself+# without waiting for an orphan to return.+_PROBE_INTERVAL_S = 60.0+# Hard cap on calls in flight per (home, provider), orphans included: failed probes pin one more thread+# each, so admission (probes too) stops here and a backend that stays wedged pins a bounded number.+_MAX_INFLIGHT_CALLS = 16# -- Signature introspection (providers are duck-typed; call shapes vary) -----@@ -133,21 +139,65 @@ def _redact_messages_for_egress(messages: Optional[list[Any]]) -> Optional[list[return None if messages is None else list(_redact_for_provider(messages))-def _stuck_calls(inflight: dict[Any, float], cutoff: float) -> int:- """How many registered calls have a timestamp before ``cutoff`` (a deadline vs now, or a start time- vs now minus the stall threshold)."""- return sum(cutoff > stamp for stamp in inflight.values())+class _StuckCallGate:+ """Process-wide admission for one kind of external provider call, keyed by (Hermes home, provider).+ A new MemoryManager is built per agent, so a per-instance view would let every fresh manager start+ one more call against a hung backend and pin one more thread. Shared, the gate closes once+ _MAX_STUCK_CALLS calls that started after the key's last successful return are past their deadline,+ admits one half-open probe per _PROBE_INTERVAL_S while closed, and never lets more than+ _MAX_INFLIGHT_CALLS be in flight. Calls not yet overdue are not counted, so healthy concurrent+ sessions run side by side. Each call removes only its own entry.+ """-def _forget_inflight(registry: dict, key: tuple[str, str], token: object) -> bool:- """Drop one call's entry from a process-wide in-flight registry (caller holds its lock); the key goes- with its last entry. True when the key is now clear."""- inflight = registry.get(key)- if inflight is not None:- inflight.pop(token, None)- if not inflight:- registry.pop(key, None)- return key not in registry+ def __init__(self) -> None:+ self.lock = threading.Lock()+ self.calls: dict[tuple[str, str], dict[object, tuple[float, float]]] = {} # token -> (start, deadline)+ self.last_ok: dict[tuple[str, str], float] = {}+ self.closed_at: dict[tuple[str, str], float] = {} # gate closed / last probe admitted+ self.skips: dict[tuple[str, str], int] = {}++ def enter(self, key: tuple[str, str], token: object, timeout: float) -> tuple[bool, int]:+ """Register ``token`` and return ``(True, 0)``, or ``(False, stuck)`` when the call is skipped."""+ now = time.monotonic()+ with self.lock:+ calls = self.calls.setdefault(key, {})+ last_ok = self.last_ok.get(key, float("-inf"))+ stuck = sum(start > last_ok and now > deadline for start, deadline in calls.values())+ if len(calls) >= _MAX_INFLIGHT_CALLS:+ admit = False+ elif stuck < _MAX_STUCK_CALLS:+ admit = True+ self.closed_at.pop(key, None)+ else:+ admit = now - self.closed_at.setdefault(key, now) >= _PROBE_INTERVAL_S+ if admit:+ self.closed_at[key] = now+ if not admit:+ if not calls:+ del self.calls[key]+ return False, stuck+ # A later outage warns on its first skip, even if an orphan keeps the key alive.+ self.skips.pop(key, None)+ calls[token] = (now, now + timeout)+ return True, 0++ def leave(self, key: tuple[str, str], token: object, *, ok: bool) -> None:+ with self.lock:+ calls = self.calls.get(key)+ if calls is None or calls.pop(token, None) is None:+ return+ if calls:+ if ok:+ self.last_ok[key] = time.monotonic()+ return+ for state in (self.calls, self.last_ok, self.closed_at, self.skips):+ state.pop(key, None)++ def count_skip(self, key: tuple[str, str]) -> int:+ with self.lock:+ skips = self.skips[key] = self.skips.get(key, 0) + 1+ return skipsclass MemoryManager:@@ -157,25 +207,12 @@ class MemoryManager:swallows per-provider exceptions."""- # Process-wide registry of in-flight external prefetch threads and their deadlines, keyed by- # (Hermes home, provider name). A new MemoryManager is built per agent init, so a- # per-instance registry would let every fresh manager spawn one more stuck thread- # against the same provider (#134392); shared, once _MAX_STUCK_CALLS calls for the key are- # past their deadline, new spawns from every manager are suppressed until one returns. Healthy concurrent prefetches from- # different sessions still run side by side, and each thread removes only its own- # entry. The home is part of the key because one gateway process can serve several- # profiles, each with its own config and credentials for a same-named provider: one- # profile's hung backend must not silence another's recall.- _EXTERNAL_PREFETCH_THREADS: dict[tuple[str, str], dict[threading.Thread, float]] = {}- _EXTERNAL_PREFETCH_LOCK = threading.Lock()- # Same idea for background writes: start times of in-flight sync_turn / queue_prefetch calls per- # (Hermes home, provider name). Python cannot stop a thread, so a call that never returns pins its- # mem-sync worker for the process lifetime; managers are rebuilt per agent, so without a shared- # view every fresh manager would start one more call against the hung backend and pin one more- # thread. Calls are skipped once _MAX_STUCK_CALLS for the key are stuck past _BACKGROUND_CALL_STALL_S.- _BACKGROUND_CALLS: dict[tuple[str, str], dict[object, float]] = {}- _BACKGROUND_SKIPS: dict[tuple[str, str], int] = {}- _BACKGROUND_CALLS_LOCK = threading.Lock()+ # Process-wide admission for external prefetch threads (#134392) and for background+ # sync_turn / queue_prefetch calls on the mem-sync worker. The home is part of the key because one+ # gateway process can serve several profiles, each with its own config and credentials for a+ # same-named provider: one profile's hung backend must not silence another's memory.+ _PREFETCH_GATE = _StuckCallGate()+ _BACKGROUND_GATE = _StuckCallGate()def __init__(self, *, external_prefetch_timeout: Optional[float] = None) -> None:self._providers: list[MemoryProvider] = []@@ -327,8 +364,7 @@ class MemoryManager:def _prefetch_provider(self, provider: MemoryProvider, query: str, *, session_id: str = "") -> str:"""Run one provider's prefetch; external providers are bounded by a timeout. A stuck external- call keeps running on its daemon thread; once _MAX_STUCK_CALLS are stuck the provider is skipped- on later turns until one returns."""+ call keeps running on its daemon thread; _PREFETCH_GATE decides whether later turns call again."""if provider.name == "builtin":return provider.prefetch(query, session_id=session_id)@@ -342,34 +378,26 @@ class MemoryManager:except Exception as exc: # pragma: no cover - re-raised by callerresult_box["error"] = excfinally:- # Every call removes its own entry when it returns, so the registry never pins- # a finished thread and one call's return never forgets another stuck one.- with self._EXTERNAL_PREFETCH_LOCK:- _forget_inflight(self._EXTERNAL_PREFETCH_THREADS, key, thread)+ self._PREFETCH_GATE.leave(key, thread, ok="error" not in result_box)thread = spawn_context_thread(_run, name=f"memory-prefetch-{provider.name}")- with self._EXTERNAL_PREFETCH_LOCK:- inflight = self._EXTERNAL_PREFETCH_THREADS.setdefault(key, {})- now = time.monotonic()- stuck = _stuck_calls(inflight, now)- if stuck >= _MAX_STUCK_CALLS:- logger.debug("Memory provider '%s' prefetch has %d stuck calls; skipping this turn",- provider.name, stuck)- return ""- inflight[thread] = now + self._external_prefetch_timeout- try:- thread.start()- except RuntimeError:- # Never started, so its finally never runs: drop the entry or it would gate the- # provider forever once its deadline passes.- _forget_inflight(self._EXTERNAL_PREFETCH_THREADS, key, thread)- raise+ admitted, stuck = self._PREFETCH_GATE.enter(key, thread, self._external_prefetch_timeout)+ if not admitted:+ logger.debug("Memory provider '%s' prefetch has %d stuck calls; skipping this turn",+ provider.name, stuck)+ return ""+ try:+ thread.start()+ except RuntimeError:+ # Never started, so its finally never runs: drop the entry or it would count forever.+ self._PREFETCH_GATE.leave(key, thread, ok=False)+ raisethread.join(self._external_prefetch_timeout)if thread.is_alive():logger.warning("Memory provider '%s' prefetch timed out after %.1fs; the call keeps running in the "- "background (the provider is skipped once %d are stuck)", provider.name,+ "background (the provider is skipped once %d such calls are stuck)", provider.name,self._external_prefetch_timeout, _MAX_STUCK_CALLS,)return ""@@ -491,36 +519,30 @@ class MemoryManager:logger.debug("Inline memory background task failed: %s", e)def _call_unless_stalled(self, provider: MemoryProvider, call: Callable[[], Any]) -> None:- """Run a background provider call unless _MAX_STUCK_CALLS earlier ones for the same provider (any- manager in this process) are stuck; skipping costs that turn's write/prefetch but stops a wedged- backend from pinning one more thread per fresh manager. The builtin provider is never gated."""+ """Run a background provider call unless _BACKGROUND_GATE skips it (the provider has stuck calls);+ skipping costs that turn's write/prefetch but stops a wedged backend from pinning one more+ thread per fresh manager. The builtin provider is never gated."""if provider.name == "builtin":call()returnkey = self._registry_key(provider)token = object()- now = time.monotonic()- with self._BACKGROUND_CALLS_LOCK:- calls = self._BACKGROUND_CALLS.setdefault(key, {})- stuck = _stuck_calls(calls, now - _BACKGROUND_CALL_STALL_S)- if stuck >= _MAX_STUCK_CALLS:- skips = self._BACKGROUND_SKIPS[key] = self._BACKGROUND_SKIPS.get(key, 0) + 1- if skips == 1 or skips % 100 == 0:- logger.warning(- "Memory provider '%s' has %d background calls stuck for over %.0fs; skipping its "- "sync/prefetch until one returns (%d skipped)", provider.name, stuck,- _BACKGROUND_CALL_STALL_S, skips,- )- return- # Admitted: a later outage warns on its first skip, even if an orphan keeps the key alive.- self._BACKGROUND_SKIPS.pop(key, None)- calls[token] = now+ admitted, stuck = self._BACKGROUND_GATE.enter(key, token, _BACKGROUND_CALL_STALL_S)+ if not admitted:+ skips = self._BACKGROUND_GATE.count_skip(key)+ if skips == 1 or skips % 100 == 0:+ logger.warning(+ "Memory provider '%s' has %d background calls stuck for over %.0fs; skipping its "+ "sync/prefetch until it recovers (%d skipped)", provider.name, stuck,+ _BACKGROUND_CALL_STALL_S, skips,+ )+ return+ ok = Falsetry:call()+ ok = Truefinally:- with self._BACKGROUND_CALLS_LOCK:- if _forget_inflight(self._BACKGROUND_CALLS, key, token):- self._BACKGROUND_SKIPS.pop(key, None)+ self._BACKGROUND_GATE.leave(key, token, ok=ok)def _registry_key(self, provider: MemoryProvider) -> tuple[str, str]:"""Process-wide registry key: the home the providers were bound to (``initialize_all``, which maydiff --git a/tests/agent/test_memory_provider.py b/tests/agent/test_memory_provider.pyindex 7160bd77a4e..6fa1165d55c 100644--- a/tests/agent/test_memory_provider.py+++ b/tests/agent/test_memory_provider.py@@ -457,6 +457,7 @@ class TestMemoryManager:import agent.memory_manager as mmmonkeypatch.setattr(mm, "_MAX_STUCK_CALLS", 1) # the gate closes on the first stuck call+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9)mgr = MemoryManager(external_prefetch_timeout=0.01)builtin = FakeMemoryProvider("builtin")builtin._prefetch_result = "builtin memory"@@ -485,16 +486,16 @@ class TestMemoryManager:external.release.set()deadline = time.monotonic() + 1.0- while (mgr._home_key, external.name) in mgr._EXTERNAL_PREFETCH_THREADS and time.monotonic() < deadline:+ while (mgr._home_key, external.name) in mgr._PREFETCH_GATE.calls and time.monotonic() < deadline:time.sleep(0.01)result = mgr.prefetch_all("query 3")assert result == "builtin memory\n\nlate external memory"assert external.prefetch_queries == ["query", "query 3"]- assert (mgr._home_key, external.name) not in mgr._EXTERNAL_PREFETCH_THREADS+ assert (mgr._home_key, external.name) not in mgr._PREFETCH_GATE.calls- def test_stuck_prefetch_thread_is_shared_across_managers(self, tmp_path):+ def test_stuck_prefetch_thread_is_shared_across_managers(self, tmp_path, monkeypatch):"""#134392: a new MemoryManager is built per agent init, so the in-flightregistry must be process-wide — fresh managers against a wedged provider keep atmost ``_MAX_STUCK_CALLS`` live ``memory-prefetch-*`` threads, not one per manager. It@@ -503,6 +504,7 @@ class TestMemoryManager:import agent.memory_manager as mmfrom hermes_constants import reset_hermes_home_override, set_hermes_home_override+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9)stuck = mm._MAX_STUCK_CALLSexternal = BlockingPrefetchProvider("cross-manager-stuck")try:@@ -534,7 +536,7 @@ class TestMemoryManager:for thread in live:thread.join(timeout=2.0)assert not thread.is_alive()- assert (mgr._home_key, external.name) not in MemoryManager._EXTERNAL_PREFETCH_THREADS+ assert (mgr._home_key, external.name) not in MemoryManager._PREFETCH_GATE.callsmgr = MemoryManager(external_prefetch_timeout=1.0)mgr.add_provider(external)mgr._prefetch_provider(external, "after", session_id="s-after")@@ -560,12 +562,40 @@ class TestMemoryManager:finally:orphan.release.set()+ def test_prefetch_orphans_from_past_outages_do_not_blank_recall(self, monkeypatch):+ """Four separate blips, each leaving one prefetch that never returns and each followed by a+ successful recall, must not turn recall off: orphans older than the last success don't count."""+ import agent.memory_manager as mm++ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9)+ healthy = FakeMemoryProvider("past-outages")+ healthy._prefetch_result = "recalled"+ orphans = []+ try:+ for i in range(mm._MAX_STUCK_CALLS):+ orphan = BlockingPrefetchProvider("past-outages")+ orphans.append(orphan)+ stuck_mgr = MemoryManager(external_prefetch_timeout=0.01)+ stuck_mgr.add_provider(orphan)+ stuck_mgr._prefetch_provider(orphan, "q", session_id=f"blip{i}")+ assert orphan.started.wait(timeout=2.0)+ mgr = MemoryManager(external_prefetch_timeout=5.0)+ mgr.add_provider(healthy)+ assert mgr._prefetch_provider(healthy, "q", session_id=f"recovered{i}") == "recalled"+ fresh = MemoryManager(external_prefetch_timeout=5.0)+ fresh.add_provider(healthy)+ assert fresh._prefetch_provider(healthy, "q", session_id="after") == "recalled"+ finally:+ for orphan in orphans:+ orphan.release.set()+def test_registry_is_keyed_by_the_home_providers_were_bound_to(self, tmp_path, monkeypatch):"""Two managers built in one ambient home but initialized for profiles A and B: A's stuckprefetch must not gate B, whose providers belong to another profile."""import agent.memory_manager as mmmonkeypatch.setattr(mm, "_MAX_STUCK_CALLS", 1)+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9)stuck, other = BlockingPrefetchProvider("bound-home"), FakeMemoryProvider("bound-home")other._prefetch_result = "profile b memory"mgr_a = MemoryManager(external_prefetch_timeout=0.01)diff --git a/tests/agent/test_memory_stalled_background_call.py b/tests/agent/test_memory_stalled_background_call.pyindex 48591d36889..9e6fb34cdaa 100644--- a/tests/agent/test_memory_stalled_background_call.py+++ b/tests/agent/test_memory_stalled_background_call.py@@ -1,7 +1,8 @@"""A background provider call that never returns pins its mem-sync worker thread forever (Python cannotstop a thread). MemoryManagers are rebuilt per agent, so without a process-wide view every fresh managerstarted one more call against the hung backend and pinned one more thread. Calls for a provider are now-skipped once _MAX_STUCK_CALLS earlier calls for it are stuck past the stall threshold."""+skipped once _MAX_STUCK_CALLS calls that started after its last success are stuck past the stall threshold,+with one half-open probe per interval and a hard cap on calls in flight."""import threadingimport pytest@@ -51,14 +52,21 @@ class _HangingProvider(MemoryProvider):self.release.wait(10)+_GATE = MemoryManager._BACKGROUND_GATE+++def _clear_gate():+ for state in (_GATE.calls, _GATE.last_ok, _GATE.closed_at, _GATE.skips):+ state.clear()++@pytest.fixture(autouse=True)def _short_stall(monkeypatch):monkeypatch.setattr(mm, "_BACKGROUND_CALL_STALL_S", 0.0)- MemoryManager._BACKGROUND_CALLS.clear()- MemoryManager._BACKGROUND_SKIPS.clear()+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9) # no half-open probe unless a test asks for one+ _clear_gate()yield- MemoryManager._BACKGROUND_CALLS.clear()- MemoryManager._BACKGROUND_SKIPS.clear()+ _clear_gate()def _manager(provider):@@ -102,8 +110,8 @@ def test_sustained_wedge_pins_at_most_max_stuck_threads():key = hung[0][0]._registry_key(hung[0][1])try:- assert MemoryManager._BACKGROUND_SKIPS[key] == 10- assert len(MemoryManager._BACKGROUND_CALLS[key]) == mm._MAX_STUCK_CALLS+ assert _GATE.skips[key] == 10+ assert len(_GATE.calls[key]) == mm._MAX_STUCK_CALLS# Idle workers of shut-down executors exit asynchronously.wait_until(lambda: len(_mem_sync_threads()) <= before, timeout=5, what="idle mem-sync workers to exit")finally:@@ -111,7 +119,7 @@ def test_sustained_wedge_pins_at_most_max_stuck_threads():provider.release.set()for mgr, _ in hung:assert mgr.flush_pending(timeout=5)- assert key not in MemoryManager._BACKGROUND_CALLS+ assert key not in _GATE.callsfor mgr, _ in hung:mgr.shutdown_all()@@ -136,21 +144,20 @@ def test_healthy_replacement_still_writes_behind_an_orphaned_call():def test_new_outage_warns_on_its_first_skip_even_with_an_orphan_alive(caplog):- """An orphan keeps the key alive after the gate reopens; the next outage must still warn on+ """Orphans keep the key alive after the gate reopens; the next outage must still warn onits first skip instead of waiting for the 100th."""hung = [_hang_one_call() for _ in range(mm._MAX_STUCK_CALLS)]key = hung[0][0]._registry_key(hung[0][1])try:_skip(_manager(_HangingProvider()), "skipped")- assert MemoryManager._BACKGROUND_SKIPS[key] == 1+ assert _GATE.skips[key] == 1hung[-1][1].release.set() # one returns: the gate reopens, three orphans keep the key aliveassert hung[-1][0].flush_pending(timeout=5)- reopened, healthy = _hang_one_call() # admitted, then it wedges too: a new outage- hung.append((reopened, healthy))+ hung += [_hang_one_call() for _ in range(mm._MAX_STUCK_CALLS)] # a new outagecaplog.clear()with caplog.at_level("WARNING", logger="agent.memory_manager"):_skip(_manager(_HangingProvider()), "skipped again")- assert MemoryManager._BACKGROUND_SKIPS[key] == 1+ assert _GATE.skips[key] == 1assert "background calls stuck" in caplog.textfinally:for _, provider in hung:@@ -159,6 +166,78 @@ def test_new_outage_warns_on_its_first_skip_even_with_an_orphan_alive(caplog):mgr.shutdown_all()+def _orphan_then_recover(orphans):+ """Each outage leaves one call that never returns, then the backend recovers for one turn."""+ for _ in range(4):+ orphans.append(_hang_one_call())+ recovered = _HangingProvider()+ recovered.hang_next = False+ _skip(_manager(recovered), "recovered")+++def test_orphans_from_past_outages_do_not_close_the_gate():+ """Four separate blips over the process lifetime, each followed by recovery, must not turn the+ provider off: a stuck call that started before the last success no longer counts."""+ orphans = []+ healthy = _HangingProvider()+ healthy.hang_next = False+ try:+ _orphan_then_recover(orphans)+ for _ in range(3):+ fresh = _manager(healthy)+ fresh.sync_all("again", "reply")+ fresh.queue_prefetch_all("query")+ assert fresh.flush_pending(timeout=5)+ fresh.shutdown_all()+ assert (healthy.sync_calls, healthy.prefetch_calls) == (3, 3)+ finally:+ for _, provider in orphans:+ provider.release.set()+ for mgr, _ in orphans:+ mgr.shutdown_all()+++def test_closed_gate_admits_a_half_open_probe_that_reopens_it(monkeypatch):+ """With the gate closed by calls that never return, one probe per interval goes through; once a+ probe succeeds (the backend recovered) every later call is admitted again."""+ hung = [_hang_one_call() for _ in range(mm._MAX_STUCK_CALLS)]+ healthy = _HangingProvider()+ healthy.hang_next = False+ try:+ _skip(_manager(healthy), "closed")+ assert healthy.sync_calls == 0 # gate closed, probe interval not yet reached+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 0.0)+ for turn in ("probe", "after the probe"):+ _skip(_manager(healthy), turn)+ assert healthy.sync_calls == 2+ finally:+ for _, provider in hung:+ provider.release.set()+ for mgr, _ in hung:+ mgr.shutdown_all()+++def test_sustained_wedge_with_probes_never_exceeds_the_inflight_cap(monkeypatch):+ """Every failed probe pins one more thread; at the cap, probing stops, so a backend that stays+ wedged pins a bounded number of threads however long it lasts."""+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 0.0)+ monkeypatch.setattr(mm, "_MAX_INFLIGHT_CALLS", mm._MAX_STUCK_CALLS + 2)+ hung = [_hang_one_call() for _ in range(mm._MAX_STUCK_CALLS)]+ key = hung[0][0]._registry_key(hung[0][1])+ try:+ for _ in range(2): # two probes admitted, and they wedge too+ hung.append(_hang_one_call())+ for turn in range(5):+ _skip(_manager(_HangingProvider()), f"capped {turn}")+ assert len(_GATE.calls[key]) == mm._MAX_INFLIGHT_CALLS+ assert _GATE.skips[key] == 5+ finally:+ for _, provider in hung:+ provider.release.set()+ for mgr, _ in hung:+ mgr.shutdown_all()++def test_slow_call_below_threshold_is_not_skipped(monkeypatch):monkeypatch.setattr(mm, "_BACKGROUND_CALL_STALL_S", 60.0)provider = _HangingProvider()