The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(memory): terminate provider writer threads on soft eviction
Jay Stothard99e56ba0edc8
diff --git a/run_agent.py b/run_agent.pyindex 6ed2df0eb42..fd7e9b2a2d5 100644--- a/run_agent.py+++ b/run_agent.py@@ -924,8 +924,10 @@ class AIAgent(def release_clients(self) -> None:"""Release LLM clients and child agents WITHOUT tearing down session tool state (gateway cache- eviction: the session may resume on the same task_id, so processes, sandbox, browser, computer-use and- memory provider are kept). Idempotent; distinct from ``close()``."""+ eviction: the session may resume on the same task_id, so processes, sandbox, browser and+ computer-use are kept). The memory manager is drained + shut down WITHOUT ``on_session_end()``+ so evicted provider writer threads don't leak (extraction is the gateway's pre-evict commit's+ job). Idempotent; distinct from ``close()``."""self._close_active_children(soft=True)# Retire (don't hard-close) the shared client: eviction runs on the gateway memory-manager thread,# and a cross-thread close can release TLS FDs under a still-unwinding worker.@@ -934,6 +936,36 @@ class AIAgent(# The Codex app-server child is an LLM client, not session tool state: the evicted instance is popped# from the cache and a rebuilt agent spawns its own child, so an unclosed one leaks for the gateway's life._quietly(self._close_codex_session)+ # Drain + shut down memory providers WITHOUT on_session_end().+ #+ # Each memory provider instance spawns its own background writer thread+ # (retaindb-writer, openviking-sync, ...). Soft eviction never terminated+ # them: one zombie daemon per evicted agent, each holding stack memory and+ # references to provider clients/queues that block GC. Under a 128-entry+ # cache with heavy turnover (e.g. concurrent 429 retry storms) this piles+ # up dozens of leaked threads and drove the 2026-07-30 gateway liveness+ # crash (exit 75).+ #+ # shutdown_memory_provider() calls on_session_end() first — too heavy for+ # soft eviction: it triggers end-of-session extraction, which a resumed+ # session would duplicate (the gateway's _commit_then_release_soft()+ # already commits extraction via commit_memory_session() BEFORE calling+ # this, so the manager's queue holds exactly that work). Instead call+ # shutdown_all() directly: it bounded-drains the sync executor (giving the+ # pre-evict commit its ~5s) and calls provider shutdown(), which sends+ # each writer its sentinel and joins the thread — no extraction side-effect.+ #+ # Safe for resume: an evicted agent is popped from the cache and never+ # re-added, and the resumed session's freshly-built AIAgent creates a new+ # MemoryManager with fresh provider instances. api_server's parked managers+ # are checked out exclusively, so a live shared manager is never reachable+ # here; delegation/review children run skip_memory=True (manager is None).+ try:+ _mm = getattr(self, "_memory_manager", None)+ if _mm is not None and not getattr(_mm, "_shutting_down", False):+ _quietly(_mm.shutdown_all)+ except Exception:+ passdef close(self) -> None:"""Release every resource this agent holds (idempotent); each phase is guarded so one failure neverdiff --git a/tests/gateway/test_agent_cache.py b/tests/gateway/test_agent_cache.pyindex 2ee30833283..d7feceee8bf 100644--- a/tests/gateway/test_agent_cache.py+++ b/tests/gateway/test_agent_cache.py@@ -546,6 +546,66 @@ class TestAgentCacheIdleResume:f"tabs and cookies gone on resume. Calls: {browser_calls}")+ def test_release_clients_shuts_down_memory_provider(self):+ """release_clients() must call shutdown_all() on the memory manager.++ Memory providers spawn background writer threads (retaindb-writer,+ openviking-sync, ...). Without shutdown on soft eviction, these threads+ pile up as zombies — one per evicted agent — eventually causing thread+ proliferation and gateway liveness crashes.++ release_clients() must drain + shut the manager down WITHOUT calling+ on_session_end() (end-of-session extraction): the gateway's+ _commit_then_release_soft() already commits extraction via+ commit_memory_session() before soft-releasing, and a resumed session+ builds a fresh manager anyway.+ """+ from run_agent import AIAgent++ agent = AIAgent(+ model="anthropic/claude-sonnet-4", api_key="test",+ base_url="https://openrouter.ai/api/v1", provider="openrouter",+ max_iterations=5, quiet_mode=True,+ skip_context_files=True, skip_memory=True,+ session_id="mem-provider-evict-test",+ )++ class _MockMM:+ def __init__(self):+ self.shutdown_all_calls = 0+ self.on_session_end_calls = 0+ def shutdown_all(self):+ self.shutdown_all_calls += 1+ def on_session_end(self, messages):+ self.on_session_end_calls += 1++ mock_mm = _MockMM()+ agent._memory_manager = mock_mm++ try:+ agent.release_clients()+ except Exception:+ pass+ # Snapshot BEFORE the final close(): close() -> shutdown_memory_provider()+ # legitimately calls on_session_end()/shutdown_all() at the hard teardown+ # boundary — the assertions below are about the soft-eviction path only.+ shutdown_calls, session_end_calls = mock_mm.shutdown_all_calls, mock_mm.on_session_end_calls++ try:+ agent.close()+ except Exception:+ pass++ assert shutdown_calls >= 1, (+ "release_clients() must call shutdown_all() to terminate memory "+ "provider writer threads. Leaked threads cause thread proliferation "+ "under cache turnover."+ )+ assert session_end_calls == 0, (+ "release_clients() must NOT call on_session_end() — that triggers "+ "end-of-session extraction, too heavy for a soft-evicted session."+ )+def test_close_vs_release_full_teardown_difference(self, monkeypatch):"""close() tears down task state; release_clients() does not.