Skip to content

The agent that grows with you

fix(memory): terminate provider writer threads on soft eviction

Jay Stothard99e56ba0edc8
diff --git a/run_agent.py b/run_agent.py
index 6ed2df0eb42..fd7e9b2a2d5 100644
--- a/run_agent.py
+++ b/run_agent.py
@@ -924,8 +924,10 @@ class AIAgent(
def release_clients(self) -> None:
"""Release LLM clients and child agents WITHOUT tearing down session tool state (gateway cache
- eviction: the session may resume on the same task_id, so processes, sandbox, browser, computer-use and
- memory provider are kept). Idempotent; distinct from ``close()``."""
+ eviction: the session may resume on the same task_id, so processes, sandbox, browser and
+ computer-use are kept). The memory manager is drained + shut down WITHOUT ``on_session_end()``
+ so evicted provider writer threads don't leak (extraction is the gateway's pre-evict commit's
+ job). Idempotent; distinct from ``close()``."""
self._close_active_children(soft=True)
# Retire (don't hard-close) the shared client: eviction runs on the gateway memory-manager thread,
# and a cross-thread close can release TLS FDs under a still-unwinding worker.
@@ -934,6 +936,36 @@ class AIAgent(
# The Codex app-server child is an LLM client, not session tool state: the evicted instance is popped
# from the cache and a rebuilt agent spawns its own child, so an unclosed one leaks for the gateway's life.
_quietly(self._close_codex_session)
+ # Drain + shut down memory providers WITHOUT on_session_end().
+ #
+ # Each memory provider instance spawns its own background writer thread
+ # (retaindb-writer, openviking-sync, ...). Soft eviction never terminated
+ # them: one zombie daemon per evicted agent, each holding stack memory and
+ # references to provider clients/queues that block GC. Under a 128-entry
+ # cache with heavy turnover (e.g. concurrent 429 retry storms) this piles
+ # up dozens of leaked threads and drove the 2026-07-30 gateway liveness
+ # crash (exit 75).
+ #
+ # shutdown_memory_provider() calls on_session_end() first — too heavy for
+ # soft eviction: it triggers end-of-session extraction, which a resumed
+ # session would duplicate (the gateway's _commit_then_release_soft()
+ # already commits extraction via commit_memory_session() BEFORE calling
+ # this, so the manager's queue holds exactly that work). Instead call
+ # shutdown_all() directly: it bounded-drains the sync executor (giving the
+ # pre-evict commit its ~5s) and calls provider shutdown(), which sends
+ # each writer its sentinel and joins the thread — no extraction side-effect.
+ #
+ # Safe for resume: an evicted agent is popped from the cache and never
+ # re-added, and the resumed session's freshly-built AIAgent creates a new
+ # MemoryManager with fresh provider instances. api_server's parked managers
+ # are checked out exclusively, so a live shared manager is never reachable
+ # here; delegation/review children run skip_memory=True (manager is None).
+ try:
+ _mm = getattr(self, "_memory_manager", None)
+ if _mm is not None and not getattr(_mm, "_shutting_down", False):
+ _quietly(_mm.shutdown_all)
+ except Exception:
+ pass
def close(self) -> None:
"""Release every resource this agent holds (idempotent); each phase is guarded so one failure never
diff --git a/tests/gateway/test_agent_cache.py b/tests/gateway/test_agent_cache.py
index 2ee30833283..d7feceee8bf 100644
--- a/tests/gateway/test_agent_cache.py
+++ b/tests/gateway/test_agent_cache.py
@@ -546,6 +546,66 @@ class TestAgentCacheIdleResume:
f"tabs and cookies gone on resume. Calls: {browser_calls}"
)
+ def test_release_clients_shuts_down_memory_provider(self):
+ """release_clients() must call shutdown_all() on the memory manager.
+
+ Memory providers spawn background writer threads (retaindb-writer,
+ openviking-sync, ...). Without shutdown on soft eviction, these threads
+ pile up as zombies — one per evicted agent — eventually causing thread
+ proliferation and gateway liveness crashes.
+
+ release_clients() must drain + shut the manager down WITHOUT calling
+ on_session_end() (end-of-session extraction): the gateway's
+ _commit_then_release_soft() already commits extraction via
+ commit_memory_session() before soft-releasing, and a resumed session
+ builds a fresh manager anyway.
+ """
+ from run_agent import AIAgent
+
+ agent = AIAgent(
+ model="anthropic/claude-sonnet-4", api_key="test",
+ base_url="https://openrouter.ai/api/v1", provider="openrouter",
+ max_iterations=5, quiet_mode=True,
+ skip_context_files=True, skip_memory=True,
+ session_id="mem-provider-evict-test",
+ )
+
+ class _MockMM:
+ def __init__(self):
+ self.shutdown_all_calls = 0
+ self.on_session_end_calls = 0
+ def shutdown_all(self):
+ self.shutdown_all_calls += 1
+ def on_session_end(self, messages):
+ self.on_session_end_calls += 1
+
+ mock_mm = _MockMM()
+ agent._memory_manager = mock_mm
+
+ try:
+ agent.release_clients()
+ except Exception:
+ pass
+ # Snapshot BEFORE the final close(): close() -> shutdown_memory_provider()
+ # legitimately calls on_session_end()/shutdown_all() at the hard teardown
+ # boundary — the assertions below are about the soft-eviction path only.
+ shutdown_calls, session_end_calls = mock_mm.shutdown_all_calls, mock_mm.on_session_end_calls
+
+ try:
+ agent.close()
+ except Exception:
+ pass
+
+ assert shutdown_calls >= 1, (
+ "release_clients() must call shutdown_all() to terminate memory "
+ "provider writer threads. Leaked threads cause thread proliferation "
+ "under cache turnover."
+ )
+ assert session_end_calls == 0, (
+ "release_clients() must NOT call on_session_end() — that triggers "
+ "end-of-session extraction, too heavy for a soft-evicted session."
+ )
+
def test_close_vs_release_full_teardown_difference(self, monkeypatch):
"""close() tears down task state; release_clients() does not.