Skip to content

The agent that grows with you

fix(memory): a failed half-open probe no longer restarts the outage warning

kshitijk4poor602d76a72d15
diff --git a/agent/memory_manager.py b/agent/memory_manager.py
index e9c64eb1122..c6525e0c1c1 100644
--- a/agent/memory_manager.py
+++ b/agent/memory_manager.py
@@ -154,7 +154,7 @@ class _StuckCallGate:
self.lock = threading.Lock()
self.calls: dict[tuple[str, str], dict[object, tuple[float, float]]] = {} # token -> (start, deadline)
self.last_ok: dict[tuple[str, str], float] = {}
- self.closed_at: dict[tuple[str, str], float] = {} # gate closed / last probe admitted
+ self.probe_at: dict[tuple[str, str], float] = {} # interval start: gate close, then each probe
self.skips: dict[tuple[str, str], int] = {}
def enter(self, key: tuple[str, str], token: object, timeout: float) -> Optional[tuple[int, int, int]]:
@@ -168,16 +168,15 @@ class _StuckCallGate:
admit = False
elif stuck < _MAX_STUCK_CALLS:
admit = True
- self.closed_at.pop(key, None)
+ self.probe_at.pop(key, None)
+ self.skips.pop(key, None) # gate open: the next outage warns on its first skip
else:
- admit = now - self.closed_at.setdefault(key, now) >= _PROBE_INTERVAL_S
+ admit = now - self.probe_at.setdefault(key, now) >= _PROBE_INTERVAL_S
if admit:
- self.closed_at[key] = now
+ self.probe_at[key] = now
if not admit:
skips = self.skips[key] = self.skips.get(key, 0) + 1
return stuck, len(calls), skips
- # A later outage warns on its first skip, even if an orphan keeps the key alive.
- self.skips.pop(key, None)
calls[token] = (now, now + timeout)
return None
@@ -190,7 +189,7 @@ class _StuckCallGate:
if ok:
self.last_ok[key] = time.monotonic()
return
- for state in (self.calls, self.last_ok, self.closed_at, self.skips):
+ for state in (self.calls, self.last_ok, self.probe_at, self.skips):
state.pop(key, None)
diff --git a/tests/agent/test_memory_stalled_background_call.py b/tests/agent/test_memory_stalled_background_call.py
index fa7ea3d84f9..5386245ba09 100644
--- a/tests/agent/test_memory_stalled_background_call.py
+++ b/tests/agent/test_memory_stalled_background_call.py
@@ -56,7 +56,7 @@ _GATE = MemoryManager._BACKGROUND_GATE
def _clear_gate():
- for state in (_GATE.calls, _GATE.last_ok, _GATE.closed_at, _GATE.skips):
+ for state in (_GATE.calls, _GATE.last_ok, _GATE.probe_at, _GATE.skips):
state.clear()
@@ -219,6 +219,25 @@ def test_closed_gate_admits_a_half_open_probe_that_reopens_it(monkeypatch):
mgr.shutdown_all()
+def test_failed_probe_does_not_restart_the_outage_warning(monkeypatch):
+ """A probe that wedges too is not a recovery: the skip count keeps running, so a long outage
+ warns on its first skip and every 100th, not after every probe."""
+ hung = [_hang_one_call() for _ in range(mm._MAX_STUCK_CALLS)]
+ key = hung[0][0]._registry_key(hung[0][1])
+ try:
+ _skip(_manager(_HangingProvider()), "closed")
+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 0.0)
+ hung.append(_hang_one_call()) # the probe is admitted and hangs
+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9)
+ _skip(_manager(_HangingProvider()), "still closed")
+ assert _GATE.skips[key] == 2
+ finally:
+ for _, provider in hung:
+ provider.release.set()
+ for mgr, _ in hung:
+ mgr.shutdown_all()
+
+
def test_sustained_wedge_with_probes_never_exceeds_the_inflight_cap(monkeypatch):
"""Every failed probe pins one more thread; at the cap, probing stops, so a backend that stays
wedged pins a bounded number of threads however long it lasts."""