Skip to content

The agent that grows with you

fix(memory): orphans from past outages stop counting; half-open probe with a hard cap

kshitijk4poor880a59fe3bcb
diff --git a/agent/memory_manager.py b/agent/memory_manager.py
index 131f43584c8..26bef8714d5 100644
--- a/agent/memory_manager.py
+++ b/agent/memory_manager.py
@@ -44,11 +44,17 @@ _EXTERNAL_PREFETCH_TIMEOUT_S = 8.0
# stuck. Well above the shutdown drain bound and the prefetch timeout so a slow-but-healthy backend
# is never counted.
_BACKGROUND_CALL_STALL_S = 30.0
-# Overdue calls per (home, provider) at which new calls stop being admitted. Python cannot stop a
-# thread, so each stuck call pins one until it returns; tolerating more than one keeps a recovered
-# backend (one orphaned socket, a healthy replacement connection) usable. Calls not yet overdue are
-# not counted (healthy concurrent sessions must run), so a burst inside one stall window can add more.
+# Admission for external provider calls per (home, provider); see _StuckCallGate. Python cannot stop a
+# thread, so every stuck call pins one until it returns. New calls stop once this many calls that started
+# after the provider's last successful return are overdue; older orphans belong to an outage it has
+# since recovered from.
_MAX_STUCK_CALLS = 4
+# While the gate is closed, one call per interval goes through, so a recovered backend proves itself
+# without waiting for an orphan to return.
+_PROBE_INTERVAL_S = 60.0
+# Hard cap on calls in flight per (home, provider), orphans included: failed probes pin one more thread
+# each, so admission (probes too) stops here and a backend that stays wedged pins a bounded number.
+_MAX_INFLIGHT_CALLS = 16
# -- Signature introspection (providers are duck-typed; call shapes vary) -----
@@ -133,21 +139,65 @@ def _redact_messages_for_egress(messages: Optional[list[Any]]) -> Optional[list[
return None if messages is None else list(_redact_for_provider(messages))
-def _stuck_calls(inflight: dict[Any, float], cutoff: float) -> int:
- """How many registered calls have a timestamp before ``cutoff`` (a deadline vs now, or a start time
- vs now minus the stall threshold)."""
- return sum(cutoff > stamp for stamp in inflight.values())
+class _StuckCallGate:
+ """Process-wide admission for one kind of external provider call, keyed by (Hermes home, provider).
+ A new MemoryManager is built per agent, so a per-instance view would let every fresh manager start
+ one more call against a hung backend and pin one more thread. Shared, the gate closes once
+ _MAX_STUCK_CALLS calls that started after the key's last successful return are past their deadline,
+ admits one half-open probe per _PROBE_INTERVAL_S while closed, and never lets more than
+ _MAX_INFLIGHT_CALLS be in flight. Calls not yet overdue are not counted, so healthy concurrent
+ sessions run side by side. Each call removes only its own entry.
+ """
-def _forget_inflight(registry: dict, key: tuple[str, str], token: object) -> bool:
- """Drop one call's entry from a process-wide in-flight registry (caller holds its lock); the key goes
- with its last entry. True when the key is now clear."""
- inflight = registry.get(key)
- if inflight is not None:
- inflight.pop(token, None)
- if not inflight:
- registry.pop(key, None)
- return key not in registry
+ def __init__(self) -> None:
+ self.lock = threading.Lock()
+ self.calls: dict[tuple[str, str], dict[object, tuple[float, float]]] = {} # token -> (start, deadline)
+ self.last_ok: dict[tuple[str, str], float] = {}
+ self.closed_at: dict[tuple[str, str], float] = {} # gate closed / last probe admitted
+ self.skips: dict[tuple[str, str], int] = {}
+
+ def enter(self, key: tuple[str, str], token: object, timeout: float) -> tuple[bool, int]:
+ """Register ``token`` and return ``(True, 0)``, or ``(False, stuck)`` when the call is skipped."""
+ now = time.monotonic()
+ with self.lock:
+ calls = self.calls.setdefault(key, {})
+ last_ok = self.last_ok.get(key, float("-inf"))
+ stuck = sum(start > last_ok and now > deadline for start, deadline in calls.values())
+ if len(calls) >= _MAX_INFLIGHT_CALLS:
+ admit = False
+ elif stuck < _MAX_STUCK_CALLS:
+ admit = True
+ self.closed_at.pop(key, None)
+ else:
+ admit = now - self.closed_at.setdefault(key, now) >= _PROBE_INTERVAL_S
+ if admit:
+ self.closed_at[key] = now
+ if not admit:
+ if not calls:
+ del self.calls[key]
+ return False, stuck
+ # A later outage warns on its first skip, even if an orphan keeps the key alive.
+ self.skips.pop(key, None)
+ calls[token] = (now, now + timeout)
+ return True, 0
+
+ def leave(self, key: tuple[str, str], token: object, *, ok: bool) -> None:
+ with self.lock:
+ calls = self.calls.get(key)
+ if calls is None or calls.pop(token, None) is None:
+ return
+ if calls:
+ if ok:
+ self.last_ok[key] = time.monotonic()
+ return
+ for state in (self.calls, self.last_ok, self.closed_at, self.skips):
+ state.pop(key, None)
+
+ def count_skip(self, key: tuple[str, str]) -> int:
+ with self.lock:
+ skips = self.skips[key] = self.skips.get(key, 0) + 1
+ return skips
class MemoryManager:
@@ -157,25 +207,12 @@ class MemoryManager:
swallows per-provider exceptions.
"""
- # Process-wide registry of in-flight external prefetch threads and their deadlines, keyed by
- # (Hermes home, provider name). A new MemoryManager is built per agent init, so a
- # per-instance registry would let every fresh manager spawn one more stuck thread
- # against the same provider (#134392); shared, once _MAX_STUCK_CALLS calls for the key are
- # past their deadline, new spawns from every manager are suppressed until one returns. Healthy concurrent prefetches from
- # different sessions still run side by side, and each thread removes only its own
- # entry. The home is part of the key because one gateway process can serve several
- # profiles, each with its own config and credentials for a same-named provider: one
- # profile's hung backend must not silence another's recall.
- _EXTERNAL_PREFETCH_THREADS: dict[tuple[str, str], dict[threading.Thread, float]] = {}
- _EXTERNAL_PREFETCH_LOCK = threading.Lock()
- # Same idea for background writes: start times of in-flight sync_turn / queue_prefetch calls per
- # (Hermes home, provider name). Python cannot stop a thread, so a call that never returns pins its
- # mem-sync worker for the process lifetime; managers are rebuilt per agent, so without a shared
- # view every fresh manager would start one more call against the hung backend and pin one more
- # thread. Calls are skipped once _MAX_STUCK_CALLS for the key are stuck past _BACKGROUND_CALL_STALL_S.
- _BACKGROUND_CALLS: dict[tuple[str, str], dict[object, float]] = {}
- _BACKGROUND_SKIPS: dict[tuple[str, str], int] = {}
- _BACKGROUND_CALLS_LOCK = threading.Lock()
+ # Process-wide admission for external prefetch threads (#134392) and for background
+ # sync_turn / queue_prefetch calls on the mem-sync worker. The home is part of the key because one
+ # gateway process can serve several profiles, each with its own config and credentials for a
+ # same-named provider: one profile's hung backend must not silence another's memory.
+ _PREFETCH_GATE = _StuckCallGate()
+ _BACKGROUND_GATE = _StuckCallGate()
def __init__(self, *, external_prefetch_timeout: Optional[float] = None) -> None:
self._providers: list[MemoryProvider] = []
@@ -327,8 +364,7 @@ class MemoryManager:
def _prefetch_provider(self, provider: MemoryProvider, query: str, *, session_id: str = "") -> str:
"""Run one provider's prefetch; external providers are bounded by a timeout. A stuck external
- call keeps running on its daemon thread; once _MAX_STUCK_CALLS are stuck the provider is skipped
- on later turns until one returns."""
+ call keeps running on its daemon thread; _PREFETCH_GATE decides whether later turns call again."""
if provider.name == "builtin":
return provider.prefetch(query, session_id=session_id)
@@ -342,34 +378,26 @@ class MemoryManager:
except Exception as exc: # pragma: no cover - re-raised by caller
result_box["error"] = exc
finally:
- # Every call removes its own entry when it returns, so the registry never pins
- # a finished thread and one call's return never forgets another stuck one.
- with self._EXTERNAL_PREFETCH_LOCK:
- _forget_inflight(self._EXTERNAL_PREFETCH_THREADS, key, thread)
+ self._PREFETCH_GATE.leave(key, thread, ok="error" not in result_box)
thread = spawn_context_thread(_run, name=f"memory-prefetch-{provider.name}")
- with self._EXTERNAL_PREFETCH_LOCK:
- inflight = self._EXTERNAL_PREFETCH_THREADS.setdefault(key, {})
- now = time.monotonic()
- stuck = _stuck_calls(inflight, now)
- if stuck >= _MAX_STUCK_CALLS:
- logger.debug("Memory provider '%s' prefetch has %d stuck calls; skipping this turn",
- provider.name, stuck)
- return ""
- inflight[thread] = now + self._external_prefetch_timeout
- try:
- thread.start()
- except RuntimeError:
- # Never started, so its finally never runs: drop the entry or it would gate the
- # provider forever once its deadline passes.
- _forget_inflight(self._EXTERNAL_PREFETCH_THREADS, key, thread)
- raise
+ admitted, stuck = self._PREFETCH_GATE.enter(key, thread, self._external_prefetch_timeout)
+ if not admitted:
+ logger.debug("Memory provider '%s' prefetch has %d stuck calls; skipping this turn",
+ provider.name, stuck)
+ return ""
+ try:
+ thread.start()
+ except RuntimeError:
+ # Never started, so its finally never runs: drop the entry or it would count forever.
+ self._PREFETCH_GATE.leave(key, thread, ok=False)
+ raise
thread.join(self._external_prefetch_timeout)
if thread.is_alive():
logger.warning(
"Memory provider '%s' prefetch timed out after %.1fs; the call keeps running in the "
- "background (the provider is skipped once %d are stuck)", provider.name,
+ "background (the provider is skipped once %d such calls are stuck)", provider.name,
self._external_prefetch_timeout, _MAX_STUCK_CALLS,
)
return ""
@@ -491,36 +519,30 @@ class MemoryManager:
logger.debug("Inline memory background task failed: %s", e)
def _call_unless_stalled(self, provider: MemoryProvider, call: Callable[[], Any]) -> None:
- """Run a background provider call unless _MAX_STUCK_CALLS earlier ones for the same provider (any
- manager in this process) are stuck; skipping costs that turn's write/prefetch but stops a wedged
- backend from pinning one more thread per fresh manager. The builtin provider is never gated."""
+ """Run a background provider call unless _BACKGROUND_GATE skips it (the provider has stuck calls);
+ skipping costs that turn's write/prefetch but stops a wedged backend from pinning one more
+ thread per fresh manager. The builtin provider is never gated."""
if provider.name == "builtin":
call()
return
key = self._registry_key(provider)
token = object()
- now = time.monotonic()
- with self._BACKGROUND_CALLS_LOCK:
- calls = self._BACKGROUND_CALLS.setdefault(key, {})
- stuck = _stuck_calls(calls, now - _BACKGROUND_CALL_STALL_S)
- if stuck >= _MAX_STUCK_CALLS:
- skips = self._BACKGROUND_SKIPS[key] = self._BACKGROUND_SKIPS.get(key, 0) + 1
- if skips == 1 or skips % 100 == 0:
- logger.warning(
- "Memory provider '%s' has %d background calls stuck for over %.0fs; skipping its "
- "sync/prefetch until one returns (%d skipped)", provider.name, stuck,
- _BACKGROUND_CALL_STALL_S, skips,
- )
- return
- # Admitted: a later outage warns on its first skip, even if an orphan keeps the key alive.
- self._BACKGROUND_SKIPS.pop(key, None)
- calls[token] = now
+ admitted, stuck = self._BACKGROUND_GATE.enter(key, token, _BACKGROUND_CALL_STALL_S)
+ if not admitted:
+ skips = self._BACKGROUND_GATE.count_skip(key)
+ if skips == 1 or skips % 100 == 0:
+ logger.warning(
+ "Memory provider '%s' has %d background calls stuck for over %.0fs; skipping its "
+ "sync/prefetch until it recovers (%d skipped)", provider.name, stuck,
+ _BACKGROUND_CALL_STALL_S, skips,
+ )
+ return
+ ok = False
try:
call()
+ ok = True
finally:
- with self._BACKGROUND_CALLS_LOCK:
- if _forget_inflight(self._BACKGROUND_CALLS, key, token):
- self._BACKGROUND_SKIPS.pop(key, None)
+ self._BACKGROUND_GATE.leave(key, token, ok=ok)
def _registry_key(self, provider: MemoryProvider) -> tuple[str, str]:
"""Process-wide registry key: the home the providers were bound to (``initialize_all``, which may
diff --git a/tests/agent/test_memory_provider.py b/tests/agent/test_memory_provider.py
index 7160bd77a4e..6fa1165d55c 100644
--- a/tests/agent/test_memory_provider.py
+++ b/tests/agent/test_memory_provider.py
@@ -457,6 +457,7 @@ class TestMemoryManager:
import agent.memory_manager as mm
monkeypatch.setattr(mm, "_MAX_STUCK_CALLS", 1) # the gate closes on the first stuck call
+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9)
mgr = MemoryManager(external_prefetch_timeout=0.01)
builtin = FakeMemoryProvider("builtin")
builtin._prefetch_result = "builtin memory"
@@ -485,16 +486,16 @@ class TestMemoryManager:
external.release.set()
deadline = time.monotonic() + 1.0
- while (mgr._home_key, external.name) in mgr._EXTERNAL_PREFETCH_THREADS and time.monotonic() < deadline:
+ while (mgr._home_key, external.name) in mgr._PREFETCH_GATE.calls and time.monotonic() < deadline:
time.sleep(0.01)
result = mgr.prefetch_all("query 3")
assert result == "builtin memory\n\nlate external memory"
assert external.prefetch_queries == ["query", "query 3"]
- assert (mgr._home_key, external.name) not in mgr._EXTERNAL_PREFETCH_THREADS
+ assert (mgr._home_key, external.name) not in mgr._PREFETCH_GATE.calls
- def test_stuck_prefetch_thread_is_shared_across_managers(self, tmp_path):
+ def test_stuck_prefetch_thread_is_shared_across_managers(self, tmp_path, monkeypatch):
"""#134392: a new MemoryManager is built per agent init, so the in-flight
registry must be process-wide — fresh managers against a wedged provider keep at
most ``_MAX_STUCK_CALLS`` live ``memory-prefetch-*`` threads, not one per manager. It
@@ -503,6 +504,7 @@ class TestMemoryManager:
import agent.memory_manager as mm
from hermes_constants import reset_hermes_home_override, set_hermes_home_override
+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9)
stuck = mm._MAX_STUCK_CALLS
external = BlockingPrefetchProvider("cross-manager-stuck")
try:
@@ -534,7 +536,7 @@ class TestMemoryManager:
for thread in live:
thread.join(timeout=2.0)
assert not thread.is_alive()
- assert (mgr._home_key, external.name) not in MemoryManager._EXTERNAL_PREFETCH_THREADS
+ assert (mgr._home_key, external.name) not in MemoryManager._PREFETCH_GATE.calls
mgr = MemoryManager(external_prefetch_timeout=1.0)
mgr.add_provider(external)
mgr._prefetch_provider(external, "after", session_id="s-after")
@@ -560,12 +562,40 @@ class TestMemoryManager:
finally:
orphan.release.set()
+ def test_prefetch_orphans_from_past_outages_do_not_blank_recall(self, monkeypatch):
+ """Four separate blips, each leaving one prefetch that never returns and each followed by a
+ successful recall, must not turn recall off: orphans older than the last success don't count."""
+ import agent.memory_manager as mm
+
+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9)
+ healthy = FakeMemoryProvider("past-outages")
+ healthy._prefetch_result = "recalled"
+ orphans = []
+ try:
+ for i in range(mm._MAX_STUCK_CALLS):
+ orphan = BlockingPrefetchProvider("past-outages")
+ orphans.append(orphan)
+ stuck_mgr = MemoryManager(external_prefetch_timeout=0.01)
+ stuck_mgr.add_provider(orphan)
+ stuck_mgr._prefetch_provider(orphan, "q", session_id=f"blip{i}")
+ assert orphan.started.wait(timeout=2.0)
+ mgr = MemoryManager(external_prefetch_timeout=5.0)
+ mgr.add_provider(healthy)
+ assert mgr._prefetch_provider(healthy, "q", session_id=f"recovered{i}") == "recalled"
+ fresh = MemoryManager(external_prefetch_timeout=5.0)
+ fresh.add_provider(healthy)
+ assert fresh._prefetch_provider(healthy, "q", session_id="after") == "recalled"
+ finally:
+ for orphan in orphans:
+ orphan.release.set()
+
def test_registry_is_keyed_by_the_home_providers_were_bound_to(self, tmp_path, monkeypatch):
"""Two managers built in one ambient home but initialized for profiles A and B: A's stuck
prefetch must not gate B, whose providers belong to another profile."""
import agent.memory_manager as mm
monkeypatch.setattr(mm, "_MAX_STUCK_CALLS", 1)
+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9)
stuck, other = BlockingPrefetchProvider("bound-home"), FakeMemoryProvider("bound-home")
other._prefetch_result = "profile b memory"
mgr_a = MemoryManager(external_prefetch_timeout=0.01)
diff --git a/tests/agent/test_memory_stalled_background_call.py b/tests/agent/test_memory_stalled_background_call.py
index 48591d36889..9e6fb34cdaa 100644
--- a/tests/agent/test_memory_stalled_background_call.py
+++ b/tests/agent/test_memory_stalled_background_call.py
@@ -1,7 +1,8 @@
"""A background provider call that never returns pins its mem-sync worker thread forever (Python cannot
stop a thread). MemoryManagers are rebuilt per agent, so without a process-wide view every fresh manager
started one more call against the hung backend and pinned one more thread. Calls for a provider are now
-skipped once _MAX_STUCK_CALLS earlier calls for it are stuck past the stall threshold."""
+skipped once _MAX_STUCK_CALLS calls that started after its last success are stuck past the stall threshold,
+with one half-open probe per interval and a hard cap on calls in flight."""
import threading
import pytest
@@ -51,14 +52,21 @@ class _HangingProvider(MemoryProvider):
self.release.wait(10)
+_GATE = MemoryManager._BACKGROUND_GATE
+
+
+def _clear_gate():
+ for state in (_GATE.calls, _GATE.last_ok, _GATE.closed_at, _GATE.skips):
+ state.clear()
+
+
@pytest.fixture(autouse=True)
def _short_stall(monkeypatch):
monkeypatch.setattr(mm, "_BACKGROUND_CALL_STALL_S", 0.0)
- MemoryManager._BACKGROUND_CALLS.clear()
- MemoryManager._BACKGROUND_SKIPS.clear()
+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 1e9) # no half-open probe unless a test asks for one
+ _clear_gate()
yield
- MemoryManager._BACKGROUND_CALLS.clear()
- MemoryManager._BACKGROUND_SKIPS.clear()
+ _clear_gate()
def _manager(provider):
@@ -102,8 +110,8 @@ def test_sustained_wedge_pins_at_most_max_stuck_threads():
key = hung[0][0]._registry_key(hung[0][1])
try:
- assert MemoryManager._BACKGROUND_SKIPS[key] == 10
- assert len(MemoryManager._BACKGROUND_CALLS[key]) == mm._MAX_STUCK_CALLS
+ assert _GATE.skips[key] == 10
+ assert len(_GATE.calls[key]) == mm._MAX_STUCK_CALLS
# Idle workers of shut-down executors exit asynchronously.
wait_until(lambda: len(_mem_sync_threads()) <= before, timeout=5, what="idle mem-sync workers to exit")
finally:
@@ -111,7 +119,7 @@ def test_sustained_wedge_pins_at_most_max_stuck_threads():
provider.release.set()
for mgr, _ in hung:
assert mgr.flush_pending(timeout=5)
- assert key not in MemoryManager._BACKGROUND_CALLS
+ assert key not in _GATE.calls
for mgr, _ in hung:
mgr.shutdown_all()
@@ -136,21 +144,20 @@ def test_healthy_replacement_still_writes_behind_an_orphaned_call():
def test_new_outage_warns_on_its_first_skip_even_with_an_orphan_alive(caplog):
- """An orphan keeps the key alive after the gate reopens; the next outage must still warn on
+ """Orphans keep the key alive after the gate reopens; the next outage must still warn on
its first skip instead of waiting for the 100th."""
hung = [_hang_one_call() for _ in range(mm._MAX_STUCK_CALLS)]
key = hung[0][0]._registry_key(hung[0][1])
try:
_skip(_manager(_HangingProvider()), "skipped")
- assert MemoryManager._BACKGROUND_SKIPS[key] == 1
+ assert _GATE.skips[key] == 1
hung[-1][1].release.set() # one returns: the gate reopens, three orphans keep the key alive
assert hung[-1][0].flush_pending(timeout=5)
- reopened, healthy = _hang_one_call() # admitted, then it wedges too: a new outage
- hung.append((reopened, healthy))
+ hung += [_hang_one_call() for _ in range(mm._MAX_STUCK_CALLS)] # a new outage
caplog.clear()
with caplog.at_level("WARNING", logger="agent.memory_manager"):
_skip(_manager(_HangingProvider()), "skipped again")
- assert MemoryManager._BACKGROUND_SKIPS[key] == 1
+ assert _GATE.skips[key] == 1
assert "background calls stuck" in caplog.text
finally:
for _, provider in hung:
@@ -159,6 +166,78 @@ def test_new_outage_warns_on_its_first_skip_even_with_an_orphan_alive(caplog):
mgr.shutdown_all()
+def _orphan_then_recover(orphans):
+ """Each outage leaves one call that never returns, then the backend recovers for one turn."""
+ for _ in range(4):
+ orphans.append(_hang_one_call())
+ recovered = _HangingProvider()
+ recovered.hang_next = False
+ _skip(_manager(recovered), "recovered")
+
+
+def test_orphans_from_past_outages_do_not_close_the_gate():
+ """Four separate blips over the process lifetime, each followed by recovery, must not turn the
+ provider off: a stuck call that started before the last success no longer counts."""
+ orphans = []
+ healthy = _HangingProvider()
+ healthy.hang_next = False
+ try:
+ _orphan_then_recover(orphans)
+ for _ in range(3):
+ fresh = _manager(healthy)
+ fresh.sync_all("again", "reply")
+ fresh.queue_prefetch_all("query")
+ assert fresh.flush_pending(timeout=5)
+ fresh.shutdown_all()
+ assert (healthy.sync_calls, healthy.prefetch_calls) == (3, 3)
+ finally:
+ for _, provider in orphans:
+ provider.release.set()
+ for mgr, _ in orphans:
+ mgr.shutdown_all()
+
+
+def test_closed_gate_admits_a_half_open_probe_that_reopens_it(monkeypatch):
+ """With the gate closed by calls that never return, one probe per interval goes through; once a
+ probe succeeds (the backend recovered) every later call is admitted again."""
+ hung = [_hang_one_call() for _ in range(mm._MAX_STUCK_CALLS)]
+ healthy = _HangingProvider()
+ healthy.hang_next = False
+ try:
+ _skip(_manager(healthy), "closed")
+ assert healthy.sync_calls == 0 # gate closed, probe interval not yet reached
+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 0.0)
+ for turn in ("probe", "after the probe"):
+ _skip(_manager(healthy), turn)
+ assert healthy.sync_calls == 2
+ finally:
+ for _, provider in hung:
+ provider.release.set()
+ for mgr, _ in hung:
+ mgr.shutdown_all()
+
+
+def test_sustained_wedge_with_probes_never_exceeds_the_inflight_cap(monkeypatch):
+ """Every failed probe pins one more thread; at the cap, probing stops, so a backend that stays
+ wedged pins a bounded number of threads however long it lasts."""
+ monkeypatch.setattr(mm, "_PROBE_INTERVAL_S", 0.0)
+ monkeypatch.setattr(mm, "_MAX_INFLIGHT_CALLS", mm._MAX_STUCK_CALLS + 2)
+ hung = [_hang_one_call() for _ in range(mm._MAX_STUCK_CALLS)]
+ key = hung[0][0]._registry_key(hung[0][1])
+ try:
+ for _ in range(2): # two probes admitted, and they wedge too
+ hung.append(_hang_one_call())
+ for turn in range(5):
+ _skip(_manager(_HangingProvider()), f"capped {turn}")
+ assert len(_GATE.calls[key]) == mm._MAX_INFLIGHT_CALLS
+ assert _GATE.skips[key] == 5
+ finally:
+ for _, provider in hung:
+ provider.release.set()
+ for mgr, _ in hung:
+ mgr.shutdown_all()
+
+
def test_slow_call_below_threshold_is_not_skipped(monkeypatch):
monkeypatch.setattr(mm, "_BACKGROUND_CALL_STALL_S", 60.0)
provider = _HangingProvider()