Skip to content

The agent that grows with you

fix(process): read notify-pending from live state instead of a post-exit timer

Ben Barclay79055ee5ed1a
diff --git a/tests/tui_gateway/test_detached_process_notify_keepalive.py b/tests/tui_gateway/test_detached_process_notify_keepalive.py
index 4827c51c9c9..1d84757052a 100644
--- a/tests/tui_gateway/test_detached_process_notify_keepalive.py
+++ b/tests/tui_gateway/test_detached_process_notify_keepalive.py
@@ -30,6 +30,11 @@ def server(tmp_path, monkeypatch):
mod._pending_ws_reaps.clear()
+def _drain(registry):
+ while not registry.completion_queue.empty():
+ registry.completion_queue.get_nowait()
+
+
@pytest.fixture()
def registry():
from tools.process_registry import process_registry
@@ -37,7 +42,9 @@ def registry():
saved = dict(process_registry._running), dict(process_registry._finished)
process_registry._running.clear()
process_registry._finished.clear()
+ _drain(process_registry)
yield process_registry
+ _drain(process_registry)
process_registry._running.clear()
process_registry._finished.clear()
process_registry._running.update(saved[0])
@@ -111,28 +118,46 @@ def test_orphan_reap_rearms_while_process_will_notify(server, registry, detached
assert server._pending_ws_reaps[sid] is _Timer.created[-1]
-def test_exited_process_keeps_session_until_its_notice_can_be_delivered(server, registry, detached):
+def test_exited_process_keeps_session_until_its_notice_is_dequeued(server, registry, detached):
"""The exit is recorded before the completion is queued, and the notice becomes a turn only once the poller
- dequeues it: a reaper firing in that window must not close the session and lose the follow-up."""
+ dequeues it: a reaper firing anywhere in that hand-off must not close the session and lose the follow-up."""
sid, session, _ = detached
proc = _spawn(registry, session["session_key"], notify_on_complete=True)
- proc.exited = True # mark_exited ran; _move_to_finished (exited_at, queue publish) has not yet
+ proc.exited = True # mark_exited ran; _move_to_finished (publish) has not yet
assert not server._session_is_reapable(sid, session)
- registry._finished[proc.id] = registry._running.pop(proc.id)
- proc.exited_at = time.time() # published, not yet dequeued by the poller
-
+ registry._move_to_finished(proc) # published: queued, not yet dequeued
+ assert not registry.completion_queue.empty()
assert not server._session_is_reapable(sid, session)
+ registry.completion_queue.get_nowait() # the poller took it
+ assert server._session_is_reapable(sid, session)
+
+
+def test_queued_notice_survives_registry_pruning(server, registry, detached):
+ """Retained history is pruned by start time and a process cap, on every spawn: a long job's queued completion
+ must keep protecting its owner after its finished entry is gone."""
+ from tools.process_registry import FINISHED_TTL_SECONDS
+
+ sid, session, _ = detached
+ proc = _spawn(registry, session["session_key"], notify_on_complete=True)
+ proc.started_at = time.time() - FINISHED_TTL_SECONDS - 1
+ proc.mark_exited(0)
+ registry._move_to_finished(proc)
+ with registry._lock:
+ registry._prune_if_needed()
+
+ assert proc.id not in registry._finished
+ assert not server._session_is_reapable(sid, session)
-def test_session_released_once_delivery_grace_passes(server, registry, detached):
- from tools.process_registry_awaiting import NOTIFY_DELIVERY_GRACE_SECONDS
+def test_session_released_once_notice_is_delivered(server, registry, detached):
sid, session, popped = detached
proc = _spawn(registry, session["session_key"], notify_on_complete=True)
- proc.exited, proc.exited_at = True, time.time() - NOTIFY_DELIVERY_GRACE_SECONDS - 1
- registry._finished[proc.id] = registry._running.pop(proc.id)
+ proc.mark_exited(0)
+ registry._move_to_finished(proc)
+ registry.completion_queue.get_nowait()
server._schedule_ws_orphan_reap(sid)
_Timer.created[-1].callback()
@@ -140,6 +165,39 @@ def test_session_released_once_delivery_grace_passes(server, registry, detached)
assert sid not in server._sessions and popped == [session]
+def test_notice_held_after_stop_does_not_pin_session(server, registry, detached):
+ """After the user's Stop the poller requeues notices until their next prompt; that wait must not be forever."""
+ sid, session, _ = detached
+ proc = _spawn(registry, session["session_key"], notify_on_complete=True)
+ proc.mark_exited(0)
+ registry._move_to_finished(proc)
+ session["_turn_cancel_requested"] = True
+
+ assert server._session_is_reapable(sid, session)
+
+
+def test_recovered_process_whose_pid_died_releases_session(server, registry, detached, monkeypatch):
+ """A checkpoint-recovered process has no reader thread to observe its exit; a dead PID must not pin its owner."""
+ sid, session, _ = detached
+ _spawn(registry, session["session_key"], notify_on_complete=True, detached=True, pid=12345)
+ monkeypatch.setattr(registry, "_detached_host_fate", lambda *_: "gone")
+
+ assert server._session_is_reapable(sid, session)
+
+
[email protected]("unit", ["", "hermes-proc.scope"])
+def test_uncollected_recovered_exit_never_pins_session(server, registry, detached, unit):
+ """A recovered entry closed without a collected exit (exited_at stays 0, a scope keeps it retained) queues no
+ completion, so it must not hold its owner however long ago it ended."""
+ sid, session, _ = detached
+ proc = _spawn(registry, session["session_key"], notify_on_complete=True, detached=True, pid=12345,
+ systemd_unit=unit)
+ registry._prune_uncollected_detached(proc)
+
+ assert proc.exited and not proc.exited_at
+ assert server._session_is_reapable(sid, session)
+
+
def test_silent_background_process_does_not_pin_session(server, registry, detached):
"""A process nobody asked to hear from (a dev server) must not make its session immortal."""
sid, session, _ = detached
diff --git a/tools/process_registry_awaiting.py b/tools/process_registry_awaiting.py
index 08a539b328b..21bfd7c50ab 100644
--- a/tools/process_registry_awaiting.py
+++ b/tools/process_registry_awaiting.py
@@ -1,32 +1,41 @@
"""Whether a background process will still report back to the agent that spawned it.
-The session reapers keep an owner alive while this is true: the completion reaches the model only through that
-owner's notification poller, and closing the owner kills the process (``agent_close``).
+The session reapers keep an owner alive while this is true: the notice reaches the model only through that owner's
+notification poller, and closing the owner kills the process (``agent_close``).
+
+Pending is read from live state, not estimated with a timer: the process is still running and notifies, its exit is
+recorded but its completion not yet published, or an event it produced is still queued. Each state ends on its own
+(the process exits, the publish lands, a poller dequeues the event), so an owner is never pinned after its notice is
+handed over, and pruning retained process history cannot release an owner whose notice is still in flight.
"""
from __future__ import annotations
-import time
-# How long an owner stays "awaited" after its notifying process exits: covers publish → poller dequeue →
-# turn claim (sub-second when idle). Bounded so an undeliverable notice cannot pin its owner.
-NOTIFY_DELIVERY_GRACE_SECONDS = 60.0
+def _notifies(s) -> bool:
+ return bool(s.notify_on_complete or (s.watch_patterns and not s._watch_disabled))
-def awaits_notification(registry, owner_task_ids, *, delivery_grace: float = NOTIFY_DELIVERY_GRACE_SECONDS) -> bool:
- """Whether a process spawned by one of ``owner_task_ids`` will still report back: it notifies
- (``notify_on_complete`` or live ``watch_patterns``) and is running, or exited less than ``delivery_grace``
- seconds ago — its completion is published after the exit is recorded and becomes an agent turn only once a
- poller dequeues it, so the owner must outlive that hand-off too. Keyed on the RAW spawning owner, the same
- field agent close kills by, so it survives compression rotating the session key."""
+def awaits_notification(registry, owner_task_ids, *, include_queued: bool = True) -> bool:
+ """Whether a process spawned by one of ``owner_task_ids`` still has a notice on its way. Keyed on the RAW
+ spawning owner, the same field agent close kills by, so it survives compression rotating the session key.
+ ``include_queued=False`` ignores already-published events — for an owner whose poller is holding them back
+ (the user pressed Stop) until a prompt that only the user can send."""
owners = {o for o in owner_task_ids if o}
if not owners:
return False
- now = time.time()
+ # Running. _any_running refreshes checkpoint-recovered entries (outside the registry lock) first, so a recovered
+ # PID that died unobserved reads as gone instead of pinning its owner.
+ if registry._any_running(lambda s: s.owner_task_id in owners and _notifies(s)):
+ return True
with registry._lock:
- sessions = [*registry._running.values(), *registry._finished.values()]
- return any(
- s.owner_task_id in owners
- and (s.notify_on_complete or (s.watch_patterns and not s._watch_disabled))
- and (not s.exited or now - (s.exited_at or now) < delivery_grace)
- for s in sessions)
+ # Exit recorded, completion not yet queued: _move_to_finished sets _completion_event only after the publish.
+ # Entries closed without a collected exit (recovered PID gone or reused) set it too, so they never count.
+ if any(s.owner_task_id in owners and s.notify_on_complete and s.exited and not s._completion_event.is_set()
+ for store in (registry._running, registry._finished) for s in store.values()):
+ return True
+ if not include_queued:
+ return False
+ queue = registry.completion_queue
+ with queue.mutex: # published, not yet dequeued (or requeued while the owner was busy)
+ return any(isinstance(evt, dict) and evt.get("owner_task_id") in owners for evt in queue.queue)
diff --git a/tui_gateway/session_notifications.py b/tui_gateway/session_notifications.py
index c05b8ac6a9a..0b2d0d23219 100644
--- a/tui_gateway/session_notifications.py
+++ b/tui_gateway/session_notifications.py
@@ -324,16 +324,17 @@ def _session_owns_live_wakeup_schedule(session: dict) -> bool:
def _session_owns_pending_process_notification(session: dict) -> bool:
"""True while a background process this session started will still report back into it (``notify_on_complete``
- or ``watch_patterns``): running, or exited so recently its completion may not have become a turn yet. The notice
+ or ``watch_patterns``): running, exited with its completion not yet published, or published but not yet dequeued. The notice
reaches the model only through THIS session's notification poller, and closing the session kills the process
(``agent_close``), so the reapers must keep it alive. Ownership is the set close() kills by — the agent's raw
- spawning task ids, which keep pre-compression ids after the session key rotates. Bounded by the process: the
- session is released a short grace after it exits."""
+ spawning task ids, which keep pre-compression ids after the session key rotates. Bounded by delivery: the session
+ is released once the notice has been dequeued."""
agent = session.get("agent")
owners = {*getattr(agent, "_process_owner_task_ids", ()), str(session.get("session_key") or "")}
from tools.process_registry import process_registry
from tools.process_registry_awaiting import awaits_notification
- return awaits_notification(process_registry, owners)
+ # After Stop the poller requeues notices until the user's next prompt: those must not make the session immortal.
+ return awaits_notification(process_registry, owners, include_queued=not session.get("_turn_cancel_requested"))
def _maybe_fire_tui_loop_tick(sid: str, session: dict) -> None: