The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(process): read notify-pending from live state instead of a post-exit timer
Ben Barclay79055ee5ed1a
diff --git a/tests/tui_gateway/test_detached_process_notify_keepalive.py b/tests/tui_gateway/test_detached_process_notify_keepalive.pyindex 4827c51c9c9..1d84757052a 100644--- a/tests/tui_gateway/test_detached_process_notify_keepalive.py+++ b/tests/tui_gateway/test_detached_process_notify_keepalive.py@@ -30,6 +30,11 @@ def server(tmp_path, monkeypatch):mod._pending_ws_reaps.clear()+def _drain(registry):+ while not registry.completion_queue.empty():+ registry.completion_queue.get_nowait()++@pytest.fixture()def registry():from tools.process_registry import process_registry@@ -37,7 +42,9 @@ def registry():saved = dict(process_registry._running), dict(process_registry._finished)process_registry._running.clear()process_registry._finished.clear()+ _drain(process_registry)yield process_registry+ _drain(process_registry)process_registry._running.clear()process_registry._finished.clear()process_registry._running.update(saved[0])@@ -111,28 +118,46 @@ def test_orphan_reap_rearms_while_process_will_notify(server, registry, detachedassert server._pending_ws_reaps[sid] is _Timer.created[-1]-def test_exited_process_keeps_session_until_its_notice_can_be_delivered(server, registry, detached):+def test_exited_process_keeps_session_until_its_notice_is_dequeued(server, registry, detached):"""The exit is recorded before the completion is queued, and the notice becomes a turn only once the poller- dequeues it: a reaper firing in that window must not close the session and lose the follow-up."""+ dequeues it: a reaper firing anywhere in that hand-off must not close the session and lose the follow-up."""sid, session, _ = detachedproc = _spawn(registry, session["session_key"], notify_on_complete=True)- proc.exited = True # mark_exited ran; _move_to_finished (exited_at, queue publish) has not yet+ proc.exited = True # mark_exited ran; _move_to_finished (publish) has not yetassert not server._session_is_reapable(sid, session)- registry._finished[proc.id] = registry._running.pop(proc.id)- proc.exited_at = time.time() # published, not yet dequeued by the poller-+ registry._move_to_finished(proc) # published: queued, not yet dequeued+ assert not registry.completion_queue.empty()assert not server._session_is_reapable(sid, session)+ registry.completion_queue.get_nowait() # the poller took it+ assert server._session_is_reapable(sid, session)+++def test_queued_notice_survives_registry_pruning(server, registry, detached):+ """Retained history is pruned by start time and a process cap, on every spawn: a long job's queued completion+ must keep protecting its owner after its finished entry is gone."""+ from tools.process_registry import FINISHED_TTL_SECONDS++ sid, session, _ = detached+ proc = _spawn(registry, session["session_key"], notify_on_complete=True)+ proc.started_at = time.time() - FINISHED_TTL_SECONDS - 1+ proc.mark_exited(0)+ registry._move_to_finished(proc)+ with registry._lock:+ registry._prune_if_needed()++ assert proc.id not in registry._finished+ assert not server._session_is_reapable(sid, session)-def test_session_released_once_delivery_grace_passes(server, registry, detached):- from tools.process_registry_awaiting import NOTIFY_DELIVERY_GRACE_SECONDS+def test_session_released_once_notice_is_delivered(server, registry, detached):sid, session, popped = detachedproc = _spawn(registry, session["session_key"], notify_on_complete=True)- proc.exited, proc.exited_at = True, time.time() - NOTIFY_DELIVERY_GRACE_SECONDS - 1- registry._finished[proc.id] = registry._running.pop(proc.id)+ proc.mark_exited(0)+ registry._move_to_finished(proc)+ registry.completion_queue.get_nowait()server._schedule_ws_orphan_reap(sid)_Timer.created[-1].callback()@@ -140,6 +165,39 @@ def test_session_released_once_delivery_grace_passes(server, registry, detached)assert sid not in server._sessions and popped == [session]+def test_notice_held_after_stop_does_not_pin_session(server, registry, detached):+ """After the user's Stop the poller requeues notices until their next prompt; that wait must not be forever."""+ sid, session, _ = detached+ proc = _spawn(registry, session["session_key"], notify_on_complete=True)+ proc.mark_exited(0)+ registry._move_to_finished(proc)+ session["_turn_cancel_requested"] = True++ assert server._session_is_reapable(sid, session)+++def test_recovered_process_whose_pid_died_releases_session(server, registry, detached, monkeypatch):+ """A checkpoint-recovered process has no reader thread to observe its exit; a dead PID must not pin its owner."""+ sid, session, _ = detached+ _spawn(registry, session["session_key"], notify_on_complete=True, detached=True, pid=12345)+ monkeypatch.setattr(registry, "_detached_host_fate", lambda *_: "gone")++ assert server._session_is_reapable(sid, session)++[email protected]("unit", ["", "hermes-proc.scope"])+def test_uncollected_recovered_exit_never_pins_session(server, registry, detached, unit):+ """A recovered entry closed without a collected exit (exited_at stays 0, a scope keeps it retained) queues no+ completion, so it must not hold its owner however long ago it ended."""+ sid, session, _ = detached+ proc = _spawn(registry, session["session_key"], notify_on_complete=True, detached=True, pid=12345,+ systemd_unit=unit)+ registry._prune_uncollected_detached(proc)++ assert proc.exited and not proc.exited_at+ assert server._session_is_reapable(sid, session)++def test_silent_background_process_does_not_pin_session(server, registry, detached):"""A process nobody asked to hear from (a dev server) must not make its session immortal."""sid, session, _ = detacheddiff --git a/tools/process_registry_awaiting.py b/tools/process_registry_awaiting.pyindex 08a539b328b..21bfd7c50ab 100644--- a/tools/process_registry_awaiting.py+++ b/tools/process_registry_awaiting.py@@ -1,32 +1,41 @@"""Whether a background process will still report back to the agent that spawned it.-The session reapers keep an owner alive while this is true: the completion reaches the model only through that-owner's notification poller, and closing the owner kills the process (``agent_close``).+The session reapers keep an owner alive while this is true: the notice reaches the model only through that owner's+notification poller, and closing the owner kills the process (``agent_close``).++Pending is read from live state, not estimated with a timer: the process is still running and notifies, its exit is+recorded but its completion not yet published, or an event it produced is still queued. Each state ends on its own+(the process exits, the publish lands, a poller dequeues the event), so an owner is never pinned after its notice is+handed over, and pruning retained process history cannot release an owner whose notice is still in flight."""from __future__ import annotations-import time-# How long an owner stays "awaited" after its notifying process exits: covers publish → poller dequeue →-# turn claim (sub-second when idle). Bounded so an undeliverable notice cannot pin its owner.-NOTIFY_DELIVERY_GRACE_SECONDS = 60.0+def _notifies(s) -> bool:+ return bool(s.notify_on_complete or (s.watch_patterns and not s._watch_disabled))-def awaits_notification(registry, owner_task_ids, *, delivery_grace: float = NOTIFY_DELIVERY_GRACE_SECONDS) -> bool:- """Whether a process spawned by one of ``owner_task_ids`` will still report back: it notifies- (``notify_on_complete`` or live ``watch_patterns``) and is running, or exited less than ``delivery_grace``- seconds ago — its completion is published after the exit is recorded and becomes an agent turn only once a- poller dequeues it, so the owner must outlive that hand-off too. Keyed on the RAW spawning owner, the same- field agent close kills by, so it survives compression rotating the session key."""+def awaits_notification(registry, owner_task_ids, *, include_queued: bool = True) -> bool:+ """Whether a process spawned by one of ``owner_task_ids`` still has a notice on its way. Keyed on the RAW+ spawning owner, the same field agent close kills by, so it survives compression rotating the session key.+ ``include_queued=False`` ignores already-published events — for an owner whose poller is holding them back+ (the user pressed Stop) until a prompt that only the user can send."""owners = {o for o in owner_task_ids if o}if not owners:return False- now = time.time()+ # Running. _any_running refreshes checkpoint-recovered entries (outside the registry lock) first, so a recovered+ # PID that died unobserved reads as gone instead of pinning its owner.+ if registry._any_running(lambda s: s.owner_task_id in owners and _notifies(s)):+ return Truewith registry._lock:- sessions = [*registry._running.values(), *registry._finished.values()]- return any(- s.owner_task_id in owners- and (s.notify_on_complete or (s.watch_patterns and not s._watch_disabled))- and (not s.exited or now - (s.exited_at or now) < delivery_grace)- for s in sessions)+ # Exit recorded, completion not yet queued: _move_to_finished sets _completion_event only after the publish.+ # Entries closed without a collected exit (recovered PID gone or reused) set it too, so they never count.+ if any(s.owner_task_id in owners and s.notify_on_complete and s.exited and not s._completion_event.is_set()+ for store in (registry._running, registry._finished) for s in store.values()):+ return True+ if not include_queued:+ return False+ queue = registry.completion_queue+ with queue.mutex: # published, not yet dequeued (or requeued while the owner was busy)+ return any(isinstance(evt, dict) and evt.get("owner_task_id") in owners for evt in queue.queue)diff --git a/tui_gateway/session_notifications.py b/tui_gateway/session_notifications.pyindex c05b8ac6a9a..0b2d0d23219 100644--- a/tui_gateway/session_notifications.py+++ b/tui_gateway/session_notifications.py@@ -324,16 +324,17 @@ def _session_owns_live_wakeup_schedule(session: dict) -> bool:def _session_owns_pending_process_notification(session: dict) -> bool:"""True while a background process this session started will still report back into it (``notify_on_complete``- or ``watch_patterns``): running, or exited so recently its completion may not have become a turn yet. The notice+ or ``watch_patterns``): running, exited with its completion not yet published, or published but not yet dequeued. The noticereaches the model only through THIS session's notification poller, and closing the session kills the process(``agent_close``), so the reapers must keep it alive. Ownership is the set close() kills by — the agent's raw- spawning task ids, which keep pre-compression ids after the session key rotates. Bounded by the process: the- session is released a short grace after it exits."""+ spawning task ids, which keep pre-compression ids after the session key rotates. Bounded by delivery: the session+ is released once the notice has been dequeued."""agent = session.get("agent")owners = {*getattr(agent, "_process_owner_task_ids", ()), str(session.get("session_key") or "")}from tools.process_registry import process_registryfrom tools.process_registry_awaiting import awaits_notification- return awaits_notification(process_registry, owners)+ # After Stop the poller requeues notices until the user's next prompt: those must not make the session immortal.+ return awaits_notification(process_registry, owners, include_queued=not session.get("_turn_cancel_requested"))def _maybe_fire_tui_loop_tick(sid: str, session: dict) -> None: