The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(tui_gateway): hold foreign owners through the post-turn drain; keep owners across agent rebuilds
Ben Barclay06bcbc2ab70b
diff --git a/tests/tui_gateway/test_detached_process_notify_keepalive.py b/tests/tui_gateway/test_detached_process_notify_keepalive.pyindex 0f630444150..a13b787142c 100644--- a/tests/tui_gateway/test_detached_process_notify_keepalive.py+++ b/tests/tui_gateway/test_detached_process_notify_keepalive.py@@ -8,10 +8,12 @@ with exit -15, no completion was ever delivered, and the promised follow-up silefrom __future__ import annotations+import contextlibimport importlibimport threadingimport timefrom pathlib import Path+from types import SimpleNamespacefrom unittest.mock import MagicMock, patchimport pytest@@ -292,3 +294,101 @@ def test_disabled_watch_does_not_pin_session(server, registry, detached):proc._watch_disabled = Trueassert server._session_is_reapable(sid, session)+++# ── the post-turn safety-net drain and agent rebuilds ──────────────────────+# Session B finishing a turn drains the shared completion queue; an event owned by idle, detached session A passes+# through B's hands before it is requeued. If A were reapable in that window, the reaper would close A and the+# requeued notice would be dropped as unowned.+++def _post_turn_followups(server, namespace_overrides):+ from tui_gateway import prompt_turn+ from tui_gateway.method_ctx import rebind++ namespace = dict(vars(server))+ namespace.update(_drain_queued_prompt=lambda *_: False,+ _session_profile_runtime_scope=lambda *_: contextlib.nullcontext(), **namespace_overrides)+ return rebind(prompt_turn._run_post_turn_followups, namespace)+++def test_foreign_notice_in_another_sessions_post_turn_drain_keeps_its_owner(server, registry, detached, monkeypatch):+ sid, session, popped = detached+ proc = _spawn(registry, session["session_key"], notify_on_complete=True)+ proc.mark_exited(0)+ registry._move_to_finished(proc)+ other_sid, other = "other-sid", dict(session, session_key="other-owner", history_lock=threading.Lock())+ server._sessions[other_sid] = other+ monkeypatch.setattr(registry, "restore_completions", lambda: 0)+ real_owns, observed = server._session_owns_notification_event, []++ def owns(draining_sid, draining_session, event):+ # B is deciding ownership of A's notice; the queue is empty, so only the in-flight hold covers A.+ if draining_sid == other_sid and event["owner_task_id"] == session["session_key"]:+ observed.append((registry.completion_queue.empty(), server._session_is_reapable(sid, session)))+ server._schedule_ws_orphan_reap(sid)+ server._pending_ws_reaps[sid].callback()+ return real_owns(draining_sid, draining_session, event)++ routed = []+ _post_turn_followups(server, {"_session_owns_notification_event": owns,+ "_notif_handle_ready": lambda *a, **k: routed.append(a[2])})(+ None, other_sid, other, {}, None)++ assert observed == [(True, False)], "A's notice was in B's hands: A must stay protected"+ assert sid in server._sessions and not popped+ assert [e["owner_task_id"] for e in registry.completion_queue.queue] == [session["session_key"]]+ assert routed == [[]], "B routes none of A's events"+++def test_post_turn_drain_routes_its_own_notice_and_releases_every_hold(server, registry, detached, monkeypatch):+ from tools import process_registry_awaiting as awaiting++ sid, session, _ = detached+ for owner in (session["session_key"], "someone-else"):+ proc = _spawn(registry, owner, notify_on_complete=True)+ proc.mark_exited(0)+ registry._move_to_finished(proc)+ monkeypatch.setattr(registry, "restore_completions", lambda: 0)+ routed = []+ _post_turn_followups(server, {"_notif_handle_ready": lambda *a, **k: routed.append(+ [(e["owner_task_id"], server._session_is_reapable(sid, session)) for e in a[2]])})(None, sid, session, {}, None)++ assert routed == [[(session["session_key"], False)]], "own notice routed while its session is still held"+ assert [e["owner_task_id"] for e in registry.completion_queue.queue] == ["someone-else"]+ assert not awaiting._in_flight, "every hold released once routing returned"+ assert server._session_is_reapable(sid, session)+++def test_post_turn_drain_releases_holds_when_routing_raises(server, registry, detached, monkeypatch):+ from tools import process_registry_awaiting as awaiting++ sid, session, _ = detached+ proc = _spawn(registry, session["session_key"], notify_on_complete=True)+ proc.mark_exited(0)+ registry._move_to_finished(proc)+ monkeypatch.setattr(registry, "restore_completions", lambda: 0)++ def fail(*_a, **_k):+ raise RuntimeError("routing failed")++ _post_turn_followups(server, {"_notif_handle_ready": fail})(None, sid, session, {}, None)+ assert not awaiting._in_flight+++def test_agent_rebuild_keeps_pre_compression_process_owners(server, registry, detached, monkeypatch):+ """A Bot Chat capability refresh replaces the agent; a process the old agent started under its pre-compression+ id must still pin the session (and still be killed by the new agent's close)."""+ sid, session, _ = detached+ session["agent"] = SimpleNamespace(_process_owner_task_ids={"pre-compression-id", session["session_key"]},+ _session_db=None, _owns_session_db=False)+ _spawn(registry, "pre-compression-id", notify_on_complete=True)+ assert not server._session_is_reapable(sid, session)+ monkeypatch.setattr(server, "_make_agent", lambda *a, **k: SimpleNamespace(+ _process_owner_task_ids={session["session_key"]}, _session_db=None))+ monkeypatch.setattr(server, "_release_build_profile_scopes", lambda *_: None)++ server._rebuild_session_agent(sid, session)++ assert "pre-compression-id" in session["agent"]._process_owner_task_ids+ assert not server._session_is_reapable(sid, session)diff --git a/tools/process_registry.py b/tools/process_registry.pyindex b285fc414a2..3b41242a38b 100644--- a/tools/process_registry.py+++ b/tools/process_registry.py@@ -2546,14 +2546,11 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOu# ----- Cleanup / Pruning -----def _prune_if_needed(self):- """Drop expired finished sessions, then the oldest survivor while over- MAX_PROCESSES. Must hold _lock. Kept either way: a session whose scope stop is still- pending (kill_all's only handle on a scope that may hold live descendants), and one- whose completion is not yet published (its owner's reaper protection reads it)."""+ """Drop expired, then oldest-over-cap finished sessions (hold _lock); keep pending scope stops and unpublished notices."""+ from tools.process_registry_awaiting import retained_for_deliverynow = time.time()prunable = {sid: s for sid, s in self._finished.items()- if not getattr(s, "_scope_stop_pending", False)- and not (getattr(s, "_finish_claimed", False) and not s._completion_event.is_set())}+ if not getattr(s, "_scope_stop_pending", False) and not retained_for_delivery(s)}expired = [sid for sid, s in prunable.items() if (now - s.started_at) > FINISHED_TTL_SECONDS]over_cap = len(self._running) + len(self._finished) - len(expired) >= MAX_PROCESSESif over_cap and (survivors := [sid for sid in prunable if sid not in expired]):diff --git a/tools/process_registry_awaiting.py b/tools/process_registry_awaiting.pyindex 0a4b86bba04..e1c7f52b0be 100644--- a/tools/process_registry_awaiting.py+++ b/tools/process_registry_awaiting.py@@ -66,14 +66,26 @@ def release(events) -> None:@contextlib.contextmanager-def holding(owner_task_ids):- """Hold ``owner_task_ids`` for a drain that dequeues through another API (``drain_notifications``)."""- owners = [o for o in owner_task_ids if o]- _adjust(owners, +1)+def drained_for(registry, owns):+ """Dequeue everything queued for a caller that consumes only the events ``owns`` accepts (the TUI's post-turn+ safety net). Every dequeued event's owner is held — foreign ones too — until the rejected events are back on+ the queue and the accepted ones (yielded) have been routed, so no owner is ever between queue and hold."""+ registry.restore_completions()+ ready = take_ready(registry.completion_queue, 0)try:- yield+ mine = [evt for evt in ready if owns(evt)]+ for evt in ready:+ if not any(evt is m for m in mine):+ registry.completion_queue.put(evt)+ yield minefinally:- _adjust(owners, -1)+ release(ready)+++def retained_for_delivery(s) -> bool:+ """A finished entry whose completion is not yet published: pruning it would erase the only record that a notice+ is coming, so the registry keeps it until publication."""+ return bool(getattr(s, "_finish_claimed", False)) and not s._completion_event.is_set()def _notifies(s) -> bool:diff --git a/tui_gateway/agent_callbacks.py b/tui_gateway/agent_callbacks.pyindex 0863f12b5e0..f2c019edf49 100644--- a/tui_gateway/agent_callbacks.py+++ b/tui_gateway/agent_callbacks.py@@ -574,6 +574,10 @@ def _rebuild_session_agent(sid: str, session: dict, **kwargs):# already closed the agent it saw, so one installed now is never closed (#49852).closed_midbuild = bool(session.get("_closing"))if not closed_midbuild:+ # The replacement inherits the raw spawning ids: processes started before a compression rotation keep+ # their old owner, and the reapers' notify keepalive (and close's kill set) match on it.+ if inherited := set(getattr(old_agent, "_process_owner_task_ids", ())):+ agent._process_owner_task_ids = {*inherited, *getattr(agent, "_process_owner_task_ids", ())}session.update(agent=agent, config_model_seen=config_model_seen)owned = opened or bool(getattr(old_agent, "_owns_session_db", False))if owned and _transfer_db_to_agent(agent, session_db):diff --git a/tui_gateway/prompt_turn.py b/tui_gateway/prompt_turn.pyindex e00e9c3d7d7..0d3d1793b16 100644--- a/tui_gateway/prompt_turn.py+++ b/tui_gateway/prompt_turn.py@@ -463,18 +463,17 @@ def _run_post_turn_followups(from tools.process_registry import process_registry# _release_turn_scopes / _post_turn_housekeeping have released the worker's runtime scopes. Queue ownership,# notification policy and nested dispatch still belong to this session.- from tools.process_registry_awaiting import holding- # This drain dequeues this session's own events while it is idle; hold its owners until routing finishes so- # the reapers cannot close it between dequeue and turn admission.- with _session_profile_runtime_scope(session), holding(_session_process_owners(session)):- drained = process_registry.drain_notifications(- session_key=session.get("session_key", ""),- owns_event=lambda e: _session_owns_notification_event(sid, session, e),- skip_poll_observed=False)- from tools.process_registry_notifications import format_process_notification+ from tools.process_registry_awaiting import drained_for+ from tools.process_registry_notifications import format_process_notification, should_surface_notification+ key = session.get("session_key", "")+ owns = lambda e: process_registry._owns_event(+ e, key, lambda evt: _session_owns_notification_event(sid, session, evt), e.get("type") == "async_delegation")+ # Every dequeued event's owner — this session's and any other session's — stays held until the foreign events+ # are requeued and ours are routed, so the reapers never see an owner whose notice is in nobody's hands.+ with _session_profile_runtime_scope(session), drained_for(process_registry, owns) as mine:deferred = []_notif_handle_ready(- sid, session, [event for event, _text in drained],+ sid, session, [evt for evt in mine if should_surface_notification(evt)],session.setdefault("_notification_emitted", set()), process_registry,format_process_notification, deferred, owned=True)for event in deferred: