Skip to content

The agent that grows with you

fix(tui): pace retained notification ownership refusals

funky-xamarince2ef6df22b0
diff --git a/tests/tui_gateway/test_notification_refusal_backoff.py b/tests/tui_gateway/test_notification_refusal_backoff.py
new file mode 100644
index 00000000000..90e9d0d4517
--- /dev/null
+++ b/tests/tui_gateway/test_notification_refusal_backoff.py
@@ -0,0 +1,65 @@
+"""Refused producer turns must not hot-loop against the canonical ledger (#140385)."""
+
+import contextlib
+import queue
+import threading
+from types import SimpleNamespace
+
+import pytest
+
+from hermes_state import SessionDB
+from hermes_state_runtime import admit_session_input, begin_runtime_epoch, claim_session_input
+from tui_gateway import server
+
+
[email protected]("kind", ["completion", "watch_match"])
+def test_canonical_busy_notification_backs_off_without_consuming_work(monkeypatch, tmp_path, kind):
+ with contextlib.closing(SessionDB(db_path=tmp_path / "state.db")) as db:
+ key = "notification-owner"
+ db.create_session(key, "desktop")
+ epoch = begin_runtime_epoch(db, instance_id="gateway")
+ admit_session_input(db, epoch=epoch, principal_id="owner", session_id=key,
+ request_id="busy", payload={"text": "running"})
+ claim_session_input(db, epoch=epoch, session_id=key)
+ monkeypatch.setattr(server, "_session_db", lambda session: contextlib.nullcontext(db))
+ monkeypatch.setattr(server, "_ensure_session_db_row", lambda session: True)
+ emitted = []
+ monkeypatch.setattr(server, "_emit", lambda *args: emitted.append(args))
+ waits = []
+ monkeypatch.setattr(server, "time", SimpleNamespace(sleep=waits.append, time=lambda: 1))
+ session = {"history_lock": threading.RLock(), "session_key": key, "running": False}
+ registry = SimpleNamespace(completion_queue=queue.Queue())
+ monkeypatch.setattr("tools.process_registry.process_registry", registry)
+ monkeypatch.setattr("tools.async_delegation.claim_event_delivery", lambda *a: "")
+ completed = []
+ monkeypatch.setattr("tools.async_delegation.complete_event_delivery", lambda *a: completed.append(a))
+ monkeypatch.setattr("tools.process_registry_notifications.ProcessNotificationBatch.render", lambda *a: "finished")
+ monkeypatch.setattr("tools.process_registry_notifications.ProcessNotificationBatch.display_text", lambda *a: "finished")
+ event = {"type": kind, "session_id": "proc", "session_key": key}
+
+ def dispatch():
+ if kind == "completion":
+ server._notif_dispatch_completions("ui", session, [(event, "finished")], registry, None)
+ else:
+ assert server._notif_claim_turn(session)
+ server._notif_dispatch_event("ui", session, event, "finished")
+
+ dispatch()
+ assert session["running"] is False
+ assert db.get_messages_as_conversation(key) == []
+ assert waits and waits[0] > 0, "refused notification must yield before its caller requeues it"
+ assert not emitted, "a retained notification must not paint a failed user turn"
+ assert registry.completion_queue.get_nowait() is event
+ assert not completed
+
+ # Human submissions still report ownership refusals.
+ assert server._run_prompt_submit("human", "ui", session, "hello") is False
+ assert emitted[0][0] == "error"
+
+ # A later accepted notification settles once and never takes the refusal delay.
+ waits.clear()
+ monkeypatch.setattr(server, "_run_prompt_submit", lambda *a, **k: True)
+ dispatch()
+ assert len(completed) == 1
+ assert registry.completion_queue.empty()
+ assert not waits
diff --git a/tui_gateway/prompt_turn.py b/tui_gateway/prompt_turn.py
index 3540abb37d1..00764dc8603 100644
--- a/tui_gateway/prompt_turn.py
+++ b/tui_gateway/prompt_turn.py
@@ -116,7 +116,7 @@ def _plan_goal_compression_recovery(
def _admit_prompt_turn(
sid: str, session: dict, text: Any, image_paths: list[str] | None,
queued_prompt_generation: int | None, display_kind: str | None,
- display_metadata: dict | None) -> tuple[list[str], Any] | None:
+ display_metadata: dict | None, *, report_ownership_refusal: bool = True) -> tuple[list[str], Any] | None:
"""Ownership + liveness gate every turn source must cross; ``(images, agent)`` or None.
Synthesized turns (auto-continue, wake-ups) call ``_run_prompt_submit`` directly — the
bypass that once let a second backend run a duplicate turn."""
@@ -130,7 +130,8 @@ def _admit_prompt_turn(
with session["history_lock"]:
session["running"] = False
session.pop("_submit_user_row", None) # no turn runs: the submit-time row stays as the send
- _emit("error", sid, {"message": str(ownership_refusal)})
+ if report_ownership_refusal:
+ _emit("error", sid, {"message": str(ownership_refusal)})
return None
with session["history_lock"]:
if session.get("_closing") or (
@@ -1091,7 +1092,7 @@ def _run_prompt_submit(
display_metadata: dict | None = None, image_paths: list[str] | None = None,
queued_prompt_generation: int | None = None,
terminal_callback: Callable[[dict[str, Any]], None] | None = None,
- turn_author: dict | None = None) -> bool:
+ turn_author: dict | None = None, report_ownership_refusal: bool = True) -> bool:
# Every dispatch binds the session's own row (session_key, real source) before the turn writes:
# the synthesized turns that enter here directly (crash auto-continue, queued-prompt drain,
# wake-ups) bypass prompt.submit's persist, and a row-less turn is otherwise materialized by
@@ -1101,7 +1102,8 @@ def _run_prompt_submit(
"prompt dispatch: session store unavailable for %s — this turn may not persist",
session.get("session_key") or sid)
admitted = _admit_prompt_turn(
- sid, session, text, image_paths, queued_prompt_generation, display_kind, display_metadata)
+ sid, session, text, image_paths, queued_prompt_generation, display_kind, display_metadata,
+ report_ownership_refusal=report_ownership_refusal)
if admitted is None:
return False
images, agent = admitted
diff --git a/tui_gateway/session_notifications.py b/tui_gateway/session_notifications.py
index a7c89af9808..e0bd82d0c4c 100644
--- a/tui_gateway/session_notifications.py
+++ b/tui_gateway/session_notifications.py
@@ -177,6 +177,9 @@ def _notif_submit(rid: str, sid: str, session: dict, text: str, what: str, **kwa
accepted = _run_prompt_submit(rid, sid, session, text, **kwargs)
if not accepted:
_notif_release_turn(session)
+ # Admission can refuse even after the local running flag was claimed.
+ # Callers requeue immediately; queue.get(timeout=...) then never waits.
+ time.sleep(0.25)
return accepted
except Exception as exc:
_notif_log_failure(what, exc)
@@ -540,7 +543,7 @@ def _notif_dispatch_event(sid: str, session: dict, evt: dict, text: str) -> None
kwargs.setdefault("display_metadata", {})["notification_category"] = "diagnostic"
try:
if not _notif_submit(f"__notif__{int(time.time() * 1000)}", sid, session, text,
- "notification poller dispatch failed", **kwargs):
+ "notification poller dispatch failed", report_ownership_refusal=False, **kwargs):
_notif_defer_event(evt, claim)
return
except Exception:
@@ -646,6 +649,7 @@ def _notif_dispatch_completions(sid, session, notifications, registry, deferred)
if text is not None:
if not _notif_submit(f"__notif__{int(time.time() * 1000)}", sid, session, text,
"completion batch dispatch failed", display_kind=PROCESS_COMPLETE_DISPLAY_KIND,
+ report_ownership_refusal=False,
display_metadata={"display_text": batch.display_text(registry)}):
for event, _text, claim in claimed:
_notif_defer_event(event, claim,