Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(chat): retry terminal backend failures from notifications (#1953)

Call Whitee4924db41bcb
diff --git a/core/backend_failure_retry.py b/core/backend_failure_retry.py
new file mode 100644
index 000000000..66456ed7a
--- /dev/null
+++ b/core/backend_failure_retry.py
@@ -0,0 +1,214 @@
+"""User-requested continuation of one authoritative failed Turn.
+
+The notice links to an existing Delivery, not a second execution state machine.
+All writers call these helpers while holding the SQLite writer reservation.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from sqlalchemy import select, update
+from sqlalchemy.engine import Connection
+
+from storage import message_deliveries as deliveries, messages_service
+from storage.delivery_states import policy_for
+from storage.message_deliveries import (
+ FAILURE_RETRY_HISTORY_KIND,
+ failure_retry_binding as retry_binding,
+)
+from storage.models import agent_sessions, message_deliveries, messages, session_turns
+
+class RetryUnavailable(ValueError):
+ def __init__(self, code: str) -> None:
+ self.code = code
+ super().__init__(code)
+
+
+def _notice(conn: Connection, session_id: str, notice_id: str) -> dict[str, Any]:
+ notice = messages_service.get_message(conn, notice_id, session_id=session_id)
+ metadata = (notice or {}).get("metadata") or {}
+ if (
+ notice is None
+ or notice["type"] != "notify"
+ or notice["author"] != "agent"
+ or notice["source"] != "agent"
+ or metadata.get("event") != "backend_failure"
+ or not metadata.get("failure_id")
+ or not metadata.get("turn_id")
+ or metadata.get("detached")
+ ):
+ raise RetryUnavailable("retry_not_available")
+ return notice
+
+
+def _retained_inputs(conn: Connection, turn: dict[str, Any]) -> list[dict[str, Any]]:
+ """Recover the exact unwritten batch, without recreating its snapshots."""
+ retained = []
+ for row in conn.execute(
+ select(message_deliveries)
+ .where(message_deliveries.c.session_id == turn["session_id"])
+ .where(message_deliveries.c.state == "queued")
+ .order_by(message_deliveries.c.submitted_at, message_deliveries.c.id)
+ ).mappings():
+ delivery = dict(row)
+ events = json.loads(delivery["delivery_history_json"]).get("events", [])
+ attempts = [event for event in events if event.get("kind") == "start"]
+ last = attempts[-1] if attempts else {}
+ claimed = next(
+ (event for event in reversed(attempts) if event.get("outcome") in {"claimed", "opened"}),
+ {},
+ )
+ if (
+ claimed.get("turn_id") == turn["id"]
+ and last.get("outcome") == "not_written"
+ and not delivery.get("message_id")
+ and not delivery.get("current_attempt_id")
+ ):
+ retained.append(delivery)
+ if not any(row["id"] == turn["initial_delivery_id"] for row in retained):
+ raise RetryUnavailable("retry_original_unavailable")
+ return retained
+
+
+def _validate_boundary(
+ conn: Connection,
+ notice: dict[str, Any],
+ *,
+ exclude_ids: set[str],
+) -> dict[str, Any]:
+ session_id = notice["session_id"]
+ turn_id = notice["metadata"]["turn_id"]
+ session = conn.execute(select(agent_sessions).where(agent_sessions.c.id == session_id)).mappings().one_or_none()
+ turn = deliveries.get_turn(conn, turn_id)
+ if (
+ session is None
+ or session["status"] != "active"
+ or session["visibility"] == "system"
+ or turn is None
+ or turn["session_id"] != session_id
+ or turn["backend"] != session["agent_backend"]
+ ):
+ raise RetryUnavailable("retry_not_available")
+ latest = conn.execute(
+ select(session_turns.c.id)
+ .where(session_turns.c.session_id == session_id)
+ .order_by(session_turns.c.created_at.desc(), session_turns.c.id.desc())
+ .limit(1)
+ ).scalar_one_or_none()
+ if latest != turn_id:
+ raise RetryUnavailable("retry_stale")
+ if turn["state"] != "terminal":
+ raise RetryUnavailable("retry_busy")
+ if turn["terminal_outcome"] not in {"failed", "not_written"} or turn["settled_by"] in {
+ "stopped",
+ "agent_run_canceled",
+ "session_archive",
+ }:
+ raise RetryUnavailable("retry_not_available")
+ if turn["terminal_outcome"] == "failed" and turn["start_receipt_outcome"] != "accepted":
+ raise RetryUnavailable("retry_acceptance_unknown")
+ # A reserved/unknown input is just as important as an active Turn: it may
+ # already have an owner on its way to admission.
+ pending = conn.execute(
+ select(message_deliveries.c.id, message_deliveries.c.state)
+ .where(message_deliveries.c.session_id == session_id)
+ .where(message_deliveries.c.id.not_in(exclude_ids))
+ .where(message_deliveries.c.state.not_in(("accepted", "retired")))
+ )
+ if any(policy_for(row.state).ordering != "terminal" for row in pending):
+ raise RetryUnavailable("retry_pending_input")
+ return turn
+
+
+def reserve_retry(
+ conn: Connection,
+ *,
+ session_id: str,
+ notice_id: str,
+ snapshot: dict[str, Any],
+) -> dict[str, Any]:
+ """Resolve duplicate clicks or reserve one canonical, draft-independent input."""
+ notice = _notice(conn, session_id, notice_id)
+ link = notice["content"].get("failure_retry") or {}
+ existing = deliveries.get_delivery(conn, str(link.get("delivery_id") or ""))
+ if existing is not None and existing["session_id"] == session_id:
+ binding = retry_binding(existing)
+ if binding is None or binding.get("notice_id") != notice_id:
+ raise RetryUnavailable("retry_not_available")
+ if policy_for(existing["state"]).submission == "admitted":
+ return existing
+ if existing["state"] == "reserved":
+ # Wake the same owner even after a lost response. Admission retires
+ # it if the boundary changed; leaving a stale reservation here would
+ # strand an ordering fence in front of subsequent normal input.
+ return existing
+
+ source_turn = deliveries.get_turn(conn, str(notice["metadata"]["turn_id"]))
+ original = (
+ _retained_inputs(conn, source_turn)
+ if source_turn is not None
+ and source_turn["session_id"] == session_id
+ and source_turn["state"] == "terminal"
+ and source_turn["terminal_outcome"] == "not_written"
+ else []
+ )
+ turn = _validate_boundary(conn, notice, exclude_ids={row["id"] for row in original})
+ delivery_id = str(turn["initial_delivery_id"]) if original else deliveries.new_delivery_id()
+ binding = {
+ "kind": FAILURE_RETRY_HISTORY_KIND,
+ "notice_id": notice_id,
+ "source_turn_id": turn["id"],
+ "delivery_ids": [row["id"] for row in original] or [delivery_id],
+ }
+ if original:
+ for row in original:
+ saved = deliveries.cas_delivery(
+ conn,
+ row["id"],
+ expected_version=row["version"],
+ expected_states=("queued",),
+ values={},
+ history_event=binding,
+ )
+ if saved is None:
+ raise RuntimeError("retry retained input claim lost")
+ else:
+ deliveries.insert_delivery(
+ conn,
+ delivery_id=delivery_id,
+ session_id=session_id,
+ priority="p3",
+ state="reserved",
+ snapshot=snapshot,
+ dispatch_text="continue",
+ history_event=binding,
+ )
+ content = dict(notice["content"])
+ content["failure_retry"] = {"delivery_id": delivery_id}
+ conn.execute(
+ update(messages)
+ .where(messages.c.id == notice_id, messages.c.session_id == session_id)
+ .values(content_json=json.dumps(content, ensure_ascii=False))
+ )
+ saved = deliveries.get_delivery(conn, delivery_id)
+ assert saved is not None
+ return saved
+
+
+def admission_denial(conn: Connection, delivery: dict[str, Any]) -> str | None:
+ """Recheck before queue admission AND every queue-drain native start."""
+ binding = retry_binding(delivery, unclaimed_only=True)
+ if binding is None:
+ return None
+ try:
+ notice = _notice(conn, delivery["session_id"], binding["notice_id"])
+ if notice["metadata"]["turn_id"] != binding["source_turn_id"]:
+ raise RetryUnavailable("retry_stale")
+ if (notice["content"].get("failure_retry") or {}).get("delivery_id") not in binding["delivery_ids"]:
+ raise RetryUnavailable("retry_stale")
+ _validate_boundary(conn, notice, exclude_ids=set(binding["delivery_ids"]))
+ except RetryUnavailable as error:
+ return error.code
+ return None
diff --git a/core/internal_server.py b/core/internal_server.py
index e1379f3dd..d7987dc95 100644
--- a/core/internal_server.py
+++ b/core/internal_server.py
@@ -923,7 +923,17 @@ def create_app(
"delivery_id": delivery_id,
},
)
- if delivery["state"] != "reserved":
+ from core.backend_failure_retry import retry_binding
+
+ failure_retry = retry_binding(delivery)
+ if delivery["state"] != "reserved" and not (
+ retry_binding(delivery, unclaimed_only=True) and delivery["state"] == "queued"
+ ):
+ if failure_retry and delivery["state"] == "retired":
+ return JSONResponse(
+ status_code=409,
+ content={"ok": False, "code": "retry_stale", "delivery_id": delivery_id},
+ )
return JSONResponse(
status_code=202,
content={
@@ -982,6 +992,11 @@ def create_app(
)
with get_cached_sqlite_engine().connect() as conn:
settled = message_deliveries.get_delivery(conn, delivery_id)
+ if failure_retry and (settled or {}).get("state") == "retired":
+ return JSONResponse(
+ status_code=409,
+ content={"ok": False, "code": "retry_stale", "delivery_id": delivery_id},
+ )
return JSONResponse(
status_code=202,
content={
diff --git a/core/session_turns.py b/core/session_turns.py
index 868a1db8a..50d915ad6 100644
--- a/core/session_turns.py
+++ b/core/session_turns.py
@@ -1895,6 +1895,19 @@ class SessionTurnManager:
if owner.session_id != session_id:
raise RuntimeError("runtime start owner does not match Delivery claim target")
+ from core.backend_failure_retry import admission_denial
+
+ stale_retries = [
+ (delivery, reason)
+ for delivery in deliveries
+ if (reason := admission_denial(conn, delivery)) is not None
+ ]
+ if stale_retries:
+ for delivery, reason in stale_retries:
+ cls._retire_delivery_not_written(
+ conn, session_id, str(delivery["id"]), reason=reason
+ )
+ return None
binding = conn.execute(
select(
agent_sessions.c.agent_backend,
@@ -2779,6 +2792,7 @@ class SessionTurnManager:
turn_id: str | None = None
delivery_turn_id: str | None = None
start_context: MessageContext | None = None
+ retry_denial: str | None = None
delivery: dict[str, Any]
backend_draining = backend in self._draining_backends
with self._runtime_start_owner(request.session_id, backend) as start_owner, self._sqlite_engine().begin() as conn:
@@ -2815,12 +2829,26 @@ class SessionTurnManager:
str(delivery.get("turn_id") or delivery.get("current_target_turn_id") or "")
or None,
)
- if delivery["state"] == "reserved":
+ from core.backend_failure_retry import admission_denial
+
+ if reason := admission_denial(conn, delivery):
+ self._retire_delivery_not_written(
+ conn, request.session_id, str(delivery["id"]), reason=reason
+ )
+ retry_denial = reason
+ delivery = delivery_store.get_delivery(conn, str(delivery["id"])) or delivery
+ # Removing this reservation may unblock a newer P3 input.
+ # Keep driving the existing FIFO, never leave that input idle
+ # behind the fence that this rejection just removed.
+ if delivery["state"] == "reserved" or (
+ delivery_store.failure_retry_binding(delivery, unclaimed_only=True)
+ and delivery_store.failure_retry_state(delivery) == "reserved"
+ ):
queued = delivery_store.cas_delivery(
conn,
str(delivery["id"]),
expected_version=int(delivery["version"]),
- expected_states=("reserved",),
+ expected_states=(str(delivery["state"]),),
values={"state": "queued"},
history_event={"kind": "queue", "reason": "p3_admission"},
)
@@ -2859,7 +2887,7 @@ class SessionTurnManager:
delivery = (
delivery_store.get_delivery(conn, str(delivery["id"])) or delivery
)
- if delivery["state"] != "queued":
+ if delivery["state"] != "queued" and not retry_denial:
break
continue
for claimed in claimed_batch.get("deliveries", []):
@@ -2878,13 +2906,15 @@ class SessionTurnManager:
await self._start_persisted_turn(turn_id, context=start_context)
return self._committed_delivery_result(
str(delivery["id"]),
- attempted_turn_id=turn_id,
+ attempted_turn_id=turn_id if not retry_denial else None,
+ reason=retry_denial,
)
return DeliveryResult(
str(delivery["id"]),
str(delivery.get("message_id") or "") or None,
str(delivery["state"]),
delivery_turn_id,
+ reason=retry_denial,
)
async def _admit_p1(
diff --git a/docs/plans/backend-failure-retry.md b/docs/plans/backend-failure-retry.md
new file mode 100644
index 000000000..5eeb1ccd1
--- /dev/null
+++ b/docs/plans/backend-failure-retry.md
@@ -0,0 +1,89 @@
+# Retry a failed conversation turn
+
+## Contract
+
+The retry button is a user action on one authoritative backend-failure notice,
+not a new backend retry protocol or an automatic retry policy.
+
+- Only an attached `notify` with `event=backend_failure`, a `failure_id`, and a
+ durable `turn_id` can offer the action. Ordinary notifications, detached
+ completions, user stops, and authentication-recovery cards are unchanged.
+- The action targets the notice's exact Session and failed Turn. The server
+ rechecks access, archive/read-only status, current Turn, and pending input.
+ A newer task, another live owner, or an unresolved native start blocks it.
+- A normal failed Turn continues through the existing P3 Delivery admission
+ with the literal backend input `continue`. It does not roll back history,
+ replay already attempted tools, or pretend to resume an arbitrary native Turn.
+- A proven `not_written` Turn reuses its retained, unaccepted initial Delivery
+ and the normal queue/admission path, preserving its prompt and attachments.
+ Unknown native acceptance is not proof that replay is safe.
+- Duplicate clicks use the same durable Delivery identity. Frontend disabling
+ is feedback, not the concurrency guard. Admission rechecks the failed-Turn
+ boundary under the existing writer transaction before starting work.
+- Retry is independent of the composer draft. The original error remains
+ visible; the button reports that retry was requested. Accepted continuation
+ input remains in the ordinary transcript for truthful history.
+- There is no implicit model switch, auth reset, service restart, automatic
+ retry loop, native history truncation, or new storage table.
+
+## Interfaces and ownership
+
+`POST /api/sessions/{session_id}/messages` accepts a top-level `retry_for`
+containing the failed notice Message ID. It derives the prompt, content,
+metadata, and Delivery identity from server-owned state, never from a supplied
+replacement prompt. Other message submissions keep their existing contract.
+
+The shared failure-retry helper validates and reserves the action within the
+caller's existing SQLite writer transaction. The notice records its Delivery
+link; the existing Delivery/Turn state machine remains the execution owner.
+The controller revalidates the action before admission to cover the interval
+between Web reservation and native dispatch.
+
+The only persisted notice field is `content.failure_retry.delivery_id`.
+Read-side `failure_retry.state` comes from that Delivery, never a separately
+maintained acceptance flag. A reserved action can wake the same Delivery again
+after a lost response; an admitted action is idempotent. Definitively retired
+unaccepted continuations can be reserved again only while the failed-Turn
+boundary still holds. Server-owned `backend_failure_retry` Delivery history
+binds the notice, source Turn, and exact retained batch for admission and queue
+drain checks. Once a native start is claimed, subsequent recovery stays owned
+by the existing Turn state machine.
+
+A retained original may already be queued before the click. Its action projects
+`reserved` until the controller records this action's P3 admission or claims a
+native start, so an unreachable controller does not lock an unsubmitted retry.
+
+The `message.updated` Web event updates an existing row without replaying a
+terminal event or increasing unread/activity counts. Reload and historical
+message reads derive the same action state from durable storage.
+
+The Web notification renderer uses the existing design-system Button with
+localized labels (`重试` / `Retry`, `已请求重试` / `Retry requested`). It exposes
+no action on read-only Sessions and cannot invoke a retry twice while submitting.
+This first change targets the external Web conversation surface. All three
+backend adapters benefit from the shared admission path without separate retry
+implementations; IM-specific notification button rendering is not added here.
+
+## Validation
+
+- Shared contract tests: each backend, terminal versus ordinary/detached notices,
+ cross-Session targets, stale Turns, busy Sessions, queued input, and unknown
+ native acceptance.
+- Real storage/admission tests: same-notice concurrency and duplicate delivery,
+ pre-write prompt/attachment preservation, failed submission recovery, and
+ composer draft preservation.
+- Web route tests: access checks, canonical server-generated payload, source
+ notice update, and reloading persisted action state.
+- UI tests: eligible notification, ineligible/read-only rows, pending/accepted
+ feedback, failure recovery, and draft-independent action; production UI build.
+- Isolated local Incus verification only; never the running workstation service
+ or a remote tenant. No real upstream model request is needed for fault cases.
+
+Scenario contracts: `MESSAGE-DELIVERY-026` through `MESSAGE-DELIVERY-028`.
+
+## Known by design
+
+Native backend APIs do not provide one verified retry operation for all terminal
+failures. Claude's recovery of certain unfinished process-exit Turns is narrower
+than this action and is not used as a general shortcut. Historical notices
+without authoritative Turn linkage are not guessed into retry targets.
diff --git a/storage/message_deliveries.py b/storage/message_deliveries.py
index f02c815c6..790a7b3a6 100644
--- a/storage/message_deliveries.py
+++ b/storage/message_deliveries.py
@@ -28,6 +28,7 @@ from storage.models import (
TURN_OWNER_STATES = ("starting", "active")
+FAILURE_RETRY_HISTORY_KIND = "backend_failure_retry"
WEB_PUSH_USER_KEY_METADATA = "_web_push_user_key"
WEB_PUSH_USER_KEYS_METADATA = "_web_push_user_keys"
WEB_PUSH_AUTHORIZATION_CONTEXTS_METADATA = "_web_push_authorization_contexts"
@@ -330,6 +331,30 @@ def delivery_has_history_event(delivery: dict[str, Any], *, kind: str) -> bool:
)
+def failure_retry_binding(
+ delivery: dict[str, Any], *, unclaimed_only: bool = False
+) -> dict[str, Any] | None:
+ """Server-written action provenance, never caller Message metadata."""
+ for event in reversed(_history(delivery.get("delivery_history_json"))["events"]):
+ if unclaimed_only and event.get("kind") == "start" and event.get("outcome") in {"claimed", "opened"}:
+ return None
+ if event.get("kind") == FAILURE_RETRY_HISTORY_KIND:
+ return event
+ return None
+
+
+def failure_retry_state(delivery: dict[str, Any]) -> str:
+ """An original queued input predates its retry; await this action's admission."""
+ state = str(delivery["state"])
+ if state == "queued":
+ for event in reversed(_history(delivery.get("delivery_history_json"))["events"]):
+ if event.get("kind") in {"start", "queue"}:
+ break
+ if event.get("kind") == FAILURE_RETRY_HISTORY_KIND:
+ return "reserved"
+ return state
+
+
def active_turn(conn: Connection, session_id: str) -> dict[str, Any] | None:
return _one(
conn,
diff --git a/storage/messages_service.py b/storage/messages_service.py
index 523f32d07..323b8e729 100644
--- a/storage/messages_service.py
+++ b/storage/messages_service.py
@@ -95,11 +95,25 @@ def _row_to_payload(
row: dict[str, Any],
*,
include_private_metadata: bool = False,
+ conn: Connection | None = None,
) -> dict[str, Any]:
try:
content = json.loads(row.get("content_json") or "{}")
except json.JSONDecodeError:
content = {}
+ link = content.get("failure_retry") if isinstance(content, dict) else None
+ if conn is not None and isinstance(link, dict):
+ from storage.message_deliveries import failure_retry_state, get_delivery
+
+ delivery = get_delivery(conn, str(link.get("delivery_id") or ""))
+ content["failure_retry"] = {
+ "delivery_id": link.get("delivery_id"),
+ "state": (
+ failure_retry_state(delivery)
+ if delivery is not None and delivery["session_id"] == row.get("session_id")
+ else "retired"
+ ),
+ }
try:
metadata = json.loads(row.get("metadata_json") or "{}")
except json.JSONDecodeError:
@@ -609,7 +623,7 @@ def get_message(
if session_id is not None:
query = query.where(messages.c.session_id == session_id)
row = conn.execute(query).mappings().first()
- return _row_to_payload(dict(row)) if row else None
+ return _row_to_payload(dict(row), conn=conn) if row else None
def native_message_exists(
@@ -917,7 +931,7 @@ def list_session_messages(
)
older = [
_row_to_payload(
- dict(row), include_private_metadata=include_private_metadata
+ dict(row), include_private_metadata=include_private_metadata, conn=conn
)
for row in conn.execute(older_q).mappings().all()
]
@@ -927,7 +941,7 @@ def list_session_messages(
anchor_rows = [
_row_to_payload(
- dict(row), include_private_metadata=include_private_metadata
+ dict(row), include_private_metadata=include_private_metadata, conn=conn
)
for row in conn.execute(query.where(messages.c.id == anchor_id)).mappings().all()
]
@@ -944,7 +958,7 @@ def list_session_messages(
)
newer = [
_row_to_payload(
- dict(row), include_private_metadata=include_private_metadata
+ dict(row), include_private_metadata=include_private_metadata, conn=conn
)
for row in conn.execute(newer_q).mappings().all()
]
@@ -972,7 +986,7 @@ def list_session_messages(
conn,
[
_row_to_payload(
- dict(row), include_private_metadata=include_private_metadata
+ dict(row), include_private_metadata=include_private_metadata, conn=conn
)
for row in conn.execute(query).mappings().all()
],
@@ -1005,7 +1019,7 @@ def list_session_messages(
conn,
[
_row_to_payload(
- dict(row), include_private_metadata=include_private_metadata
+ dict(row), include_private_metadata=include_private_metadata, conn=conn
)
for row in conn.execute(query).mappings().all()
],
@@ -1039,7 +1053,7 @@ def list_session_messages(
conn,
[
_row_to_payload(
- dict(row), include_private_metadata=include_private_metadata
+ dict(row), include_private_metadata=include_private_metadata, conn=conn
)
for row in conn.execute(query).mappings().all()
],
diff --git a/tests/backend_failure_retry_helpers.py b/tests/backend_failure_retry_helpers.py
new file mode 100644
index 000000000..9326e9284
--- /dev/null
+++ b/tests/backend_failure_retry_helpers.py
@@ -0,0 +1,110 @@
+"""Real durable failed-Turn fixtures shared by retry boundary tests."""
+
+from storage import message_deliveries as deliveries, messages_service
+
+
+def seed_failed_notice(
+ conn,
+ *,
+ session_id,
+ scope_id=None,
+ backend="codex",
+ not_written=False,
+ content=None,
+ texts=None,
+):
+ inputs = [
+ deliveries.insert_delivery(
+ conn,
+ delivery_id=deliveries.new_delivery_id(),
+ session_id=session_id,
+ priority="p3",
+ state="queued",
+ snapshot=deliveries.message_snapshot(
+ scope_id=scope_id,
+ session_id=session_id,
+ platform="avibe",
+ author="user",
+ source="user",
+ text=text,
+ content=content,
+ message_kind="original",
+ ),
+ dispatch_text=text,
+ )
+ for text in (texts or ["检查这张图"])
+ ]
+ turn_id = deliveries.new_turn_id()
+ deliveries.claim_start_batch(
+ conn,
+ turn_id=turn_id,
+ session_id=session_id,
+ backend=backend,
+ deliveries=inputs,
+ dispatch_text="\n".join(row["dispatch_text"] for row in inputs),
+ )
+ turn = deliveries.get_turn(conn, turn_id)
+ if not not_written:
+ deliveries.bind_native_start(
+ conn,
+ turn_id,
+ expected_version=turn["version"],
+ runtime_key="test-runtime",
+ runtime_turn_id="test-native-turn",
+ native_turn_id="test-native-turn",
+ )
+ deliveries.materialize_start_acceptance(conn, turn_id=turn_id, evidence={"kind": "test_acceptance"})
+ deliveries.terminalize_turn(
+ conn,
+ turn_id,
+ outcome="not_written" if not_written else "failed",
+ settled_by="terminal_result",
+ evidence_kind="definitive_prewrite_failure" if not_written else "terminal_result",
+ )
+ if not_written:
+ for row in inputs:
+ current = deliveries.get_delivery(conn, row["id"])
+ deliveries.record_definitive_attempt(
+ conn,
+ row["id"],
+ expected_version=current["version"],
+ expected_states=("claimed",),
+ outcome="not_written",
+ next_state="queued",
+ )
+ notice = messages_service.append(
+ conn,
+ session_id=session_id,
+ scope_id=scope_id,
+ platform="avibe",
+ author="agent",
+ source="agent",
+ message_type="notify",
+ text="Backend interrupted",
+ metadata={
+ "event": "backend_failure",
+ "backend": backend,
+ "turn_id": turn_id,
+ "failure_id": f"turn:{turn_id}",
+ },
+ )
+ return notice, deliveries.get_turn(conn, turn_id), inputs
+
+
+def reserve_failure_retry(conn, notice):
+ from core.backend_failure_retry import reserve_retry
+
+ return reserve_retry(
+ conn,
+ session_id=notice["session_id"],
+ notice_id=notice["id"],
+ snapshot=deliveries.message_snapshot(
+ scope_id=notice["scope_id"],
+ session_id=notice["session_id"],
+ platform="avibe",
+ author="user",
+ source="user",
+ text="continue",
+ message_kind="quick_reply",
+ ),
+ )
diff --git a/tests/scenarios/INDEX.yaml b/tests/scenarios/INDEX.yaml
index 01a2fed33..5463d5b02 100644
--- a/tests/scenarios/INDEX.yaml
+++ b/tests/scenarios/INDEX.yaml
@@ -226,6 +226,9 @@ capabilities:
- tests/test_agent_service.py
- tests/test_ui_session_stream.py
- ui/src/components/workbench/Composer.attachmentRetry.test.tsx
+ - tests/test_backend_failure_retry.py
+ - ui/src/components/workbench/FailureRetry.test.tsx
+ - ui/src/components/workbench/ChatRowOptimisticWrites.test.tsx
- id: tunnel_quality
name: remote-access tunnel quality and smooth route recovery
status: active
diff --git a/tests/scenarios/message_delivery/catalog.yaml b/tests/scenarios/message_delivery/catalog.yaml
index 506841921..52c269303 100644
--- a/tests/scenarios/message_delivery/catalog.yaml
+++ b/tests/scenarios/message_delivery/catalog.yaml
@@ -164,6 +164,24 @@ scenarios:
kind: regression
layer: contract
test: ui/src/components/workbench/Composer.attachmentRetry.test.tsx::MESSAGE-DELIVERY-025
+ - id: MESSAGE-DELIVERY-026
+ name: Failed-turn retry crosses Web and controller admission without replaying accepted work
+ status: covered
+ kind: recovery
+ layer: contract
+ test: tests/test_backend_failure_retry.py::test_failure_retry_web_controller_round_trip
+ - id: MESSAGE-DELIVERY-027
+ name: Concurrent failed-notice clicks share one durable continuation owner
+ status: covered
+ kind: concurrency
+ layer: contract
+ test: tests/test_backend_failure_retry.py::test_failure_retry_concurrent_clicks_have_one_owner
+ - id: MESSAGE-DELIVERY-028
+ name: Deferred failed-turn retry cannot execute after newer work
+ status: covered
+ kind: concurrency
+ layer: contract
+ test: tests/test_backend_failure_retry.py::test_failure_retry_queue_drain_rechecks_the_boundary
- id: MESSAGE-DELIVERY-101
name: Unresolved fallback-capable delivery fences only later FIFO submissions
status: covered
diff --git a/tests/scenarios/message_delivery/observations.yaml b/tests/scenarios/message_delivery/observations.yaml
index 6600e2f13..265d0adc9 100644
--- a/tests/scenarios/message_delivery/observations.yaml
+++ b/tests/scenarios/message_delivery/observations.yaml
@@ -1,6 +1,20 @@
version: 1
capability: message_delivery
observations:
+ - id: OBS-MESSAGE-DELIVERY-006
+ source: user_feedback
+ related_scenarios:
+ - MESSAGE-DELIVERY-026
+ - MESSAGE-DELIVERY-027
+ - MESSAGE-DELIVERY-028
+ summary: A terminal backend-failure notice left Web users typing continue manually.
+ previous_assumption: Backend-native retry was needed to offer a retry action.
+ observed_behavior: The three backends share durable ordinary-input admission but no verified universal native retry protocol.
+ required_updates:
+ - bind a Retry action to the exact failure notice and Turn
+ - continue accepted work through one canonical continue Delivery
+ - reuse retained unwritten inputs and their attachments only with definitive negative delivery evidence
+ - recheck ownership at reservation and queue drain and preserve the composer draft
- id: OBS-MESSAGE-DELIVERY-001
source: live_local_diagnosis
related_scenarios:
diff --git a/tests/test_backend_failure_retry.py b/tests/test_backend_failure_retry.py
new file mode 100644
index 000000000..6c27afb1c
--- /dev/null
+++ b/tests/test_backend_failure_retry.py
@@ -0,0 +1,409 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from concurrent.futures import ThreadPoolExecutor
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from sqlalchemy import update
+
+from core.session_turns import DeliveryRequest
+from storage import message_deliveries, messages_service
+from storage.agent_session_rows import reserve_write_lock
+from storage.db import create_sqlite_engine
+from storage.models import messages, session_turns
+from tests.backend_failure_retry_helpers import reserve_failure_retry, seed_failed_notice
+from tests.test_session_delivery_fsm import _context, _fsm_schema_template, managers # noqa: F401
+from tests.test_ui_session_stream import _accepted_dispatch, _make_session, isolated_state # noqa: F401
+from tests.ui_server_test_helpers import csrf_headers
+
+
+def _retry_notice(tmp_path, *, backend="claude", not_written=False, content=None):
+ scope_id, session_id = _make_session(tmp_path, agent_backend=backend)
+ with create_sqlite_engine().begin() as conn:
+ notice, turn, inputs = seed_failed_notice(
+ conn,
+ session_id=session_id,
+ scope_id=scope_id,
+ backend=backend,
+ not_written=not_written,
+ content=content,
+ )
+ return session_id, notice, turn, inputs
+
+
[email protected]("backend", ["claude", "codex", "opencode"])
+def test_failure_retry_canonical_continue_and_draft(isolated_state, tmp_path, backend):
+ from vibe.ui_server import app
+
+ session_id, notice, _turn, _inputs = _retry_notice(tmp_path, backend=backend)
+ client = app.test_client()
+ headers = csrf_headers(client)
+ draft = client.put(
+ f"/api/sessions/{session_id}/draft",
+ headers=headers,
+ json={"text": "不要丢掉这段草稿", "expected_updated_at": None},
+ ).get_json()["draft"]
+ dispatch = _accepted_dispatch(session_id)
+ with patch("vibe.internal_client.dispatch_async", dispatch):
+ response = client.post(
+ f"/api/sessions/{session_id}/messages",
+ headers=headers,
+ json={
+ "retry_for": notice["id"],
+ "text": "replay everything",
+ "content": {"attachments": [{"token": "forged"}]},
+ "metadata": {"quick_reply_for": "forged", "resource_user_context": {}},
+ },
+ )
+ assert response.status_code == 201
+ body = response.get_json()
+ assert body["text"] == "continue"
+ assert body["draft_advanced"] is False
+ assert body["draft"] == draft
+ assert body["retry_notice"]["content"]["failure_retry"]["state"] == "accepted"
+ assert dispatch.await_args.args[0]["files"] == []
+ with create_sqlite_engine().connect() as conn:
+ reloaded = messages_service.list_session_messages(conn, session_id=session_id)
+ persisted = next(row for row in reloaded["messages"] if row["id"] == notice["id"])
+ assert persisted["content"]["failure_retry"]["state"] == "accepted"
+
+
[email protected]("exception_name", ["InternalServerTimeout", "InternalServerUnavailable"])
+def test_failure_retry_dispatch_loss(isolated_state, tmp_path, exception_name):
+ from vibe import internal_client
+ from vibe.ui_server import app
+
+ session_id, notice, _turn, _inputs = _retry_notice(tmp_path)
+ client = app.test_client()
+ headers = csrf_headers(client)
+ url = f"/api/sessions/{session_id}/messages"
+ with patch("vibe.internal_client.dispatch_async", side_effect=getattr(internal_client, exception_name)("offline")):
+ failed = client.post(url, headers=headers, json={"retry_for": notice["id"]})
+ assert failed.status_code in {502, 504}
+ first_id = failed.get_json()["id"]
+ with patch("vibe.internal_client.dispatch_async", _accepted_dispatch(session_id)):
+ recovered = client.post(url, headers=headers, json={"retry_for": notice["id"]})
+ assert recovered.status_code == 201
+ assert (recovered.get_json()["id"] == first_id) is (exception_name == "InternalServerTimeout")
+ with create_sqlite_engine().connect() as conn:
+ visible = messages_service.list_session_messages(conn, session_id=session_id)["messages"]
+ assert sum(row["text"] == "continue" for row in visible) == 1
+
+
[email protected]("change", ["ordinary", "detached", "unlinked", "wrong_session", "stopped", "unknown", "newer"])
+def test_failure_retry_invalid_or_stale_notice(isolated_state, tmp_path, change):
+ from vibe.ui_server import app
+
+ session_id, notice, turn, _inputs = _retry_notice(tmp_path)
+ with create_sqlite_engine().begin() as conn:
+ meta = dict(notice["metadata"])
+ if change == "ordinary":
+ meta["event"] = "progress"
+ elif change == "detached":
+ meta["detached"] = True
+ elif change == "unlinked":
+ meta.pop("turn_id")
+ elif change == "wrong_session":
+ meta["turn_id"] = "trn_missing"
+ elif change == "stopped":
+ conn.execute(
+ update(session_turns).where(session_turns.c.id == turn["id"]).values(terminal_outcome="canceled")
+ )
+ elif change == "unknown":
+ conn.execute(
+ update(session_turns).where(session_turns.c.id == turn["id"]).values(start_receipt_outcome="unknown")
+ )
+ elif change == "newer":
+ seed_failed_notice(conn, session_id=session_id, scope_id=notice["scope_id"], backend="claude")
+ conn.execute(update(messages).where(messages.c.id == notice["id"]).values(metadata_json=json.dumps(meta)))
+ client = app.test_client()
+ with patch("vibe.internal_client.dispatch_async", AsyncMock()) as dispatch:
+ response = client.post(
+ f"/api/sessions/{session_id}/messages",
+ headers=csrf_headers(client),
+ json={"retry_for": notice["id"]},
+ )
+ assert response.status_code == 409
+ dispatch.assert_not_awaited()
+
+
+def test_failure_retry_retains_prompt_and_attachment(isolated_state, tmp_path):
+ from vibe.ui_server import app
+
+ session_id, notice, _turn, inputs = _retry_notice(
+ tmp_path,
+ not_written=True,
+ content={"attachments": [{"token": "server-owned-token", "name": "中文.png"}]},
+ )
+
+ async def dispatch(payload):
+ assert payload["message_id"] == inputs[0]["id"]
+ assert payload["text"] == "检查这张图"
+ assert payload["files"] == [{"path": "/isolated/中文.png"}]
+ return {"status_code": 202, "body": {"delivery_state": "queued", "queued": True}}
+
+ client = app.test_client()
+ with (
+ patch("vibe.internal_client.dispatch_async", side_effect=dispatch),
+ patch(
+ "core.workbench_media.resolve_attachment_specs", return_value=[{"path": "/isolated/中文.png"}]
+ ) as resolve,
+ ):
+ response = client.post(
+ f"/api/sessions/{session_id}/messages",
+ headers=csrf_headers(client),
+ json={"retry_for": notice["id"]},
+ )
+ assert response.status_code == 202
+ assert resolve.call_args.kwargs["attachments"][0]["token"] == "server-owned-token"
+ with create_sqlite_engine().connect() as conn:
+ retained = message_deliveries.get_delivery(conn, inputs[0]["id"])
+ assert retained["snapshot_json"] == inputs[0]["snapshot_json"]
+
+
+def test_failure_retry_concurrent_clicks_have_one_owner(managers):
+ """MESSAGE-DELIVERY-027: duplicate notice clicks share one native start."""
+ manager_a, manager_b, engine_a, engine_b, starts = managers
+ with engine_a.begin() as conn:
+ notice, _turn, _inputs = seed_failed_notice(conn, session_id="ses_fsm")
+
+ def reserve(engine):
+ with engine.begin() as conn:
+ reserve_write_lock(conn)
+ return reserve_failure_retry(conn, notice)
+
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ results = list(executor.map(reserve, [engine_a, engine_b]))
+ assert results[0]["id"] == results[1]["id"]
+
+ async def admit_both():
+ await asyncio.gather(
+ *[
+ manager.deliver(
+ DeliveryRequest(
+ session_id="ses_fsm",
+ priority="p3",
+ content="forged",
+ delivery_id=results[0]["id"],
+ ),
+ context=_context(),
+ )
+ for manager in [manager_a, manager_b]
+ ]
+ )
+
+ asyncio.run(admit_both())
+ assert len(starts) == 1
+ assert starts[0][1] == "continue"
+
+
[email protected]("new_work", ["reserved", "active", "completed"])
+def test_failure_retry_admission_rejects_newer_work(managers, new_work):
+ manager, _other, engine, _engine_b, starts = managers
+ with engine.begin() as conn:
+ notice, _turn, _inputs = seed_failed_notice(conn, session_id="ses_fsm")
+ retry = reserve_failure_retry(conn, notice)
+ if new_work == "reserved":
+ message_deliveries.insert_delivery(
+ conn,
+ delivery_id=message_deliveries.new_delivery_id(),
+ session_id="ses_fsm",
+ priority="p3",
+ state="reserved",
+ snapshot=message_deliveries.message_snapshot(
+ scope_id=None, session_id="ses_fsm", platform="avibe", author="user", source="user", text="new task"
+ ),
+ dispatch_text="new task",
+ )
+ else:
+ _notice, newer, _inputs = seed_failed_notice(conn, session_id="ses_fsm")
+ if new_work == "active":
+ conn.execute(
+ update(session_turns)
+ .where(session_turns.c.id == newer["id"])
+ .values(
+ state="active",
+ terminal_outcome=None,
+ terminal_at=None,
+ )
+ )
+ result = asyncio.run(
+ manager.deliver(
+ DeliveryRequest(session_id="ses_fsm", priority="p3", content="continue", delivery_id=retry["id"]),
+ context=_context(),
+ )
+ )
+ assert result.state == "retired"
+ assert result.reason in {"retry_stale", "retry_pending_input"}
+ assert starts == []
+
+
[email protected]("texts", [["原输入"], ["原输入一", "原输入二"]])
+def test_failure_retry_unwritten_original_batch(managers, texts):
+ manager, _other, engine, _engine_b, starts = managers
+ with engine.begin() as conn:
+ notice, _turn, originals = seed_failed_notice(
+ conn,
+ session_id="ses_fsm",
+ not_written=True,
+ texts=texts,
+ )
+ retry = reserve_failure_retry(conn, notice)
+ assert retry["id"] == originals[0]["id"]
+ result = asyncio.run(
+ manager.deliver(
+ DeliveryRequest(session_id="ses_fsm", priority="p3", content="continue", delivery_id=retry["id"]),
+ context=_context(),
+ )
+ )
+ assert result.state == "claimed"
+ assert len(starts) == 1
+ assert all(text in starts[0][1] for text in texts)
+ assert "continue" not in starts[0][1]
+
+
[email protected]("not_written", [False, True])
+def test_failure_retry_web_controller_round_trip(isolated_state, tmp_path, monkeypatch, not_written):
+ """MESSAGE-DELIVERY-026: Web -> real admission -> native boundary -> replay."""
+ import httpx
+ from core import internal_server
+ from tests.test_internal_server import _bind_test_native_start, _build_controller_double
+ from vibe import ui_server
+
+ session_id, notice, _turn, _inputs = _retry_notice(tmp_path, not_written=not_written)
+ engine = create_sqlite_engine()
+ controller = _build_controller_double()
+ internal_app = internal_server.create_app(controller)
+ manager = controller.session_turns
+ starts = []
+
+ async def capture_run(sid, context, text, *, logical_turn_id=None, **kwargs):
+ starts.append((sid, text))
+ _bind_test_native_start(engine, context)
+ manager._terminalize_durable_turn(
+ logical_turn_id,
+ "completed",
+ settled_by="terminal_result",
+ evidence_kind="test_terminal",
+ )
+
+ manager._run = capture_run
+ transport = httpx.ASGITransport(app=internal_app)
+
+ async def dispatch(payload):
+ async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as internal:
+ response = await internal.post("/internal/dispatch_async", json=payload)
+ return {"status_code": response.status_code, "body": response.json()}
+
+ monkeypatch.setattr("vibe.internal_client.dispatch_async", dispatch)
+ client = ui_server.app.test_client()
+ headers = csrf_headers(client)
+ url = f"/api/sessions/{session_id}/messages"
+ first = client.post(url, json={"retry_for": notice["id"]}, headers=headers)
+ duplicate = client.post(url, json={"retry_for": notice["id"]}, headers=headers)
+ assert first.status_code == duplicate.status_code == 201
+ assert first.get_json()["id"] == duplicate.get_json()["id"]
+ assert starts == [(session_id, "检查这张图" if not_written else "continue")]
+
+
+def test_failure_retry_queue_drain_rechecks_the_boundary(managers):
+ """MESSAGE-DELIVERY-028: a deferred retry cannot overtake newer work."""
+ manager, _other, engine, _engine_b, starts = managers
+ with engine.begin() as conn:
+ notice, _turn, _inputs = seed_failed_notice(conn, session_id="ses_fsm")
+ retry = reserve_failure_retry(conn, notice)
+
+ async def queue_then_drain():
+ manager.begin_backend_drain("codex")
+ queued = await manager.deliver(
+ DeliveryRequest(session_id="ses_fsm", priority="p3", content="continue", delivery_id=retry["id"]),
+ context=_context(),
+ )
+ assert queued.state == "queued"
+ with engine.begin() as conn:
+ seed_failed_notice(conn, session_id="ses_fsm")
+ await manager.end_backend_drain("codex")
+
+ asyncio.run(queue_then_drain())
+ with engine.connect() as conn:
+ assert message_deliveries.get_delivery(conn, retry["id"])["state"] == "retired"
+ assert starts == []
+
+
[email protected]("restriction", ["archived", "access", "cross_session"])
+def test_failure_retry_keeps_session_access_gates(isolated_state, tmp_path, restriction):
+ from core.services import sessions
+ from storage.models import agent_sessions
+ from vibe.ui_server import app
+
+ session_id, notice, _turn, _inputs = _retry_notice(tmp_path)
+ with create_sqlite_engine().begin() as conn:
+ if restriction == "archived":
+ conn.execute(update(agent_sessions).where(agent_sessions.c.id == session_id).values(status="archived"))
+ elif restriction == "cross_session":
+ original = sessions.get_session(conn, session_id)
+ session_id = sessions.create_session(
+ conn,
+ scope_id=notice["scope_id"],
+ agent_backend="claude",
+ agent_id=original["agent_id"],
+ agent_name=original["agent_name"],
+ )["id"]
+ client = app.test_client()
+ with (
+ patch("vibe.internal_client.dispatch_async", AsyncMock()) as dispatch,
+ patch(
+ "core.vibe_agents.ensure_session_agent_access",
+ side_effect=PermissionError("denied") if restriction == "access" else None,
+ ),
+ ):
+ response = client.post(
+ f"/api/sessions/{session_id}/messages",
+ headers=csrf_headers(client),
+ json={"retry_for": notice["id"]},
+ )
+ assert response.status_code == (403 if restriction == "access" else 409)
+ dispatch.assert_not_awaited()
+
+
+def test_unwritten_retry_remains_clickable_when_controller_cannot_be_reached(isolated_state, tmp_path):
+ from vibe import internal_client, ui_server
+
+ session_id, notice, _turn, _inputs = _retry_notice(tmp_path, not_written=True)
+ client = ui_server.app.test_client()
+ with patch("vibe.internal_client.dispatch_async", side_effect=internal_client.InternalServerUnavailable("offline")):
+ response = client.post(
+ f"/api/sessions/{session_id}/messages", headers=csrf_headers(client),
+ json={"retry_for": notice["id"]},
+ )
+ assert response.status_code == 502
+ assert response.get_json()["retry_notice"]["content"]["failure_retry"]["state"] == "reserved"
+ with create_sqlite_engine().connect() as conn:
+ persisted = messages_service.get_message(conn, notice["id"])
+ assert persisted["content"]["failure_retry"]["state"] == "reserved"
+
+
+def test_stale_retry_releases_a_newer_input_waiting_behind_its_reservation(managers):
+ manager, _other, engine, _engine_b, starts = managers
+ with engine.begin() as conn:
+ notice, _turn, _inputs = seed_failed_notice(conn, session_id="ses_fsm")
+ retry = reserve_failure_retry(conn, notice)
+
+ async def admit():
+ newer = await manager.deliver(
+ DeliveryRequest(session_id="ses_fsm", priority="p3", content="new task"),
+ context=_context(),
+ )
+ assert newer.state == "queued" # the reserved retry is an ordering fence
+ refused = await manager.deliver(
+ DeliveryRequest(session_id="ses_fsm", priority="p3", content="continue", delivery_id=retry["id"]),
+ context=_context(),
+ )
+ assert refused.state == "retired"
+ assert refused.turn_id is None
+
+ asyncio.run(admit())
+ assert len(starts) == 1
+ assert starts[0][1] == "new task"
diff --git a/tests/test_instance_authorization.py b/tests/test_instance_authorization.py
index 7237fea83..073445597 100644
--- a/tests/test_instance_authorization.py
+++ b/tests/test_instance_authorization.py
@@ -542,6 +542,8 @@ def test_workbench_events_follow_role_boundaries() -> None:
member = _context("member", remote=True)
owner = _context("owner", remote=True)
assert can_receive_workbench_event(viewer, "message.new")
+ for context in (viewer, editor, member, owner):
+ assert can_receive_workbench_event(context, "message.updated")
assert not can_receive_workbench_event(viewer, "queue.updated")
assert can_receive_workbench_event(editor, "queue.updated")
assert not can_receive_workbench_event(editor, "runs.updated")
diff --git a/tests/test_ui_project_access.py b/tests/test_ui_project_access.py
index adb13c10e..0c5dc1a83 100644
--- a/tests/test_ui_project_access.py
+++ b/tests/test_ui_project_access.py
@@ -3,6 +3,8 @@ from __future__ import annotations
import json
from types import SimpleNamespace
+import pytest
+
from config.v2_config import (
AgentsConfig,
PlatformsConfig,
@@ -1273,6 +1275,54 @@ def test_show_page_payload_redacts_path_when_session_is_missing(monkeypatch) ->
assert "path" not in payload
[email protected]("role", ["viewer", "editor", "member"])
+def test_retry_notice_update_stream_keeps_remote_session_acl(monkeypatch, tmp_path, role) -> None:
+ import asyncio
+
+ from vibe.sse_broker import broker
+ from vibe.ui_compat import g
+
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ _config, ids = _setup_state(tmp_path)
+ context = AuthorizationContext(
+ instance_role=role,
+ subject="user-editor",
+ instance_access_source="organization_group",
+ organization_id="org-1",
+ organization_member_id="member-1",
+ organization_role="member",
+ is_remote=True,
+ )
+ visible = {
+ "id": "notice",
+ "session_id": ids["session_a"],
+ "content": {"failure_retry": {"delivery_id": "retry-delivery", "state": "queued"}},
+ }
+
+ async def collect():
+ with ui_server.app.test_request_context("/api/events"):
+ g.authorization_context = context
+ response = await ui_server.workbench_events()
+ iterator = response.body_iterator.__aiter__()
+ try:
+ for _ in range(3):
+ await iterator.__anext__()
+ broker.publish("message.updated", {**visible, "session_id": ids["session_b"]})
+ broker.publish("message.updated", {**visible, "session_id": "missing-session"})
+ broker.publish("message.updated", visible)
+ return await asyncio.wait_for(iterator.__anext__(), timeout=1)
+ finally:
+ await iterator.aclose()
+
+ frame = asyncio.run(collect())
+ if isinstance(frame, bytes):
+ frame = frame.decode("utf-8")
+ assert "event: message.updated\n" in frame
+ data = next(line.removeprefix("data: ") for line in frame.splitlines() if line.startswith("data: "))
+ assert json.loads(data)["data"] == visible
+
+
def test_project_access_filters_sse_and_show_websocket(monkeypatch, tmp_path) -> None:
monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
config, ids = _setup_state(tmp_path)
diff --git a/ui/src/components/workbench/ChatPage.tsx b/ui/src/components/workbench/ChatPage.tsx
index 2ecebd589..7d031cf75 100644
--- a/ui/src/components/workbench/ChatPage.tsx
+++ b/ui/src/components/workbench/ChatPage.tsx
@@ -18,7 +18,7 @@ import { readChatViewMode, writeChatViewMode } from '../../lib/chatViewMemory';
import { normalizeChatMessageFontSize } from '../../lib/chatDisplay';
import { setConfigField } from '../../lib/configMutations';
import { annotationStandIn, annotationTitleKey, readAnnotationView } from '../../lib/annotationView';
-import { isAgentActivityBoundaryMessage, isTerminalAgentMessage, isTranscriptMessage, shouldRefreshAgentActivityForMessage } from '../../lib/chatMessageTypes';
+import { isAgentActivityBoundaryMessage, isRetryableFailureNotice, isTerminalAgentMessage, isTranscriptMessage, shouldRefreshAgentActivityForMessage } from '../../lib/chatMessageTypes';
import { chatRowKind, drawsEmptyBodyPlaceholder, isAgentAuthored } from '../../lib/chatRowKind';
import { useIosKeyboardInset } from '../../lib/useIosKeyboardInset';
import { isProxyMediaUrl } from '../../lib/mediaProxy';
@@ -59,6 +59,7 @@ import {
vaultCallbackStatusKey,
} from '../../lib/chatTrigger';
import { AnnotationMessage } from './AnnotationMessage';
+import { FailureRetry } from './FailureRetry';
import { AGENT_BUBBLE, SYSTEM_BUBBLE, USER_BUBBLE } from './chatBubble';
import { RoleAvatar } from './RoleAvatar';
import { useFileDrop } from '../../lib/useFileDrop';
@@ -1617,6 +1618,12 @@ export const ChatPage: React.FC = () => {
void refreshSessionRow();
};
const disconnect = api.connectWorkbenchEvents({
+ onMessageUpdated: (msg) => {
+ if (msg.session_id !== sessionIdRef.current) return;
+ // An action-state update is not another terminal event, unread message,
+ // or reason to leave the reader's historical window.
+ setMessages((current) => current.map((row) => row.id === msg.id ? msg : row));
+ },
// NB: match against sessionIdRef.current (the CURRENT route), NOT the
// captured ``sessionId`` — there is a window after a chat switch before
// React runs this subscription's cleanup, during which an event for the
@@ -2144,6 +2151,47 @@ export const ChatPage: React.FC = () => {
[sendMessage],
);
+ const handleFailureRetry = useCallback(async (messageId: string): Promise<boolean> => {
+ const sid = sessionId;
+ if (!sid || !writable) return false;
+ setError(null);
+ try {
+ // A notice-bound action does not read, clear, or restore the composer.
+ const response = await apiFetch(`/api/sessions/${encodeURIComponent(sid)}/messages`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ retry_for: messageId }),
+ });
+ const body = await response.json().catch(() => null);
+ if (sid !== sessionIdRef.current) return false;
+ if (body?.retry_notice) {
+ const notice = body.retry_notice as WorkbenchMessage;
+ setMessages((current) => current.map((row) => row.id === notice.id ? notice : row));
+ }
+ if (isSessionArchivedConflict(response.status, body)) {
+ api.convergeSessionArchived(sid);
+ }
+ if (!response.ok) {
+ const parsed = body ? selectApiErrorFields(body, t('chat.retryFailed')) : null;
+ setError(parsed?.code
+ ? t(`errors.${parsed.code}`, { defaultValue: parsed.fallback })
+ : parsed?.fallback ?? t('chat.retryFailed'));
+ return false;
+ }
+ if (body?.author && body?.id && body?.state !== 'reserved' && !body?.queued) {
+ if (historicalWindowRef.current) await reloadLatestMessages();
+ else appendMessage(body as WorkbenchMessage);
+ }
+ void refreshQueue();
+ return ['queued', 'claimed', 'accepted'].includes(body?.delivery_state ?? body?.state ?? '');
+ } catch (err) {
+ if (sid === sessionIdRef.current) setError(errorMessage(err) ?? t('chat.retryFailed'));
+ return false;
+ } finally {
+ if (sid === sessionIdRef.current) void syncTurnStateRef.current?.();
+ }
+ }, [sessionId, writable, api, appendMessage, reloadLatestMessages, refreshQueue, t]);
+
const stopMessage = useCallback(async () => {
if (!sessionId || !working) return;
try {
@@ -2805,6 +2853,7 @@ export const ChatPage: React.FC = () => {
highlightedId={highlightedId}
messageFontSize={messageFontSize}
onQuickReply={handleQuickReply}
+ onFailureRetry={handleFailureRetry}
provisionRequestsByMessage={provisionPlacement.byMessageId}
onVaultRequestResolved={refreshVaultRequests}
onQuoteSelection={quoteSelectionToComposer}
@@ -3645,6 +3694,7 @@ interface TranscriptProps {
highlightedId: string | null;
messageFontSize: number;
onQuickReply: (messageId: string, choice: string) => boolean | void | Promise<boolean | void>;
+ onFailureRetry?: (messageId: string) => Promise<boolean>;
provisionRequestsByMessage: Map<string, VaultRequest[]>;
onVaultRequestResolved: () => void;
// Chat-selection toolbar: quote the selection into the composer, or fork +
@@ -3704,6 +3754,7 @@ export const Transcript: React.FC<TranscriptProps> = ({
highlightedId,
messageFontSize,
onQuickReply,
+ onFailureRetry,
provisionRequestsByMessage,
onVaultRequestResolved,
onQuoteSelection,
@@ -3753,6 +3804,9 @@ export const Transcript: React.FC<TranscriptProps> = ({
}
}, [fileViewer, navigate, openApp]);
const selectionActions = transcriptSelectionActions(session, readOnly);
+ const latestRetryBoundary = messages.reduce((last, message, index) =>
+ message.type === 'user' || message.type === 'harness' || isRetryableFailureNotice(message)
+ ? index : last, -1);
const forkSourceSessionId =
typeof session.metadata?.fork_source_session_id === 'string'
? session.metadata.fork_source_session_id
@@ -4207,7 +4261,7 @@ export const Transcript: React.FC<TranscriptProps> = ({
) : null}
{/* Degenerate null-anchor groups render at the TOP (never the tail). */}
{activity?.enabled && activity.topGroups.map((group) => renderActivityChip(group))}
- {messages.map((message) => {
+ {messages.map((message, index) => {
// Agent Activity chips positioned relative to THIS row: 'before' it
// (done/failed, hugging the reply from above) and 'after' it (interrupted,
// just below the turn's trigger).
@@ -4222,6 +4276,8 @@ export const Transcript: React.FC<TranscriptProps> = ({
agentDisplayName={agentDisplayName}
messageFontSize={messageFontSize}
onQuickReply={onQuickReply}
+ onFailureRetry={onFailureRetry}
+ retryDisabled={working || index < latestRetryBoundary}
vaultRequests={provisionRequestsByMessage.get(message.id)}
onVaultRequestResolved={onVaultRequestResolved}
onOpenLocalFile={openLocalFile}
@@ -4350,6 +4406,8 @@ type MessageRowProps = {
agentDisplayName?: string | null;
messageFontSize: number;
onQuickReply?: (messageId: string, choice: string) => boolean | void | Promise<boolean | void>;
+ onFailureRetry?: (messageId: string) => Promise<boolean>;
+ retryDisabled?: boolean;
vaultRequests?: VaultRequest[];
onVaultRequestResolved?: () => void;
onOpenLocalFile?: (target: LocalFileLinkTarget) => void | Promise<void>;
@@ -4380,6 +4438,8 @@ export const MessageRow = memo(function MessageRow({
agentDisplayName,
messageFontSize,
onQuickReply,
+ onFailureRetry,
+ retryDisabled,
vaultRequests,
onVaultRequestResolved,
onOpenLocalFile,
@@ -4561,6 +4621,14 @@ export const MessageRow = memo(function MessageRow({
)}
</span>
</div>
+ {onFailureRetry && (
+ <FailureRetry
+ message={message}
+ readOnly={readOnly}
+ disabled={retryDisabled}
+ onRetry={onFailureRetry}
+ />
+ )}
{time}
</div>
</div>
diff --git a/ui/src/components/workbench/ChatRowOptimisticWrites.test.tsx b/ui/src/components/workbench/ChatRowOptimisticWrites.test.tsx
index 11627e2d6..b735e17d1 100644
--- a/ui/src/components/workbench/ChatRowOptimisticWrites.test.tsx
+++ b/ui/src/components/workbench/ChatRowOptimisticWrites.test.tsx
@@ -14,6 +14,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { MemoryRouter, Route, Routes, useNavigate } from 'react-router-dom';
import type { AgentRoutePatch } from './AgentRoutePicker';
+import type { WorkbenchMessage } from '../../context/ApiContext';
const mocks = vi.hoisted(() => ({
api: {
@@ -38,6 +39,8 @@ const mocks = vi.hoisted(() => ({
// events — the arrival point that has no request to be ordered against.
events: null as null | {
onSessionActivity?: (data: { session_id: string; event: string; title?: string | null }) => void;
+ onMessageNew?: (data: WorkbenchMessage) => void;
+ onMessageUpdated?: (data: WorkbenchMessage) => void;
},
// Captured from the mocked leaf components: the route pick and the send are
// both props ChatPage hands down, so driving them needs no DOM choreography.
@@ -331,6 +334,43 @@ describe('the chat row under an optimistic write', () => {
vi.unstubAllGlobals();
});
+ it('retries the bound failure without submitting the draft or quick-reply label', async () => {
+ await mountChat();
+ const failure = {
+ id: 'msg-failure', session_id: SESSION_ID, author: 'agent', source: 'agent',
+ type: 'notify', text: 'failed', content: {},
+ metadata: { event: 'backend_failure', turn_id: 'failed-turn', failure_id: 'failed-turn' },
+ } as WorkbenchMessage;
+ act(() => mocks.events?.onMessageNew?.(failure));
+ mocks.apiFetch.mockResolvedValue({
+ ok: true, status: 202,
+ json: async () => ({ delivery_state: 'claimed', retry_notice: {
+ ...failure, content: { failure_retry: { delivery_id: 'delivery', state: 'claimed' } },
+ } }),
+ });
+ await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'common.retry' })); });
+ const [, request] = mocks.apiFetch.mock.calls.at(-1)!;
+ expect(JSON.parse(request.body)).toEqual({ retry_for: failure.id });
+ expect(mocks.api.reconcileSessionDraftAfterSend).not.toHaveBeenCalled();
+ expect(mocks.api.recoverSessionDraftAfterRejectedSend).not.toHaveBeenCalled();
+ expect(screen.getByRole('button', { name: 'chat.retryRequested' })).toBeTruthy();
+ });
+
+ it('updates an existing notify action without appending another message', async () => {
+ await mountChat();
+ const failure = {
+ id: 'msg-failure', session_id: SESSION_ID, author: 'agent', source: 'agent',
+ type: 'notify', text: 'failed', content: {},
+ metadata: { event: 'backend_failure', turn_id: 'failed-turn', failure_id: 'failed-turn' },
+ } as WorkbenchMessage;
+ act(() => mocks.events?.onMessageNew?.(failure));
+ act(() => mocks.events?.onMessageUpdated?.({
+ ...failure, content: { failure_retry: { delivery_id: 'delivery', state: 'accepted' } },
+ }));
+ expect(document.querySelectorAll('[data-message-id="msg-failure"]')).toHaveLength(1);
+ expect(screen.getByRole('button', { name: 'chat.retryRequested' })).toBeTruthy();
+ });
+
it('sends without waiting for the route write to land', async () => {
const patchGate = deferred<unknown>();
mocks.api.updateSession.mockReturnValue(patchGate.promise);
diff --git a/ui/src/components/workbench/FailureRetry.test.tsx b/ui/src/components/workbench/FailureRetry.test.tsx
new file mode 100644
index 000000000..13e3ab8bc
--- /dev/null
+++ b/ui/src/components/workbench/FailureRetry.test.tsx
@@ -0,0 +1,96 @@
+/** @vitest-environment jsdom */
+import { act, cleanup, fireEvent, render, screen } from '@testing-library/react';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { createInstance } from 'i18next';
+import { I18nextProvider, initReactI18next } from 'react-i18next';
+
+import type { WorkbenchMessage } from '../../context/ApiContext';
+import zh from '../../i18n/zh.json';
+import { FailureRetry } from './FailureRetry';
+
+const i18n = createInstance();
+void i18n.use(initReactI18next).init({
+ lng: 'zh', resources: { zh: { translation: zh } }, interpolation: { escapeValue: false },
+});
+const notice = {
+ id: 'notice', session_id: 'session', type: 'notify', author: 'agent', source: 'agent',
+ text: '模型连接中断', content: {},
+ metadata: { event: 'backend_failure', failure_id: 'failed', turn_id: 'turn' },
+} as WorkbenchMessage;
+
+afterEach(cleanup);
+
+function mount(props: Partial<Parameters<typeof FailureRetry>[0]> = {}) {
+ const onRetry = vi.fn().mockResolvedValue(true);
+ render(
+ <I18nextProvider i18n={i18n}>
+ <FailureRetry message={notice} onRetry={onRetry} {...props} />
+ </I18nextProvider>,
+ );
+ return onRetry;
+}
+
+describe('failed-turn retry action', () => {
+ it('locks a click burst and reports only an admitted retry', async () => {
+ let finish!: (value: boolean) => void;
+ const pending = new Promise<boolean>((resolve) => { finish = resolve; });
+ const onRetry = vi.fn().mockReturnValue(pending);
+ mount({ onRetry });
+ const button = screen.getByRole('button', { name: '重试' });
+ fireEvent.click(button);
+ fireEvent.click(button);
+ expect(onRetry).toHaveBeenCalledExactlyOnceWith('notice');
+ expect((button as HTMLButtonElement).disabled).toBe(true);
+ await act(async () => { finish(true); });
+ expect(screen.getByRole('button', { name: '已请求重试' })).toBeTruthy();
+ });
+
+ it('unlocks after an unsuccessful submission', async () => {
+ const onRetry = vi.fn().mockResolvedValueOnce(false).mockResolvedValueOnce(true);
+ mount({ onRetry });
+ await act(async () => { fireEvent.click(screen.getByRole('button')); });
+ expect((screen.getByRole('button') as HTMLButtonElement).disabled).toBe(false);
+ await act(async () => { fireEvent.click(screen.getByRole('button')); });
+ expect(onRetry).toHaveBeenCalledTimes(2);
+ });
+
+ it.each(['queued', 'claimed', 'accepted'])('restores %s feedback on reload', (state) => {
+ mount({ message: { ...notice, content: { failure_retry: { delivery_id: 'delivery', state } } } });
+ expect((screen.getByRole('button', { name: '已请求重试' }) as HTMLButtonElement).disabled).toBe(true);
+ });
+
+ it.each(['queued', 'claimed'])('unlocks a %s retry after durable retirement without remounting', async (state) => {
+ const onRetry = vi.fn().mockResolvedValue(true);
+ const row = (message: WorkbenchMessage) => (
+ <I18nextProvider i18n={i18n}>
+ <FailureRetry message={message} onRetry={onRetry} />
+ </I18nextProvider>
+ );
+ const { rerender } = render(row(notice));
+ await act(async () => { fireEvent.click(screen.getByRole('button')); });
+ expect((screen.getByRole('button', { name: '已请求重试' }) as HTMLButtonElement).disabled).toBe(true);
+ rerender(row({ ...notice, content: { failure_retry: { delivery_id: 'delivery', state } } }));
+ expect((screen.getByRole('button') as HTMLButtonElement).disabled).toBe(true);
+ rerender(row({
+ ...notice, content: { failure_retry: { delivery_id: 'delivery', state: 'retired' } },
+ }));
+ expect((screen.getByRole('button', { name: '重试' }) as HTMLButtonElement).disabled).toBe(false);
+ await act(async () => { fireEvent.click(screen.getByRole('button')); });
+ expect(onRetry).toHaveBeenCalledTimes(2);
+ rerender(row({
+ ...notice, content: { failure_retry: { delivery_id: 'next-delivery', state: 'accepted' } },
+ }));
+ expect((screen.getByRole('button', { name: '已请求重试' }) as HTMLButtonElement).disabled).toBe(true);
+ });
+
+ it.each([
+ { readOnly: true },
+ { message: { ...notice, metadata: { ...notice.metadata, event: 'progress' } } },
+ { message: { ...notice, metadata: { ...notice.metadata, detached: true } } },
+ { message: { ...notice, metadata: { event: 'backend_failure' } } },
+ { message: { ...notice, author: 'user' } as WorkbenchMessage },
+ ])('does not offer an ineligible action', (props) => {
+ mount(props);
+ expect(screen.queryByRole('button')).toBeNull();
+ });
+});
diff --git a/ui/src/components/workbench/FailureRetry.tsx b/ui/src/components/workbench/FailureRetry.tsx
new file mode 100644
index 000000000..11e0de4c7
--- /dev/null
+++ b/ui/src/components/workbench/FailureRetry.tsx
@@ -0,0 +1,54 @@
+import { useRef, useState } from 'react';
+import { Check, RotateCcw } from 'lucide-react';
+import { useTranslation } from 'react-i18next';
+
+import type { WorkbenchMessage } from '../../context/ApiContext';
+import { isRetryableFailureNotice } from '../../lib/chatMessageTypes';
+import { Button } from '../ui/button';
+
+export function FailureRetry({
+ message,
+ readOnly,
+ disabled,
+ onRetry,
+}: {
+ message: WorkbenchMessage;
+ readOnly?: boolean;
+ disabled?: boolean;
+ onRetry: (messageId: string) => Promise<boolean>;
+}) {
+ const { t } = useTranslation();
+ const pending = useRef(false);
+ const [sending, setSending] = useState(false);
+ const [requested, setRequested] = useState(false);
+ const action = message.content?.failure_retry as { state?: string } | undefined;
+ // Local success bridges the response only until a durable state is present.
+ // In particular, a later retirement must unlock this same mounted row.
+ const admitted = action?.state
+ ? ['queued', 'claimed', 'accepted'].includes(action.state)
+ : requested;
+ if (readOnly || !isRetryableFailureNotice(message)) return null;
+ return (
+ <Button
+ type="button"
+ size="sm"
+ variant="secondary"
+ disabled={disabled || sending || admitted}
+ aria-busy={sending}
+ onClick={async () => {
+ if (pending.current || admitted || disabled) return;
+ pending.current = true;
+ setSending(true);
+ try {
+ setRequested(await onRetry(message.id));
+ } finally {
+ pending.current = false;
+ setSending(false);
+ }
+ }}
+ >
+ {admitted ? <Check className="size-3.5" /> : <RotateCcw className="size-3.5" />}
+ {t(admitted ? 'chat.retryRequested' : 'common.retry')}
+ </Button>
+ );
+}
diff --git a/ui/src/context/ApiContext.streamLiveness.test.tsx b/ui/src/context/ApiContext.streamLiveness.test.tsx
index 06cc6df4e..e60adf1a2 100644
--- a/ui/src/context/ApiContext.streamLiveness.test.tsx
+++ b/ui/src/context/ApiContext.streamLiveness.test.tsx
@@ -72,10 +72,12 @@ class FakeEventSource {
* stream that never broke must not produce it a second time.
*/
const onConnected = vi.fn();
+const onMessageNew = vi.fn();
+const onMessageUpdated = vi.fn();
const Subscriber = () => {
const api = useApi();
- useEffect(() => api.connectWorkbenchEvents({ onConnected }), [api]);
+ useEffect(() => api.connectWorkbenchEvents({ onConnected, onMessageNew, onMessageUpdated }), [api]);
return null;
};
@@ -170,6 +172,8 @@ beforeEach(() => {
apiFetch.mockResolvedValue({ ok: true, status: 200, json: async () => ({}) });
showToast.mockReset();
onConnected.mockReset();
+ onMessageNew.mockReset();
+ onMessageUpdated.mockReset();
FakeEventSource.instances = [];
vi.stubGlobal('EventSource', FakeEventSource);
Object.defineProperty(document, 'visibilityState', { configurable: true, value: 'visible' });
@@ -183,6 +187,14 @@ afterEach(() => {
});
describe('ApiProvider workbench stream liveness', () => {
+ it('routes an action update without replaying a terminal message', () => {
+ mountConnectedStream();
+ const notice = { id: 'failed-notice', session_id: 'session', type: 'notify' };
+ FakeEventSource.latest().emit('message.updated', { type: 'message.updated', data: notice });
+ expect(onMessageUpdated).toHaveBeenCalledExactlyOnceWith(notice);
+ expect(onMessageNew).not.toHaveBeenCalled();
+ });
+
it('leaves a stream that keeps proving itself alone across reactivations', () => {
mountConnectedStream();
expect(onConnected).toHaveBeenCalledTimes(1);
diff --git a/ui/src/context/ApiContext.tsx b/ui/src/context/ApiContext.tsx
index d0f0b74c6..d60c09bbb 100644
--- a/ui/src/context/ApiContext.tsx
+++ b/ui/src/context/ApiContext.tsx
@@ -771,7 +771,7 @@ export type ApiContextType = {
q: string,
opts?: { limit?: number; includeArchived?: boolean },
) => Promise<MessageSearchResult>;
- sendSessionMessage: (sessionId: string, payload: { text?: string; content?: Record<string, unknown>; metadata?: Record<string, unknown>; author_id?: string; author_name?: string }) => Promise<WorkbenchMessage>;
+ sendSessionMessage: (sessionId: string, payload: { text?: string; content?: Record<string, unknown>; metadata?: Record<string, unknown>; author_id?: string; author_name?: string; retry_for?: string }) => Promise<WorkbenchMessage>;
markSessionRead: (sessionId: string, untilMessageId?: string, opts?: { handleError?: boolean }) => Promise<{ updated: number; unread_counts: Record<string, number>; unread_by_session?: Record<string, number> }>;
cancelSession: (
sessionId: string,
@@ -1223,6 +1223,7 @@ export type WorkbenchEventHandlers = {
instance_authorization_revision?: number;
}) => void;
onMessageNew?: (data: WorkbenchMessage) => void;
+ onMessageUpdated?: (data: WorkbenchMessage) => void;
// ``visibility`` (contract A6): the backend carries the session's current
// foreground/background on visibility/scope changes so the Inbox can drop /
// restore the card live. Absent on pre-M1 backends ⇒ consumers no-op.
@@ -3127,6 +3128,15 @@ export const ApiProvider: React.FC<{ children: React.ReactNode }> = ({ children
handlers.onMessageNew?.(envelope.data);
});
});
+ source.addEventListener('message.updated', (e: MessageEvent) => {
+ const envelope = parseWorkbenchEnvelope<WorkbenchMessage>(e.data);
+ if (!envelope?.data.session_id) return;
+ clearSessionReadCache(envelope.data.session_id);
+ dispatchToWorkbenchHandlers((handlers) => {
+ handlers.onAny?.(envelope);
+ handlers.onMessageUpdated?.(envelope.data);
+ });
+ });
source.addEventListener('session.activity', (e: MessageEvent) => {
const envelope = parseWorkbenchEnvelope<any>(e.data);
if (!envelope) return;
diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.json
index 5c014e005..0bb1236a9 100644
--- a/ui/src/i18n/en.json
+++ b/ui/src/i18n/en.json
@@ -2051,6 +2051,12 @@
"retry": "Retry"
},
"errors": {
+ "retry_not_available": "This notice cannot be retried. Send a message to continue.",
+ "retry_stale": "Newer work has already started. Send a message if you want to continue.",
+ "retry_busy": "The previous turn is still settling. Try again after it stops.",
+ "retry_acceptance_unknown": "The previous request's delivery is still uncertain. Retry is disabled to avoid repeating work.",
+ "retry_pending_input": "This conversation already has pending input. Retry would duplicate or reorder it.",
+ "retry_original_unavailable": "The original unsent input is no longer available for a safe retry.",
"runtime_install_inspection_failed": "Show Runtime status could not be inspected. No install or repair was attempted.",
"model_hub_engine_archive_checksum_mismatch": "The CPA download failed its integrity check. Retry the installation.",
"model_hub_engine_archive_download_failed": "CPA could not be downloaded. Check your network connection and retry.",
@@ -4133,6 +4139,8 @@
"streaming": "Streaming",
"thinking": "Thinking…",
"notifyLabel": "Notify",
+ "retryRequested": "Retry requested",
+ "retryFailed": "Could not request retry. You can try again safely.",
"collapse": "Collapse",
"turnInProgress": "A turn is already running — wait for the reply.",
"stopFailed": "Couldn't reach the agent to stop it — it may still be running.",
diff --git a/ui/src/i18n/zh.json b/ui/src/i18n/zh.json
index cefe412cf..f9fb369d9 100644
--- a/ui/src/i18n/zh.json
+++ b/ui/src/i18n/zh.json
@@ -2051,6 +2051,12 @@
"retry": "重试"
},
"errors": {
+ "retry_not_available": "这条通知无法重试,可以发送消息继续。",
+ "retry_stale": "会话已经开始了后续任务,如需继续请发送消息。",
+ "retry_busy": "上一轮还在结束处理中,请稍后再试。",
+ "retry_acceptance_unknown": "暂时无法确认原请求是否已送达,为避免重复执行,不能重试。",
+ "retry_pending_input": "会话已有待处理输入,请先等待处理,避免重复或打乱顺序。",
+ "retry_original_unavailable": "原始未送达输入已不可用,无法安全重试。",
"runtime_install_inspection_failed": "无法检查 Show Runtime 状态。未执行安装或修复。",
"model_hub_engine_archive_checksum_mismatch": "CPA 下载文件未通过完整性校验,请重试安装。",
"model_hub_engine_archive_download_failed": "无法下载 CPA,请检查网络连接后重试。",
@@ -4133,6 +4139,8 @@
"streaming": "实时输出中",
"thinking": "思考中…",
"notifyLabel": "通知",
+ "retryRequested": "已请求重试",
+ "retryFailed": "未能提交重试,可以再次点击。",
"collapse": "收起",
"turnInProgress": "当前回合还在进行中,请等待回复后再发送。",
"stopFailed": "没能联系到 Agent 来停止,它可能还在运行。",
diff --git a/ui/src/lib/chatMessageTypes.ts b/ui/src/lib/chatMessageTypes.ts
index a2dc0e9c8..9b3c66ba8 100644
--- a/ui/src/lib/chatMessageTypes.ts
+++ b/ui/src/lib/chatMessageTypes.ts
@@ -52,6 +52,23 @@ export const isBoundaryMessage = (message: TerminalMessageCandidate): boolean =>
type TerminalAgentMessageCandidate = TerminalMessageCandidate & { author: string };
+// This is action eligibility, not transcript visibility. The server additionally
+// rechecks the durable failed-Turn boundary and current Session authority.
+export function isRetryableFailureNotice(
+ message: TerminalAgentMessageCandidate & { source?: string | null },
+): boolean {
+ const metadata = message.metadata;
+ return message.type === 'notify'
+ && message.author === 'agent'
+ && message.source === 'agent'
+ && metadata?.event === 'backend_failure'
+ && typeof metadata.failure_id === 'string'
+ && !!metadata.failure_id
+ && typeof metadata.turn_id === 'string'
+ && !!metadata.turn_id
+ && !metadata.detached;
+}
+
// A terminal reply the TRANSCRIPT shows: the catalog's terminal activity role
// intersected with transcript visibility (``silent`` is terminal for activity
// bookkeeping but never rendered), plus the conditional terminals that only settle
diff --git a/vibe/authorization.py b/vibe/authorization.py
index db80bb883..a45886e5f 100644
--- a/vibe/authorization.py
+++ b/vibe/authorization.py
@@ -66,6 +66,7 @@ _VIEWER_WORKBENCH_EVENTS = frozenset(
"inbox.session.updated",
"inbox.unread.changed",
"message.new",
+ "message.updated",
"projects.changed",
"session.activity",
"session.status",
diff --git a/vibe/ui_server.py b/vibe/ui_server.py
index 5f4d83e6a..8d15bb895 100644
--- a/vibe/ui_server.py
+++ b/vibe/ui_server.py
@@ -11270,6 +11270,9 @@ async def sessions_messages_create(session_id: str):
or, when a turn is already running, promotes it to ``queued`` itself
(send-while-busy). The legacy per-turn ``?stream=1`` SSE proxy was retired
in Step 6 — the session-scoped stream replaced it.
+
+ ``retry_for`` is a notice-bound action. Its input and idempotency identity
+ are derived from durable server state; it never consumes the composer draft.
"""
from core.services import sessions as workbench_sessions_service
@@ -11283,6 +11286,13 @@ async def sessions_messages_create(session_id: str):
from vibe import internal_client
payload = request.json or {}
+ retry_for = payload.get("retry_for")
+ if retry_for is not None:
+ if not isinstance(retry_for, str) or not retry_for.strip():
+ return jsonify({"error": "invalid retry_for"}), 400
+ # This is an action, not a caller-supplied replacement prompt. Do not
+ # accept attachments, identities, or quick-reply metadata alongside it.
+ payload = {"text": "continue"}
text = payload.get("text")
content = payload.get("content")
if text is None and not content:
@@ -11290,6 +11300,8 @@ async def sessions_messages_create(session_id: str):
# A quick-reply click tags the row with the agent message it answers.
quick_reply_for = (payload.get("metadata") or {}).get("quick_reply_for")
message_kind = workbench_message_kind(payload, quick_reply_for)
+ if retry_for:
+ message_kind = "quick_reply"
web_push_user_key = _web_push_user_key()
workbench_author_id = _workbench_author_id()
web_push_authorization_context = web_push_authorization_context_record(
@@ -11379,43 +11391,74 @@ async def sessions_messages_create(session_id: str):
message_metadata[WEB_PUSH_AUTHORIZATION_CONTEXTS_METADATA] = [
web_push_authorization_context
]
- row = message_deliveries.insert_delivery(
- conn,
- delivery_id=delivery_id,
+ snapshot = message_deliveries.message_snapshot(
+ scope_id=session["scope_id"],
session_id=session_id,
- priority="p3",
- state="reserved",
- snapshot=message_deliveries.message_snapshot(
- scope_id=session["scope_id"],
- session_id=session_id,
- platform="avibe",
- author="user",
- source="user",
- text=text if isinstance(text, str) else None,
- content=content if isinstance(content, dict) else None,
- metadata=message_metadata,
- author_id=workbench_author_id,
- author_name=payload.get("author_name"),
- message_kind=message_kind,
- ),
- dispatch_text=dispatch_text,
- history_event={"kind": "admission", "priority": "p3", "state": "reserved"},
+ platform="avibe",
+ author="user",
+ source="user",
+ text=text if isinstance(text, str) else None,
+ content=content if isinstance(content, dict) else None,
+ metadata=message_metadata,
+ author_id=workbench_author_id,
+ author_name=payload.get("author_name"),
+ message_kind=message_kind,
)
- if not quick_reply_for:
+ if retry_for:
+ from core.backend_failure_retry import reserve_retry
+
+ row = reserve_retry(
+ conn,
+ session_id=session_id,
+ notice_id=retry_for,
+ snapshot=snapshot,
+ )
+ else:
+ row = message_deliveries.insert_delivery(
+ conn,
+ delivery_id=delivery_id,
+ session_id=session_id,
+ priority="p3",
+ state="reserved",
+ snapshot=snapshot,
+ dispatch_text=dispatch_text,
+ history_event={"kind": "admission", "priority": "p3", "state": "reserved"},
+ )
+ if not quick_reply_for and not retry_for:
message_deliveries.set_draft(conn, session_id, None)
draft = message_deliveries.get_draft_state(conn, session_id)
workbench_sessions_service.touch_session(conn, session_id)
result = message_deliveries.public_delivery_payload(row)
result["draft"] = _session_draft_payload(draft)
- result["draft_advanced"] = not bool(quick_reply_for)
+ result["draft_advanced"] = not bool(quick_reply_for or retry_for)
return result
# Reserve the row FIRST (pending), then decide by the dispatch outcome.
- message = _persist_user_row()
+ from core.backend_failure_retry import RetryUnavailable
+
+ try:
+ message = _persist_user_row()
+ except RetryUnavailable as error:
+ return _coded_error_response(error.code, error.code, 409)
if message is None:
# Archived between the pre-flight check and the reservation — stay terminal.
return _session_archived_response()
- if not dispatch_text.strip() and not attachment_specs:
+ if retry_for:
+ # Retained pre-write input comes from its immutable Delivery, including
+ # attachments. The controller independently reconstructs the same input.
+ content = message.get("content") or {}
+ dispatch_text = message.get("dispatch_text") or message.get("text") or ""
+ from core.workbench_media import resolve_attachment_specs
+
+ with engine.connect() as conn:
+ attachment_specs = resolve_attachment_specs(
+ conn, session_id=session_id, attachments=content.get("attachments") or []
+ )
+ if (
+ not dispatch_text.strip()
+ and not attachment_specs
+ and not (retry_for and message.get("state") == "accepted")
+ ):
from storage import message_deliveries
with engine.begin() as conn:
@@ -11451,11 +11494,21 @@ async def sessions_messages_create(session_id: str):
with engine.connect() as conn:
current = message_deliveries.get_delivery(conn, str(message["id"]))
+ retry_notice = (
+ messages_service.get_message(conn, retry_for, session_id=session_id)
+ if retry_for else None
+ )
if current is None:
return dict(message)
payload = message_deliveries.public_delivery_payload(current)
payload["draft"] = message["draft"]
payload["draft_advanced"] = message["draft_advanced"]
+ if retry_notice is not None:
+ from vibe.sse_broker import broker
+
+ payload["retry_notice"] = retry_notice
+ # An upsert of an existing notice is not new user communication.
+ broker.publish("message.updated", retry_notice)
if current["state"] == "queued":
payload["type"] = "queued"
payload["queued"] = True
@@ -11522,6 +11575,8 @@ async def sessions_messages_create(session_id: str):
if status == 202:
delivery_state = str(body.get("delivery_state") or "")
current = _current_delivery_response()
+ if retry_for and current.get("state") == "retired":
+ return jsonify({**current, "code": "retry_stale", "error": "retry_stale"}), 409
if delivery_state == "accepted":
accepted_message_id = str(
body.get("message_id")
@@ -11544,10 +11599,13 @@ async def sessions_messages_create(session_id: str):
**body,
"draft": message["draft"],
"draft_advanced": message["draft_advanced"],
+ **({"retry_notice": current["retry_notice"]} if retry_for else {}),
}
), 201
return jsonify({**current, **body}), 202
current = _retire_unclaimed_delivery(f"internal_dispatch_rejected_{status}")
+ if retry_for and status == 409:
+ return jsonify({**current, **body}), 409
return jsonify(
{
**current,