Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(chat): allow retrying service restart interruptions (#1987)

Call Whitec66eca5e790d
diff --git a/core/services/session_fork.py b/core/services/session_fork.py
index a5d7e7303..976009bd3 100644
--- a/core/services/session_fork.py
+++ b/core/services/session_fork.py
@@ -572,6 +572,15 @@ def fork_source_state(fork: dict[str, Any] | None) -> ForkSourceState:
)
== 1,
),
+ ~and_(
+ messages.c.type == "notify",
+ func.coalesce(
+ func.json_extract(messages.c.metadata_json, "$.event"), ""
+ ) == "backend_failure",
+ func.coalesce(
+ func.json_extract(messages.c.metadata_json, "$.replayed"), 0
+ ) == 1,
+ ),
)
.order_by(transcript_order_value().desc(), messages.c.id.desc())
.limit(1)
diff --git a/core/session_turns.py b/core/session_turns.py
index 1cbdff799..8a1a536bf 100644
--- a/core/session_turns.py
+++ b/core/session_turns.py
@@ -32,6 +32,7 @@ from sqlalchemy.exc import IntegrityError
from core.web_push_notifications import WEB_PUSH_USER_KEY_METADATA, WEB_PUSH_USER_KEYS_METADATA
from core.delivery_target import normalize_message_kind
from core.agent_input import AgentInputMetadata
+from core.backend_failure import backend_failure_notification_output
from core.message_context import (
resolve_turn_sink_key,
)
@@ -80,7 +81,7 @@ from storage.models import (
session_turns as session_turn_rows,
)
from storage.workbench_sessions_service import derive_session_harness_activities
-from core.message_output import terminal_turn_output
+from core.message_output import MessageOutput, terminal_turn_output
from core.runtime_activation import (
RuntimeActivationIdentity,
RuntimeActivationRegistry,
@@ -641,7 +642,7 @@ class SessionTurnManager:
] = weakref.WeakValueDictionary()
# Interruption reports owed to turns whose platform was not connected yet
# when recovery ran, keyed by platform. See ``_report_lost_im_turn``.
- self._pending_lost_turn_reports: dict[str, list[tuple[str, str]]] = {}
+ self._pending_lost_turn_reports: dict[str, list[tuple[str, str, str, str]]] = {}
# One in-flight retry task per platform for the reports above.
self._lost_turn_retry_tasks: dict[str, asyncio.Task[None]] = {}
# The live turn sink per TURN SINK KEY. Each is
@@ -6251,6 +6252,8 @@ class SessionTurnManager:
self,
session_id: str,
origin_native_message_id: str,
+ turn_id: str,
+ backend: str,
) -> None:
"""Tell an IM turn's author that its runtime died with the service.
@@ -6288,11 +6291,11 @@ class SessionTurnManager:
return
platform = str(getattr(context, "platform", "") or "")
if self._transport_can_deliver(platform) and await self._emit_lost_turn_report(
- context, session_id, origin_native_message_id
+ context, session_id, origin_native_message_id, turn_id, backend
):
return
self._pending_lost_turn_reports.setdefault(platform, []).append(
- (session_id, str(origin_native_message_id or ""))
+ (session_id, str(origin_native_message_id or ""), turn_id, backend)
)
logger.info(
"lost turn report held until %s transport can deliver (session=%s)",
@@ -6336,8 +6339,8 @@ class SessionTurnManager:
if not pending or self.controller is None:
return 0
reported = 0
- unsent: list[tuple[str, str]] = []
- for session_id, origin_native_message_id in pending:
+ unsent: list[tuple[str, str, str, str]] = []
+ for session_id, origin_native_message_id, turn_id, backend in pending:
try:
context = self._delivery_context(session_id)
except Exception:
@@ -6348,7 +6351,7 @@ class SessionTurnManager:
)
continue
if await self._emit_lost_turn_report(
- context, session_id, origin_native_message_id
+ context, session_id, origin_native_message_id, turn_id, backend
):
reported += 1
else:
@@ -6356,7 +6359,7 @@ class SessionTurnManager:
# transient API error still loses the notice. Popping happened
# first, so an unsent report has to be put BACK or the only
# record of the interruption is gone for the process's lifetime.
- unsent.append((session_id, str(origin_native_message_id or "")))
+ unsent.append((session_id, str(origin_native_message_id or ""), turn_id, backend))
if unsent:
self._pending_lost_turn_reports.setdefault(platform, []).extend(unsent)
logger.info(
@@ -6423,6 +6426,8 @@ class SessionTurnManager:
context: "MessageContext",
session_id: str,
origin_native_message_id: str,
+ turn_id: str,
+ backend: str,
) -> bool:
"""Emit one interruption notice. ``False`` means it did NOT reach the user.
@@ -6438,6 +6443,16 @@ class SessionTurnManager:
context,
"notify",
i18n_t("turn.interrupted.serviceRestart", self._controller_language()),
+ # Recovery already settled this Turn. Carry its exact identity
+ # through delayed sends without granting another settlement or
+ # guessing the target from the Session's current Turn.
+ output=backend_failure_notification_output(
+ context,
+ backend,
+ output=MessageOutput(metadata={"turn_id": turn_id, "replayed": True}),
+ failure_id=f"turn:{turn_id}",
+ failure_id_authoritative=True,
+ ),
)
except Exception:
logger.warning(
@@ -6657,7 +6672,7 @@ class SessionTurnManager:
continue
recovered.append(target_session)
if not owning_run_ids:
- await self._report_lost_im_turn(target_session, origin_message_id)
+ await self._report_lost_im_turn(target_session, origin_message_id, turn_id, backend)
successor_turn_id = str(terminal.get("successor_turn_id") or "")
if successor_turn_id:
await self._start_persisted_turn(successor_turn_id)
diff --git a/docs/plans/backend-failure-retry.md b/docs/plans/backend-failure-retry.md
index 5eeb1ccd1..e82bfe02f 100644
--- a/docs/plans/backend-failure-retry.md
+++ b/docs/plans/backend-failure-retry.md
@@ -8,6 +8,21 @@ not a new backend retry protocol or an automatic retry policy.
- Only an attached `notify` with `event=backend_failure`, a `failure_id`, and a
durable `turn_id` can offer the action. Ordinary notifications, detached
completions, user stops, and authentication-recovery cards are unchanged.
+- A service restart that terminalizes an accepted, non-restorable conversation
+ Turn emits the same failure-notice contract, with the original Turn ID and
+ backend retained across transport readiness waits and failed-send retries.
+ Its visible restart explanation is unchanged. The notice does not settle
+ another Turn or replay the original request; clicking continues through the
+ existing admission path. Harness-owned interruptions retain their existing
+ notice owner, and a restorable live backend does not acquire a retry action.
+ Recovery sets `metadata.replayed=true`: the notice describes an already-ended
+ Turn, even on immediate delivery. This is not detached completion and does not
+ disable Retry. The Web classifier refreshes durable Activity without settling
+ its live generation or hiding a newer Turn's thinking indicator, on live
+ delivery and on reload. The Python Activity classifier and fork progress
+ filter apply the same nonterminal provenance, so history readback cannot
+ reinterpret the notice as newer work ending. Ordinary live failures keep
+ their terminal UI role.
- The action targets the notice's exact Session and failed Turn. The server
rechecks access, archive/read-only status, current Turn, and pending input.
A newer task, another live owner, or an unresolved native start blocks it.
@@ -75,11 +90,14 @@ implementations; IM-specific notification button rendering is not added here.
- Web route tests: access checks, canonical server-generated payload, source
notice update, and reloading persisted action state.
- UI tests: eligible notification, ineligible/read-only rows, pending/accepted
- feedback, failure recovery, and draft-independent action; production UI build.
+ feedback, failure recovery, and draft-independent action; mounted ChatPage
+ preservation of newer Activity/thinking after replayed notices, with ordinary
+ terminal controls; production UI build.
- Isolated local Incus verification only; never the running workstation service
or a remote tenant. No real upstream model request is needed for fault cases.
Scenario contracts: `MESSAGE-DELIVERY-026` through `MESSAGE-DELIVERY-028`.
+Restart notification binding and continuation: `MESSAGE-DELIVERY-030`.
## Known by design
diff --git a/tests/scenarios/message_delivery/catalog.yaml b/tests/scenarios/message_delivery/catalog.yaml
index 4dac397e3..845106609 100644
--- a/tests/scenarios/message_delivery/catalog.yaml
+++ b/tests/scenarios/message_delivery/catalog.yaml
@@ -188,6 +188,12 @@ scenarios:
kind: recovery
layer: contract
test: tests/test_claude_session_recovery.py::test_missing_session_queue_waits_for_explicit_recovery
+ - id: MESSAGE-DELIVERY-030
+ name: Restart interruption retains its failed Turn binding through delivery and one Web continuation
+ status: covered
+ kind: recovery
+ layer: contract
+ test: tests/test_backend_failure_retry.py::test_restart_notice_retries_through_web_and_controller
- id: MESSAGE-DELIVERY-101
name: Unresolved fallback-capable delivery fences only later FIFO submissions
status: covered
diff --git a/tests/scenarios/message_delivery/observations.yaml b/tests/scenarios/message_delivery/observations.yaml
index 265d0adc9..2fa492c55 100644
--- a/tests/scenarios/message_delivery/observations.yaml
+++ b/tests/scenarios/message_delivery/observations.yaml
@@ -1,6 +1,19 @@
version: 1
capability: message_delivery
observations:
+ - id: OBS-MESSAGE-DELIVERY-007
+ source: user_feedback
+ related_scenarios:
+ - MESSAGE-DELIVERY-030
+ summary: Service-restart interruption notifications lacked the Retry action offered for backend failures.
+ previous_assumption: All accepted failed Turns emitted the shared failure-notice metadata.
+ observed_behavior: Restart recovery settled the correct durable Turn but emitted an unlinked text-only notification.
+ required_updates:
+ - preserve the original Turn and backend through transport readiness and failed-send retries
+ - emit the existing non-settling failure-notice contract rather than a second retry mechanism
+ - mark already-ended recovery notices as replayed so delayed delivery cannot settle newer UI Activity or hide thinking
+ - exercise recovery, real message persistence, Web admission, draft preservation, and duplicate clicks together
+ - keep unknown acceptance, user Stop, restored live runtimes, and unlinked historical notices unchanged
- id: OBS-MESSAGE-DELIVERY-006
source: user_feedback
related_scenarios:
diff --git a/tests/test_agent_activity_service.py b/tests/test_agent_activity_service.py
index d531322c9..46d31c2b9 100644
--- a/tests/test_agent_activity_service.py
+++ b/tests/test_agent_activity_service.py
@@ -1756,6 +1756,44 @@ def test_backend_failure_notify_is_failed(isolated_state):
assert [g["status"] for g in groups] == ["failed"]
+def test_replayed_failure_preserves_newer_activity_on_readback(isolated_state):
+ """MESSAGE-DELIVERY-030: a delayed recovery notice is not a new terminal."""
+ engine = create_sqlite_engine()
+ sid = "ses_replayed_failure"
+ with engine.begin() as conn:
+ scope = _seed_session(conn, session_id=sid)
+ _msg(conn, scope, sid, mid="old-input", mtype="user", author="user",
+ created_at="2026-06-01T10:00:00Z", metadata={"turn_id": "old-turn"})
+ _msg(conn, scope, sid, mid="old-step", mtype="assistant", author="agent",
+ created_at="2026-06-01T10:00:01Z", text="old work", metadata={"turn_id": "old-turn"})
+ _evt(conn, scope, sid, eid="old-terminal", created_at="2026-06-01T10:00:02Z",
+ event_type="silent_terminal", text="", turn_id="old-turn",
+ metadata={"terminal_outcome": "failed"})
+ _msg(conn, scope, sid, mid="new-input", mtype="user", author="user",
+ created_at="2026-06-01T10:01:00Z", metadata={"turn_id": "new-turn"})
+ _msg(conn, scope, sid, mid="new-step-1", mtype="assistant", author="agent",
+ created_at="2026-06-01T10:01:01Z", text="new work before notice", metadata={"turn_id": "new-turn"})
+ _msg(conn, scope, sid, mid="old-notice", mtype="notify", author="agent",
+ created_at="2026-06-01T10:01:02Z", text="服务重启导致上一轮中断",
+ metadata={"event": "backend_failure", "replayed": True, "turn_id": "old-turn",
+ "failure_id": "turn:old-turn", "detached": False})
+ _msg(conn, scope, sid, mid="new-step-2", mtype="assistant", author="agent",
+ created_at="2026-06-01T10:01:03Z", text="new work after notice", metadata={"turn_id": "new-turn"})
+
+ with engine.connect() as conn:
+ groups = agent_activity_service.list_turn_groups(conn, session_id=sid)["groups"]
+ assert len(groups) == 2
+ assert groups[0]["status"] == "failed"
+ assert groups[0]["anchor_message_id"] == "old-input"
+ assert groups[0]["open"] is False
+ assert groups[1]["id"] == "new-step-1"
+ assert groups[1]["anchor_message_id"] == "new-input"
+ assert groups[1]["open"] is True
+ assert groups[1]["steps"] == 2
+ detail = agent_activity_service.get_turn_group(conn, session_id=sid, group_id=groups[1]["id"])
+ assert [row["text"] for row in detail["rows"]] == ["new work before notice", "new work after notice"]
+
+
def test_stop_without_terminal_stays_interrupted(isolated_state):
"""Cancel/Stop writes NO marker (no visible terminal, no silent marker), so a
turn with activity and no terminal before the next turn stays ``interrupted``."""
diff --git a/tests/test_backend_failure_retry.py b/tests/test_backend_failure_retry.py
index 6c27afb1c..87aaf18f5 100644
--- a/tests/test_backend_failure_retry.py
+++ b/tests/test_backend_failure_retry.py
@@ -19,6 +19,166 @@ from tests.test_ui_session_stream import _accepted_dispatch, _make_session, isol
from tests.ui_server_test_helpers import csrf_headers
[email protected](autouse=True)
+def _isolate_web_push(monkeypatch):
+ # Real notice persistence is in scope; asynchronous external push delivery
+ # is not. Do not let its worker outlive a test's temporary state directory.
+ monkeypatch.setattr("core.web_push_notifications.maybe_notify_inbox_message", lambda *_args: None)
+
+
+def _restart_failure_notice(tmp_path, *, backend, delivery="ready"):
+ """Recover an accepted Turn through the real notice dispatcher and storage."""
+ from core.message_dispatcher import ConsolidatedMessageDispatcher
+ from core.session_turns import SessionTurnManager
+ from tests.scenario_harness.message_delivery import MessageDeliveryController
+ from tests.test_internal_server import _bind_test_native_start
+
+ _scope_id, session_id = _make_session(tmp_path, agent_backend=backend)
+ engine = create_sqlite_engine()
+ controller = MessageDeliveryController(platform="avibe")
+ controller.config.language = "zh"
+
+ def context(sid):
+ value = _context(sid)
+ value.platform_specific["agent_session_target"]["agent_backend"] = backend
+ return value
+
+ manager = SessionTurnManager(controller, build_context=context)
+ manager._engine = engine
+ controller.session_turns = manager
+ dispatcher = ConsolidatedMessageDispatcher(controller)
+ controller.emit_agent_message = dispatcher.emit_agent_message
+ if delivery == "send_failed":
+ controller.im_client.send_message = AsyncMock(side_effect=[OSError("transport offline"), "msg-recovered"])
+ starts = []
+
+ async def accepted(sid, ctx, text, **_kwargs):
+ starts.append((sid, text))
+ _bind_test_native_start(engine, ctx)
+
+ manager._run = accepted
+ original = asyncio.run(manager.deliver(
+ DeliveryRequest(session_id=session_id, priority="p3", content="完成剩余工作,不要重复提交"),
+ context=context(session_id),
+ ))
+ starts.clear()
+ manager._active_identity = lambda *_args: None
+ manager._transport_can_deliver = lambda _platform: delivery != "unready"
+ asyncio.run(manager.recover_durable_delivery_state(session_id, service_restart=True))
+ assert starts == []
+ if delivery != "ready":
+ with engine.connect() as conn:
+ assert not any(
+ row["type"] == "notify"
+ for row in messages_service.list_session_messages(conn, session_id=session_id)["messages"]
+ )
+ assert asyncio.run(manager.notify_transport_ready("avibe")) == 1
+ with engine.connect() as conn:
+ turn = message_deliveries.get_turn(conn, str(original.turn_id))
+ notices = [
+ row for row in messages_service.list_session_messages(conn, session_id=session_id)["messages"]
+ if row["type"] == "notify"
+ ]
+ assert turn["terminal_evidence_kind"] == "restart_runtime_missing"
+ assert turn["start_receipt_outcome"] == "accepted"
+ assert len(notices) == 1
+ notice = notices[0]
+ assert "服务在它运行期间重启" in notice["text"]
+ assert notice["metadata"]["event"] == "backend_failure"
+ assert notice["metadata"]["turn_id"] == turn["id"]
+ assert notice["metadata"]["failure_id"] == f"turn:{turn['id']}"
+ assert notice["metadata"]["backend"] == backend
+ assert notice["metadata"]["detached"] is False
+ assert notice["metadata"]["replayed"] is True
+ # Neither another recovery pass nor another readiness hook emits a duplicate.
+ asyncio.run(manager.recover_durable_delivery_state(session_id, service_restart=True))
+ assert asyncio.run(manager.notify_transport_ready("avibe")) == 0
+ return session_id, notice
+
+
[email protected]("backend", ["claude", "codex"])
[email protected]("delivery", ["ready", "unready", "send_failed"])
+def test_restart_notice_retries_through_web_and_controller(isolated_state, tmp_path, monkeypatch, backend, delivery):
+ """MESSAGE-DELIVERY-030: restart -> persisted notice -> Web -> one continuation."""
+ import httpx
+ from core import internal_server
+ from tests.test_internal_server import _bind_test_native_start, _build_controller_double
+ from vibe import ui_server
+
+ session_id, notice = _restart_failure_notice(tmp_path, backend=backend, delivery=delivery)
+ engine = create_sqlite_engine()
+ controller = _build_controller_double()
+ internal_app = internal_server.create_app(controller)
+ manager = controller.session_turns
+ starts = []
+
+ async def capture_run(sid, context, text, *, logical_turn_id=None, **_kwargs):
+ starts.append((sid, text))
+ _bind_test_native_start(engine, context)
+ manager._terminalize_durable_turn(
+ logical_turn_id, "completed", settled_by="terminal_result", evidence_kind="test_terminal",
+ )
+
+ manager._run = capture_run
+ transport = httpx.ASGITransport(app=internal_app)
+
+ async def dispatch(payload):
+ async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as internal:
+ response = await internal.post("/internal/dispatch_async", json=payload)
+ return {"status_code": response.status_code, "body": response.json()}
+
+ monkeypatch.setattr("vibe.internal_client.dispatch_async", dispatch)
+ client = ui_server.app.test_client()
+ headers = csrf_headers(client)
+ draft = client.put(
+ f"/api/sessions/{session_id}/draft", headers=headers,
+ json={"text": "未发送的草稿", "expected_updated_at": None},
+ ).get_json()["draft"]
+ url = f"/api/sessions/{session_id}/messages"
+ first = client.post(url, json={"retry_for": notice["id"], "text": "repeat everything"}, headers=headers)
+ duplicate = client.post(url, json={"retry_for": notice["id"]}, headers=headers)
+ assert first.status_code == duplicate.status_code == 201
+ assert first.get_json()["id"] == duplicate.get_json()["id"]
+ assert first.get_json()["draft"] == draft
+ assert first.get_json()["draft_advanced"] is False
+ assert first.get_json()["retry_notice"]["content"]["failure_retry"]["state"] == "accepted"
+ assert starts == [(session_id, "continue")]
+ with engine.connect() as conn:
+ reloaded = messages_service.get_message(conn, notice["id"])
+ assert reloaded["text"] == notice["text"]
+ assert reloaded["content"]["failure_retry"]["state"] == "accepted"
+
+
[email protected]("change", ["newer", "busy", "unlinked"])
+def test_restart_notice_keeps_retry_boundary_guards(isolated_state, tmp_path, change):
+ from vibe.ui_server import app
+
+ session_id, notice = _restart_failure_notice(tmp_path, backend="codex")
+ with create_sqlite_engine().begin() as conn:
+ if change == "newer":
+ seed_failed_notice(conn, session_id=session_id, scope_id=notice["scope_id"])
+ elif change == "busy":
+ message_deliveries.insert_delivery(
+ conn, delivery_id=message_deliveries.new_delivery_id(),
+ session_id=session_id, priority="p3", state="reserved",
+ snapshot=message_deliveries.message_snapshot(
+ scope_id=notice["scope_id"], session_id=session_id, platform="avibe",
+ author="user", source="user", text="另一个请求",
+ ),
+ dispatch_text="另一个请求",
+ )
+ else:
+ conn.execute(update(messages).where(messages.c.id == notice["id"]).values(metadata_json="{}"))
+ client = app.test_client()
+ with patch("vibe.internal_client.dispatch_async", AsyncMock()) as dispatch:
+ response = client.post(
+ f"/api/sessions/{session_id}/messages", headers=csrf_headers(client),
+ json={"retry_for": notice["id"]},
+ )
+ assert response.status_code == 409
+ dispatch.assert_not_awaited()
+
+
def _retry_notice(tmp_path, *, backend="claude", not_written=False, content=None):
scope_id, session_id = _make_session(tmp_path, agent_backend=backend)
with create_sqlite_engine().begin() as conn:
diff --git a/tests/test_message_type_catalog.py b/tests/test_message_type_catalog.py
index 093a7a28f..59a6f0159 100644
--- a/tests/test_message_type_catalog.py
+++ b/tests/test_message_type_catalog.py
@@ -250,6 +250,15 @@ def test_detached_completion_classification_is_catalog_owned() -> None:
)
+def test_replayed_failure_is_not_foreground_activity() -> None:
+ metadata = {"event": "backend_failure", "replayed": True, "detached": False}
+ assert activity_role_for("notify", metadata) == "none"
+ assert not is_detached_completion("notify", metadata)
+ assert not agent_activity_service._is_terminal("notify", "agent", metadata)
+ assert activity_role_for("notify", {**metadata, "replayed": False}) == "terminal"
+ assert activity_role_for("result", metadata) == "terminal"
+
+
def test_web_push_candidate_exact_and_unread_sets_match_current_service() -> None:
expected_candidates = {"result", "error", "notify"}
expected_unread = {"result"}
diff --git a/tests/test_session_delivery_fsm.py b/tests/test_session_delivery_fsm.py
index 1a821df5e..589bdd897 100644
--- a/tests/test_session_delivery_fsm.py
+++ b/tests/test_session_delivery_fsm.py
@@ -5026,7 +5026,7 @@ def test_lost_im_turn_report_survives_a_failed_send(managers) -> None:
# dispatcher swallows a send failure and answers None, and the turn is
# already terminal, so treating that as done would discard the only account
# of the interruption the user will ever get.
- first, restarted, _engine, _engine_b, _starts = managers
+ first, restarted, engine, _engine_b, _starts = managers
context = _context()
admitted = asyncio.run(
first.deliver(
@@ -5055,9 +5055,11 @@ def test_lost_im_turn_report_survives_a_failed_send(managers) -> None:
emitted, stamped = _capture_lost_turn_report(restarted)
restarted._active_identity = lambda *_args: None
delivers = False
+ outputs = []
- async def _emit(_context, kind, text, **_kwargs):
+ async def _emit(_context, kind, text, **kwargs):
emitted.append((kind, text))
+ outputs.append(kwargs["output"])
return "msg-1" if delivers else None
restarted.controller.emit_agent_message = _emit
@@ -5068,10 +5070,24 @@ def test_lost_im_turn_report_survives_a_failed_send(managers) -> None:
assert [kind for kind, _text in emitted] == ["notify"]
assert stamped == []
+ # The old notice may arrive after another task starts. Retain the original
+ # failed Turn identity instead of borrowing the current live context.
+ newer_turn, newer_context = asyncio.run(_activate(restarted, text="new work"))
+ restarted._delivery_context = lambda _session_id: newer_context
+ with engine.connect() as conn:
+ before = delivery_store.get_turn(conn, newer_turn)
delivers = True
assert asyncio.run(restarted.notify_transport_ready("avibe")) == 1
assert len(emitted) == 2
assert stamped == [("m-origin", INTERRUPTED_REACTION_EMOJI)]
+ assert outputs[0] == outputs[1]
+ assert outputs[1].provenance(newer_context)["turn_id"] == turn_id
+ assert outputs[1].metadata["failure_id"] == f"turn:{turn_id}"
+ assert outputs[1].metadata["replayed"] is True
+ assert outputs[1].completes_turn is False
+ assert outputs[1].settles_run is False
+ with engine.connect() as conn:
+ assert delivery_store.get_turn(conn, newer_turn) == before
def test_retained_lost_turn_report_is_retried_on_its_own_clock(managers) -> None:
@@ -5082,7 +5098,7 @@ def test_retained_lost_turn_report_is_retried_on_its_own_clock(managers) -> None
_first, restarted, _engine, _engine_b, _starts = managers
emitted, _stamped = _capture_lost_turn_report(restarted)
restarted.LOST_TURN_RETRY_DELAYS = (0.0, 0.0)
- restarted._pending_lost_turn_reports["slack"] = [("ses_fsm", "m-origin")]
+ restarted._pending_lost_turn_reports["slack"] = [("ses_fsm", "m-origin", "trn-origin", "codex")]
failures = 1
async def _emit(_context, kind, text, **_kwargs):
@@ -5116,7 +5132,7 @@ def test_lost_turn_retry_gives_up_instead_of_spinning(managers) -> None:
_first, restarted, _engine, _engine_b, _starts = managers
emitted, _stamped = _capture_lost_turn_report(restarted)
restarted.LOST_TURN_RETRY_DELAYS = (0.0, 0.0)
- restarted._pending_lost_turn_reports["slack"] = [("ses_fsm", "m-origin")]
+ restarted._pending_lost_turn_reports["slack"] = [("ses_fsm", "m-origin", "trn-origin", "codex")]
async def _emit(_context, kind, text, **_kwargs):
emitted.append((kind, text))
@@ -5133,7 +5149,7 @@ def test_lost_turn_retry_gives_up_instead_of_spinning(managers) -> None:
# One initial attempt plus one per configured delay, then it stops.
assert len(emitted) == 3
- assert restarted._pending_lost_turn_reports["slack"] == [("ses_fsm", "m-origin")]
+ assert restarted._pending_lost_turn_reports["slack"] == [("ses_fsm", "m-origin", "trn-origin", "codex")]
def test_lost_turn_owning_a_run_leaves_the_notice_to_the_harness_lane(managers) -> None:
diff --git a/tests/test_session_fork.py b/tests/test_session_fork.py
index dd98789ef..4922db56b 100644
--- a/tests/test_session_fork.py
+++ b/tests/test_session_fork.py
@@ -1375,8 +1375,9 @@ def test_fork_source_state_treats_backend_failure_notify_anchor_as_terminal(
engine.dispose()
-def test_fork_source_state_keeps_detached_failure_outside_current_turn(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
[email protected]("provenance", [{"detached": True}, {"replayed": True, "detached": False}])
+def test_fork_source_state_keeps_historical_failure_outside_current_turn(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provenance: dict
) -> None:
from config import paths
from storage.importer import ensure_sqlite_state
@@ -1400,7 +1401,7 @@ def test_fork_source_state_keeps_detached_failure_outside_current_turn(
metadata={
"event": "backend_failure",
"failure_id": "failure_detached",
- "detached": True,
+ **provenance,
},
)
@@ -1413,8 +1414,12 @@ def test_fork_source_state_keeps_detached_failure_outside_current_turn(
engine.dispose()
-def test_fork_source_state_keeps_terminal_visible_past_later_detached_completion(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
[email protected]("historical_type,provenance", [
+ ("result", {"detached": True}),
+ ("notify", {"event": "backend_failure", "replayed": True, "detached": False}),
+])
+def test_fork_source_state_keeps_terminal_visible_past_later_historical_completion(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, historical_type: str, provenance: dict
) -> None:
from config import paths
from storage.importer import ensure_sqlite_state
@@ -1452,9 +1457,9 @@ def test_fork_source_state_keeps_terminal_visible_past_later_detached_completion
session_id=source_id,
platform="avibe",
author="agent",
- message_type="result",
+ message_type=historical_type,
text="Older background work completed",
- metadata={"detached": True, "turn_id": "turn-background"},
+ metadata={**provenance, "turn_id": "turn-background"},
)
state = fork_source_state({"source_session_id": source_id, "source_message_id": user["id"]})
diff --git a/ui/src/components/workbench/ChatPage.hydration.test.tsx b/ui/src/components/workbench/ChatPage.hydration.test.tsx
index 500ab7632..5655fb235 100644
--- a/ui/src/components/workbench/ChatPage.hydration.test.tsx
+++ b/ui/src/components/workbench/ChatPage.hydration.test.tsx
@@ -409,7 +409,17 @@ describe('ChatPage transcript hydration', () => {
expect(screen.queryByText(projected.text)).toBeNull();
});
- it('refreshes detached Activity without settling the current live generation', async () => {
+ it.each([
+ { kind: 'detached', type: 'result', metadata: { detached: true, activity_id: 'background-1' } },
+ {
+ kind: 'replayed restart', type: 'notify',
+ metadata: {
+ event: 'backend_failure', replayed: true, detached: false,
+ turn_id: 'interrupted-turn', failure_id: 'turn:interrupted-turn',
+ },
+ },
+ ])('refreshes $kind Activity without settling the current live generation', async ({ type, metadata }) => {
+ // MESSAGE-DELIVERY-030: the old notice must not retire newer live UI work.
mocks.api.getSession.mockResolvedValue({ id: 'session-new' });
mocks.api.getSessionBootstrap.mockResolvedValue({
...bootstrapPayload('session-new'),
@@ -438,11 +448,11 @@ describe('ChatPage transcript hydration', () => {
source: 'agent',
});
mocks.events?.onMessageNew({
- ...projectedMessage('detached-result', 'background completed'),
+ ...projectedMessage('old-completion', 'earlier work ended'),
author: 'agent',
- type: 'result',
+ type,
source: 'agent',
- metadata: { detached: true, activity_id: 'background-1' },
+ metadata,
});
mocks.events?.onMessageNew({
...projectedMessage('active-step-2', 'second active step'),
@@ -457,6 +467,48 @@ describe('ChatPage transcript hydration', () => {
expect(screen.getByText('second active step')).toBeTruthy();
});
+ describe.each([false, true])('restart notice with Activity enabled=%s', (activityEnabled) => {
+ it.each(['live', 'bootstrap'])('keeps newer work thinking after a %s notice', async (delivery) => {
+ // MESSAGE-DELIVERY-030: cover both message.new and a persisted reload.
+ const notice = {
+ ...projectedMessage('restart-notice', '服务重启导致上一轮中断'),
+ author: 'agent', source: 'agent', type: 'notify',
+ metadata: {
+ event: 'backend_failure', replayed: true, detached: false,
+ turn_id: 'interrupted-turn', failure_id: 'turn:interrupted-turn',
+ },
+ };
+ mocks.api.getSession.mockResolvedValue({ id: 'session-new' });
+ mocks.api.getSessionBootstrap.mockResolvedValue({
+ ...bootstrapPayload('session-new'),
+ config: { ui: { show_agent_activity: activityEnabled } },
+ turn_state: { ...idleTurnState, foreground: 'running', in_flight: true },
+ messages: delivery === 'bootstrap' ? [notice] : [],
+ });
+ render(
+ <MemoryRouter initialEntries={['/chat/session-new']}>
+ <Routes>
+ <Route path="/chat/:sessionId" element={<ChatPage />} />
+ </Routes>
+ </MemoryRouter>,
+ );
+ await screen.findByText('chat.thinking');
+ if (delivery === 'live') act(() => mocks.events?.onMessageNew(notice));
+ expect(screen.getByText(notice.text, { exact: false })).toBeTruthy();
+ expect(screen.getByText('chat.thinking')).toBeTruthy();
+ expect(screen.getByRole('button', { name: 'common.retry' })).toBeTruthy();
+ expect(mocks.composer?.busy).toBe(true);
+
+ // A real foreground failure still ends its visible response immediately.
+ act(() => mocks.events?.onMessageNew({
+ ...notice, id: 'zz-current-failure', created_at: '2026-08-15T00:01:00Z', metadata: {
+ event: 'backend_failure', turn_id: 'current-turn', failure_id: 'current-failure',
+ },
+ }));
+ expect(screen.queryByText('chat.thinking')).toBeNull();
+ });
+ });
+
it('does not render an open activity group as interrupted before the switched chat turn state is known', async () => {
const openGroup = {
id: 'open-turn',
diff --git a/ui/src/components/workbench/FailureRetry.test.tsx b/ui/src/components/workbench/FailureRetry.test.tsx
index 13e3ab8bc..926b1f4c8 100644
--- a/ui/src/components/workbench/FailureRetry.test.tsx
+++ b/ui/src/components/workbench/FailureRetry.test.tsx
@@ -31,6 +31,23 @@ function mount(props: Partial<Parameters<typeof FailureRetry>[0]> = {}) {
}
describe('failed-turn retry action', () => {
+ it('offers retry for a restart interruption with authoritative Turn metadata', async () => {
+ const onRetry = mount({
+ message: {
+ ...notice,
+ text: '⚠️ 本轮执行被中断——Avibe 服务在它运行期间重启。如果这项工作仍然需要完成,请重新发送请求。',
+ metadata: {
+ event: 'backend_failure', backend: 'codex',
+ failure_id: 'turn:interrupted-turn', turn_id: 'interrupted-turn', detached: false,
+ replayed: true,
+ },
+ },
+ });
+ await act(async () => { fireEvent.click(screen.getByRole('button', { name: '重试' })); });
+ expect(onRetry).toHaveBeenCalledExactlyOnceWith('notice');
+ expect(screen.getByRole('button', { name: '已请求重试' })).toBeTruthy();
+ });
+
it('locks a click burst and reports only an admitted retry', async () => {
let finish!: (value: boolean) => void;
const pending = new Promise<boolean>((resolve) => { finish = resolve; });
@@ -88,6 +105,7 @@ describe('failed-turn retry action', () => {
{ message: { ...notice, metadata: { ...notice.metadata, event: 'progress' } } },
{ message: { ...notice, metadata: { ...notice.metadata, detached: true } } },
{ message: { ...notice, metadata: { event: 'backend_failure' } } },
+ { message: { ...notice, text: 'Avibe 服务重启,本轮执行被中断', metadata: {} } },
{ message: { ...notice, author: 'user' } as WorkbenchMessage },
])('does not offer an ineligible action', (props) => {
mount(props);
diff --git a/ui/src/lib/chatMessageTypes.test.ts b/ui/src/lib/chatMessageTypes.test.ts
index 4948c41a1..e1305e0e7 100644
--- a/ui/src/lib/chatMessageTypes.test.ts
+++ b/ui/src/lib/chatMessageTypes.test.ts
@@ -5,6 +5,7 @@ import {
isBoundaryMessage,
isDetachedCompletionMessage,
isNotifyMessageType,
+ isRetryableFailureNotice,
isTerminalAgentMessage,
isTranscriptMessage,
shouldRefreshAgentActivityForMessage,
@@ -86,6 +87,26 @@ describe('isTerminalAgentMessage', () => {
expect(isTerminalAgentMessage({ author: 'agent', type, metadata: { detached: true } }), type).toBe(false);
}
});
+
+ it('keeps replayed failure notices retryable without granting foreground authority', () => {
+ const message = {
+ author: 'agent', source: 'agent', type: 'notify',
+ metadata: {
+ event: 'backend_failure', replayed: true, detached: false,
+ turn_id: 'interrupted-turn', failure_id: 'turn:interrupted-turn',
+ },
+ };
+ expect(isTerminalAgentMessage(message)).toBe(false);
+ expect(isAgentActivityBoundaryMessage(message)).toBe(false);
+ expect(isDetachedCompletionMessage(message)).toBe(false);
+ expect(isTranscriptMessage(message)).toBe(true);
+ expect(isRetryableFailureNotice(message)).toBe(true);
+ expect(shouldRefreshAgentActivityForMessage(message)).toBe(true);
+ expect(isTerminalAgentMessage({
+ ...message, metadata: { ...message.metadata, replayed: false },
+ })).toBe(true);
+ expect(isTerminalAgentMessage({ author: 'agent', type: 'result', metadata: { replayed: true } })).toBe(true);
+ });
});
describe('isBoundaryMessage', () => {
diff --git a/ui/src/lib/chatMessageTypes.ts b/ui/src/lib/chatMessageTypes.ts
index 9b3c66ba8..2f9a55735 100644
--- a/ui/src/lib/chatMessageTypes.ts
+++ b/ui/src/lib/chatMessageTypes.ts
@@ -30,11 +30,18 @@ export const isDetachedCompletionMessage = (message: TerminalMessageCandidate):
return message.metadata?.detached === true && spec.detachedCompletion;
};
+// Recovery reports a Turn that already ended, even when transport delivery is
+// immediate. This provenance removes foreground authority, not Retry eligibility.
+const isReplayedFailureNotice = (message: TerminalMessageCandidate): boolean =>
+ message.type === 'notify'
+ && message.metadata?.event === 'backend_failure'
+ && message.metadata.replayed === true;
+
const activityRoleForMessage = (
message: TerminalMessageCandidate,
): ReturnType<typeof specFor>['activityRole'] => {
const spec = specFor(message.type);
- if (isDetachedCompletionMessage(message)) return 'none';
+ if (isDetachedCompletionMessage(message) || isReplayedFailureNotice(message)) return 'none';
const event = message.metadata?.event;
if (typeof event === 'string' && spec.terminalWhenEvents.includes(event)) return 'terminal';
return spec.activityRole;
@@ -85,12 +92,13 @@ export const isTerminalAgentMessage = (message: TerminalAgentMessageCandidate):
export const isAgentActivityBoundaryMessage = (message: TerminalAgentMessageCandidate): boolean =>
message.author === 'agent' && activityRoleForMessage(message) === 'boundary';
-// Terminal replies, nonterminal phase boundaries, and detached completions all
-// require a durable Activity refresh. Only terminal replies settle the live Turn.
+// Historical failures and detached completions refresh durable Activity without
+// affecting its live generation. Only current terminal replies settle the Turn.
export const shouldRefreshAgentActivityForMessage = (
message: TerminalAgentMessageCandidate,
): boolean => {
if (message.author !== 'agent') return false;
const role = activityRoleForMessage(message);
- return role === 'boundary' || role === 'terminal' || isDetachedCompletionMessage(message);
+ return role === 'boundary' || role === 'terminal'
+ || isDetachedCompletionMessage(message) || isReplayedFailureNotice(message);
};
diff --git a/vibe/message_types.py b/vibe/message_types.py
index 7fa5d1eee..367cbfe74 100644
--- a/vibe/message_types.py
+++ b/vibe/message_types.py
@@ -151,6 +151,14 @@ def activity_role_for(message_type: str, metadata: Any = None) -> str:
values = metadata if isinstance(metadata, Mapping) else {}
if is_detached_completion(message_type, values):
return "none"
+ # Recovery has already ended the referenced Turn. A delayed visible notice
+ # must not close a newer chronological Activity group during history reads.
+ if (
+ message_type == "notify"
+ and values.get("event") == "backend_failure"
+ and values.get("replayed") is True
+ ):
+ return "none"
if values.get("event") in spec["terminalWhenEvents"]:
return "terminal"
return str(spec["activityRole"])