Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(vibey): continue work cut by an Avibe restart (#2819)

Call White5d3ffd7e174b
diff --git a/core/agent_core/agent/recovery.py b/core/agent_core/agent/recovery.py
index c0c0bdc36..b8e58efe5 100644
--- a/core/agent_core/agent/recovery.py
+++ b/core/agent_core/agent/recovery.py
@@ -17,6 +17,8 @@ from core.agent_core.tools.base import CallInstance, JobHost, JobStatus, ToolRes
#: ``None`` means the job holds no result for the call (it never ran, or ended for an unrecorded reason).
RecoveryRenderer = Callable[[ToolCallBlock, str, JobStatus, Optional[str]], Awaitable[Optional[ToolResult]]]
+#: For a job still running: settle it as its live call would; its status then, and the Watch it went to, if any.
+ForegroundWait = Callable[[ToolCallBlock, str], Awaitable[tuple[JobStatus, Optional[str]]]]
# recovery.md T2: no job evidence can establish whether a file mutation ran.
UNRECORDED_EFFECT = "[tool call interrupted; it may or may not have completed; re-read the file before continuing]"
@@ -31,6 +33,7 @@ async def settle_open_calls(
jobs: JobHost,
job_ids: Mapping[CallInstance, str],
render_result: RecoveryRenderer,
+ wait_running: Optional[ForegroundWait] = None,
) -> tuple[ContextEntry, ...]:
"""Commit one result per open call, using original-owner job lookup.
@@ -43,10 +46,17 @@ async def settle_open_calls(
Store writes remain under the caller's single-writer ownership. A concurrent
second recovery or Agent.run for that Session is not permitted.
Without job state, the result explicitly leaves the effect uncertain.
- The adapter settles the interrupted Turn after this step; no tool or model
- is invoked and only a new user input starts another Turn (T4). The adapter
- also runs it when a run ends without completing (a Stop), so the calls that
- run left open carry their jobs' recorded reason at once.
+ No tool or model is invoked here; after this step the Turn owner continues
+ the interrupted Turn from the committed transcript or settles it (T4). The
+ adapter also runs it when a run ends without completing (a Stop), so the
+ calls that run left open carry their jobs' recorded reason at once.
+
+ ``wait_running`` is the Turn that continues after a restart: a job still
+ running is settled as its live call would settle it (its foreground window
+ from the job's launch, then a Watch, or the foreground when no Watch can take
+ it; a Stop ends it), so only a job still running then is handed over. The
+ wait commits nothing, so a crash during it leaves the call open for the next
+ settlement, and no job is ever started here.
"""
committed: list[ContextEntry] = []
rows = list(await store.load(session_id))
@@ -59,7 +69,10 @@ async def settle_open_calls(
else:
status = jobs.status(job_id)
details["job_id"] = job_id
- watch_id = await jobs.hand_over(job_id) if status.state == "running" else None
+ if status.state == "running" and wait_running is not None:
+ status, watch_id = await wait_running(deepcopy(call), job_id)
+ else:
+ watch_id = await jobs.hand_over(job_id) if status.state == "running" else None
result = await render_result(deepcopy(call), job_id, status, watch_id)
if result is not None:
message = ToolResultMessage(call.id, call.name, result.content, result.is_error)
diff --git a/core/agent_core/tools/bash.py b/core/agent_core/tools/bash.py
index 85e98b93e..4862d08dd 100644
--- a/core/agent_core/tools/bash.py
+++ b/core/agent_core/tools/bash.py
@@ -15,9 +15,11 @@ import logging
import os
import re
import time
+from datetime import datetime, timezone
from typing import Any, Mapping, Optional
from core.agent_core.cancel import CancelToken
+from core.agent_core.messages import ToolCallBlock
from core.agent_core.tools.args import (
ToolInputError,
error_result,
@@ -208,6 +210,68 @@ async def recovered_result(
return await stopped_result(output, line) if line else None
+async def settle_in_foreground(
+ jobs: LocalJobHost,
+ call: ToolCallBlock,
+ job_id: str,
+ *,
+ stop: asyncio.Event,
+ foreground_window_s: float = DEFAULT_FOREGROUND_WINDOW_S,
+) -> tuple[JobStatus, Optional[str]]:
+ """Settle a job that survived a restart as its live call would (``recovery.md`` T2).
+
+ The call's foreground window counts from the job's launch (``meta.created_at``),
+ so downtime spends it: a window that already passed hands over at once. A job
+ still running then goes to a Watch, retried once as ``BashTool`` does; when no
+ Watch can take it, it stays in the foreground until it ends. A Stop ends the
+ wait and the command, as it ends a live foreground command. Returns the job's
+ status and its Watch, if it went to one.
+ """
+ window_s = foreground_window_s
+ if (call.arguments or {}).get("watch") is True:
+ window_s = min(window_s, _WATCH_GRACE_S)
+ try:
+ launched = datetime.fromisoformat(str(jobs.meta(job_id).get("created_at") or "").replace("Z", "+00:00"))
+ # A wall clock that stepped back never stretches the window past its length.
+ remaining: Optional[float] = min(window_s, window_s - (datetime.now(timezone.utc) - launched).total_seconds())
+ except ValueError:
+ remaining = 0.0
+ status = await _wait_or_stop(jobs, job_id, max(0.0, remaining or 0.0), stop)
+ if status.state != "running":
+ return status, None
+ error: Optional[Exception] = None
+ for _ in range(2):
+ if stop.is_set():
+ # As the live call checks its Stop before each handover: the Stop ends the command.
+ return await _wait_or_stop(jobs, job_id, 0, stop), None
+ try:
+ return status, await jobs.hand_over(job_id)
+ except JobHandOverUnavailable as exc:
+ error = exc
+ break
+ except Exception as exc:
+ error = exc
+ logger.info("Job %s stays in the foreground: %s", job_id, error)
+ return await _wait_or_stop(jobs, job_id, None, stop), None
+
+
+async def _wait_or_stop(jobs: LocalJobHost, job_id: str, seconds: Optional[float], stop: asyncio.Event) -> JobStatus:
+ """The job's status after it ends, ``seconds`` pass (``None``: no limit), or ``stop``, which kills it."""
+ if not stop.is_set() and (seconds is None or seconds > 0):
+ waiting = asyncio.ensure_future(jobs.wait(job_id, deadline_s=seconds))
+ stopped = asyncio.ensure_future(stop.wait())
+ try:
+ await asyncio.wait({waiting, stopped}, return_when=asyncio.FIRST_COMPLETED)
+ finally:
+ for task in (waiting, stopped):
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(waiting, stopped, return_exceptions=True)
+ if stop.is_set() and jobs.status(job_id).state == "running":
+ await jobs.kill(job_id, reason=STOP_ABORTED)
+ return jobs.status(job_id)
+
+
async def settle_bash_call(jobs: LocalJobHost, call: CallInstance) -> Optional[ToolResult]:
"""The durable result for a ``bash`` call left open by a crash; a running job is handed to Watch first."""
job_id = jobs.find_job(call)
diff --git a/core/controller.py b/core/controller.py
index 4649a9d58..497802794 100644
--- a/core/controller.py
+++ b/core/controller.py
@@ -1342,9 +1342,11 @@ class Controller:
if coordinator is not None:
coordinator.restore_migration_blocks()
- # recovery.md: T2 before T4. Vibey hands the commands a restart left running
- # to their Watches first, so the interruption notice of their Turn can say so instead
- # of asking for a resend. Best effort: its single owner retries what it could not settle.
+ # recovery.md: T2 before T4. Vibey settles what a restart left open first; a
+ # command still running for a Turn not yet settled is left to the Turn owner
+ # below, which continues the Turn (its own T2 waits for the command) or has the
+ # command handed to a Watch before its notice names it. Best effort: its single
+ # owner retries what it could not settle.
try:
await self._recover_vibey_agent_runtime_state()
except Exception:
diff --git a/core/handlers/message_handler.py b/core/handlers/message_handler.py
index 95cf4143d..a88103520 100644
--- a/core/handlers/message_handler.py
+++ b/core/handlers/message_handler.py
@@ -781,6 +781,16 @@ class MessageHandler(BaseHandler):
}
context.platform_specific = spec
+ system_prompt = vibe_agent.system_prompt if vibe_agent else None
+ spec = context.platform_specific or {}
+ recorded = spec.get("vibey_execution") if spec.get("vibey_resume") is True else None
+ if isinstance(recorded, dict):
+ # A Turn rerun after a restart runs with the request it was accepted with, as
+ # its native start recorded it, not the Agent settings edited since: those
+ # reach the next Turn. Here, before the shared model validation.
+ effective_model = recorded.get("model")
+ effective_reasoning_effort = recorded.get("reasoning_effort")
+ system_prompt = recorded.get("system_prompt")
request = self._build_agent_request(
context=context,
message=message,
@@ -801,7 +811,7 @@ class MessageHandler(BaseHandler):
vibe_agent_reasoning_effort=effective_reasoning_effort,
vibe_agent_model_explicit="model" in explicit_overrides,
vibe_agent_reasoning_effort_explicit="reasoning_effort" in explicit_overrides,
- vibe_agent_system_prompt=vibe_agent.system_prompt if vibe_agent else None,
+ vibe_agent_system_prompt=system_prompt,
processing_indicator=processing_indicator,
files=processed_files,
)
diff --git a/core/message_dispatcher.py b/core/message_dispatcher.py
index 81e1d3dc7..aa55f42e7 100644
--- a/core/message_dispatcher.py
+++ b/core/message_dispatcher.py
@@ -32,9 +32,11 @@ from core import failure_notices
from core.message_context import TERMINAL_COMMIT_KEY, resolve_turn_sink_key
from core.message_mirror import (
agent_message_exists,
+ delivered_committed_row,
persist_agent_message,
persist_silent_terminal,
promote_turn_text_to_result,
+ resumed_turn_text,
)
from core.message_output import (
HARNESS_RUN_ID_TRIGGER_KINDS,
@@ -3046,6 +3048,15 @@ class ConsolidatedMessageDispatcher:
# controller loop. It adds no cancellation point: a cancellation
# that arrives meanwhile waits for it and is delivered at the next
# await, as the Run terminal record below does.
+ if (
+ turn_text_row_id is None
+ and mutates_turn_lifecycle
+ and level != "silent"
+ and terminal_reason == "done"
+ ):
+ # A rerun after a restart: what its Turn showed before was delivered
+ # by the process that died, so it is read from the Turn's rows.
+ turn_text_row_id = resumed_turn_text(context)
if (
turn_text_row_id is not None
and level != "silent"
@@ -3203,8 +3214,14 @@ class ConsolidatedMessageDispatcher:
accepted_message = candidate_message
native_output_id = candidate
break
+ if accepted_message is None and existing_row_id and canonical_type == "result":
+ # A committed row's own display write is its receipt (a Turn rerun after a
+ # restart ends with the final its dead process may have delivered).
+ accepted_message = delivered_committed_row(
+ target_context, existing_row_id, outward=not suppresses_outward_delivery
+ )
if accepted_message:
- logger.info("Skipping duplicate agent output %s", native_output_id)
+ logger.info("Skipping duplicate agent output %s", native_output_id or existing_row_id)
# A dispatcher that never saw the first emit (a recovery replay) learns
# the row here; an older output's retry keeps the newer row recorded.
if remembers_turn_text:
diff --git a/core/message_mirror.py b/core/message_mirror.py
index 019529ecf..15e96c761 100644
--- a/core/message_mirror.py
+++ b/core/message_mirror.py
@@ -943,6 +943,71 @@ def agent_message_exists(
return None
+def delivered_committed_row(
+ context: MessageContext,
+ row_id: str | None,
+ *,
+ outward: bool,
+) -> Optional[dict[str, Any]]:
+ """The receipt of a backend-committed row an earlier emit already delivered to ``context``.
+
+ A row its backend committed (the Avibe Agent's) has no native id a repeated emit
+ could be recognized by, but its display is written only as its delivery: on the
+ Web that write is the delivery, and on IM it follows a send that returned an id
+ (``persist_agent_message``). A row written for this platform is therefore the
+ receipt a native id is for an inserted row, except local-only history for an
+ outward send. A repeat between a send and that write sends again.
+ """
+
+ platform = str(context.platform or "").strip()
+ if not platform or not row_id:
+ return None
+ try:
+ engine = get_cached_sqlite_engine()
+ with engine.connect() as conn:
+ row = messages_service.get_message(conn, row_id, include_local_error_detail=True)
+ if row is None or row.get("author") != "agent" or row.get("platform") != platform:
+ return None
+ if row.get("scope_id") != _agent_message_scope_id(conn, context):
+ return None
+ except Exception:
+ logger.debug("delivered_committed_row: lookup failed open", exc_info=True)
+ return None
+ content, metadata = row.get("content"), row.get("metadata")
+ if not isinstance(content, dict) or "kind" not in content:
+ return None # as committed: never delivered
+ if outward and isinstance(metadata, dict) and metadata.get("delivery_suppressed") is True:
+ return None
+ return row
+
+
+def resumed_turn_text(context: MessageContext) -> Optional[str]:
+ """The last visible text of a Turn a restart reran, from the Turn's own rows.
+
+ The dispatcher remembers each Turn's visible text as it delivers it; what a rerun
+ Turn showed before the restart was delivered by the process that died, so a
+ silent final of the rerun reads it from the transcript (``last_turn_text``).
+ """
+
+ spec = context.platform_specific or {}
+ session_id, turn_id = str(spec.get("agent_session_id") or ""), str(spec.get("turn_token") or "")
+ if spec.get("vibey_resume") is not True or not session_id or not turn_id:
+ return None
+ from storage import message_deliveries
+ from storage.agent_transcript import last_turn_text
+
+ try:
+ with get_cached_sqlite_engine().connect() as conn:
+ turn = message_deliveries.get_turn(conn, turn_id)
+ if turn is None or turn["session_id"] != session_id:
+ return None
+ found = last_turn_text(conn, session_id, since=str(turn.get("started_at") or turn["created_at"]))
+ except Exception:
+ logger.debug("resumed_turn_text: lookup failed", exc_info=True)
+ return None
+ return found[0] if found else None
+
+
def mirror_harness_inbound(
context: MessageContext,
text: str,
diff --git a/core/processing_indicator.py b/core/processing_indicator.py
index 1ac50b9f0..d65e2d641 100644
--- a/core/processing_indicator.py
+++ b/core/processing_indicator.py
@@ -921,6 +921,18 @@ class ProcessingIndicatorService:
if getattr(request, "processing_indicator", None) is None:
request.processing_indicator = handle
+ def _stale_running_reaction_target(self, handle: ProcessingIndicatorHandle) -> Optional[str]:
+ """The message a restart rerun's dead process may have left its 👀 on, unless the
+ handle owns that reaction itself (its own end retires it then)."""
+ if (handle.context.platform_specific or {}).get("vibey_resume") is not True:
+ return None
+ if not self._capabilities(handle.context).supports_reaction_indicator:
+ return None
+ target = self._reaction_target_message_id(handle.context)
+ if not target or (target == handle.ack_reaction_message_id and handle.ack_reaction_emoji == ACK_REACTION_EMOJI):
+ return None
+ return target
+
async def finish(
self,
request_or_handle: Any,
@@ -973,6 +985,18 @@ class ProcessingIndicatorService:
if request is not None:
request.typing_indicator_active = False
+ stale_target = self._stale_running_reaction_target(handle)
+ if stale_target:
+ # A Turn rerun after a restart answers a message that may still wear the 👀 its
+ # dead process added; this Turn's end retires it too, whatever its own
+ # indicator. There may be none, so a failed removal decides nothing below.
+ try:
+ await self._get_im_client(handle.context).remove_reaction(
+ handle.context, stale_target, ACK_REACTION_EMOJI
+ )
+ except Exception as err:
+ logger.debug("Failed to remove a rerun's stale running reaction: %s", err)
+
if terminal_emoji and not handle.ack_reaction_emoji:
# The receipt is not a *reaction* feature, it is the only trace a
# silent ending leaves. ack_mode defaults to 'typing' on every
diff --git a/core/restart_resume.py b/core/restart_resume.py
new file mode 100644
index 000000000..5a24f3d80
--- /dev/null
+++ b/core/restart_resume.py
@@ -0,0 +1,130 @@
+"""How startup recovery settles a Vibey Turn an Avibe restart cut.
+
+The Turn reruns in place (``docs/plans/vibey-restart-auto-resume.md``): it keeps its
+identity, accepted start, Deliveries, and Runs. This module only decides; the Turn
+owner (``core.session_turns``) writes. Every read is bounded by the Turn or by the
+Session's context tail, since recovery runs it on the controller loop.
+"""
+
+from __future__ import annotations
+
+import secrets
+from typing import Any, Literal
+
+from sqlalchemy import select
+from sqlalchemy.engine import Connection
+
+from storage import agent_transcript, message_deliveries as deliveries
+from storage.background import normalize_run_status
+from storage.models import agent_runs, agent_sessions
+
+#: The restart that would be the third in a row to cut a Turn with no committed model
+#: response in between falls back to the restart notice.
+RESTART_RESUME_LIMIT = 3
+#: This process: a restart is counted once, however often its recovery retries.
+PROCESS_TOKEN = secrets.token_hex(8)
+_PENDING_CONTROL_STATES = frozenset({"pending", "interrupting", "waiting_terminal", "reconciling"})
+
+Plan = Literal["wait", "rerun", "unavailable"]
+
+
+def restart_streak(conn: Connection, turn: dict[str, Any]) -> tuple[int, int | None]:
+ """The restarts in a row that have cut ``turn``, this one included, and the response mark.
+
+ The mark is the context sequence of the Session's last committed model response.
+ A response committed since the previous restart resets the streak to this one.
+ """
+ previous = deliveries.restart_receipt(turn)
+ streak = int(previous.get("restart_streak") or 0)
+ if streak and previous.get("restart_process") == PROCESS_TOKEN:
+ # This process already counted this restart (its recovery is being retried).
+ return streak, previous.get("restart_response_seq")
+ responded = agent_transcript.last_response(conn, str(turn["session_id"]))
+ response_seq = responded.context_seq if responded is not None else None
+ if streak and previous.get("restart_response_seq") != response_seq:
+ streak = 0
+ return streak + 1, response_seq
+
+
+def _turn_runs(conn: Connection, turn_id: str) -> list[tuple[str, bool, bool]]:
+ """``(run_id, live, cancel_requested)`` for each Run the Turn's accepted Deliveries own."""
+ run_ids = deliveries.accepted_agent_run_ids_for_turn(conn, turn_id)
+ if not run_ids:
+ return []
+ rows = conn.execute(
+ select(agent_runs.c.id, agent_runs.c.status, agent_runs.c.cancel_requested).where(
+ agent_runs.c.id.in_(run_ids)
+ )
+ ).mappings()
+ return [
+ (str(row["id"]), normalize_run_status(row["status"]) in {"queued", "running"}, bool(row["cancel_requested"]))
+ for row in rows
+ ]
+
+
+def stop_requested(conn: Connection, turn: dict[str, Any]) -> bool:
+ """Whether a stop was asked of ``turn`` and is not yet settled: a Stop or replacement
+ control, or a cancel of one of its live Runs.
+
+ The one rule for a Turn a restart cut: it never reruns, and T2 ends its surviving
+ command, as the stop would have, instead of handing it to a Watch.
+ """
+ if turn.get("control_mode") or turn.get("control_state") in _PENDING_CONTROL_STATES:
+ return True
+ return any(live and cancel for _run_id, live, cancel in _turn_runs(conn, str(turn["id"])))
+
+
+def _initial_run_ended(conn: Connection, turn_id: str) -> bool:
+ """Whether a Run the Turn started for has ended.
+
+ The runner refuses to dispatch a Turn whose initial Run already ended
+ (``_turn_has_terminal_run``), so such a Turn keeps the notice instead.
+ """
+ runs = _turn_runs(conn, turn_id)
+ if not runs:
+ return False
+ initial = {
+ run_id
+ for delivery in deliveries.initial_deliveries_for_turn(conn, turn_id)
+ for run_id in deliveries.agent_run_ids_for_delivery(conn, delivery)
+ }
+ return any(not live and run_id in initial for run_id, live, _cancel in runs)
+
+
+def restart_resume_plan(conn: Connection, turn: dict[str, Any]) -> Plan:
+ """How recovery settles an accepted Vibey Turn a restart cut.
+
+ ``"unavailable"``: it keeps the restart notice (start not accepted, Stop or
+ replacement requested, a Run cancelled or already ended, Session archived or
+ system, Agent off Vibey, or the loop bound reached). Decided first: a steer still
+ unsettled then settles against the ended Turn, as before a rerun existed.
+ ``"wait"``: a Turn that would rerun has a steer still unsettled (its receipt could
+ not be materialized yet); it stays open and recovery retries, so the steer joins
+ the Turn before it reruns.
+ ``"rerun"``: the Turn reruns in place from its committed transcript. One whose
+ final response was committed ends there, as its live run would have.
+ """
+ if (
+ turn.get("backend") != "vibey"
+ or turn.get("state") != "active"
+ or turn.get("start_receipt_outcome") != "accepted"
+ ):
+ return "unavailable"
+ session = conn.execute(
+ select(agent_sessions.c.status, agent_sessions.c.visibility, agent_sessions.c.agent_backend).where(
+ agent_sessions.c.id == turn["session_id"]
+ )
+ ).mappings().one_or_none()
+ if (
+ session is None
+ or session["status"] != "active"
+ or session["visibility"] == "system"
+ or session["agent_backend"] != "vibey"
+ or stop_requested(conn, turn)
+ or _initial_run_ended(conn, str(turn["id"]))
+ or restart_streak(conn, turn)[0] >= RESTART_RESUME_LIMIT
+ ):
+ return "unavailable"
+ if deliveries.turn_has_unresolved_steer(conn, str(turn["id"])):
+ return "wait"
+ return "rerun"
diff --git a/core/session_turns.py b/core/session_turns.py
index 57d76b003..3acad4224 100644
--- a/core/session_turns.py
+++ b/core/session_turns.py
@@ -16,11 +16,12 @@ from __future__ import annotations
import asyncio
import inspect
+import itertools
import json
import logging
import uuid
import weakref
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone
from functools import partial
@@ -68,7 +69,7 @@ from core.services.agent_steering import (
)
from storage import loop_writers, messages_service
from storage import message_deliveries as delivery_store
-from storage.agent_transcript import recovered_watch_ids
+from storage.agent_transcript import last_response, recovered_watch_ids
from storage.agent_session_rows import reserve_write_lock
from storage.db import get_cached_sqlite_engine
from storage.background import (
@@ -728,6 +729,12 @@ class SessionTurnManager:
# enough that a hard outage does not spin; a class attribute so tests can
# shrink it instead of sleeping.
LOST_TURN_RETRY_DELAYS: tuple[float, ...] = (5.0, 30.0, 120.0)
+ # Backoff for a cut Vibey Turn startup had to leave open (a steer receipt still to
+ # be materialized, or a running command no Watch could take yet before its
+ # notice), retried like Vibey's own recovery owner retries a Session, then every
+ # period.
+ RESTART_RECOVERY_RETRY_DELAYS: tuple[float, ...] = (5.0, 30.0, 120.0)
+ RESTART_RECOVERY_RETRY_PERIOD: float = 600.0
def __init__(
self,
@@ -752,6 +759,14 @@ class SessionTurnManager:
self._pending_lost_turn_reports: dict[str, list[tuple[str, str, str, str, str]]] = {}
# One in-flight retry task per platform for the reports above.
self._lost_turn_retry_tasks: dict[str, asyncio.Task[None]] = {}
+ # Cut Vibey Turns whose rerun waits for their platform to deliver, held like the
+ # reports above: ``(session_id, turn_id)``. The Turn stays open, so another
+ # restart reruns it whatever this queue held.
+ self._reruns_awaiting_transport: dict[str, list[tuple[str, str]]] = {}
+ # One retry per Session whose cut Vibey Turn startup had to leave open, and what wakes
+ # it before its next pass: that Turn settling (a Stop has no runtime to reach then).
+ self._restart_recovery_retries: dict[str, asyncio.Task[None]] = {}
+ self._restart_recovery_wakes: dict[str, tuple[asyncio.AbstractEventLoop, asyncio.Event]] = {}
# The live turn sink per TURN SINK KEY. Each is
# ``{on_chunk, done_event, turn_token}`` — the turn's stream callback +
# completion event + correlation token. Every dispatched turn registers one,
@@ -4659,46 +4674,7 @@ class SessionTurnManager:
await self._resume_post_terminal(str(turn["session_id"]))
return False
try:
- delivery_payload = self._hydrate_delivery_batch_context(resolved, deliveries)
- resolved.platform_specific["turn_token"] = turn_id
- resolved.platform_specific["delivery_start_attempt_id"] = attempt_id
- metadata = delivery_payload.get("metadata") or {}
- provenance = delivery_store.scheduled_delivery_provenance(delivery_payload)
- source = SOURCE_HUMAN
- if isinstance(provenance, dict):
- source = SOURCE_SCHEDULED
- native_message_id = str(provenance.get("message_id") or "").strip()
- resolved.message_id = native_message_id or str(delivery["id"])
- preserved = provenance.get("platform_specific")
- if isinstance(preserved, dict):
- if resolved.platform_specific is None:
- resolved.platform_specific = {}
- resolved.platform_specific.update(
- {
- key: value
- for key, value in preserved.items()
- if key not in _EXECUTION_ROUTING_KEYS
- }
- )
- else:
- native_message_id = str(
- delivery_payload.get("native_message_id") or ""
- ).strip()
- resolved.message_id = (
- native_message_id
- if resolved.platform != "avibe" and native_message_id
- else str(delivery["id"])
- )
- text = str(turn.get("dispatch_text") or "")
- if source == SOURCE_HUMAN:
- stored = "\n".join(str(row.get("dispatch_text") or "") for row in deliveries if row.get("dispatch_text"))
- if stored and text.endswith(stored):
- text = text[: -len(stored)] + _segment_dispatch_text(deliveries)
- if source == SOURCE_SCHEDULED:
- # Keep the raw prompt until MessageHandler parses an explicit
- # ``subagent: prompt`` prefix. The handler prepares structured
- # metadata; only the native write renders it as text.
- self._restore_scheduled_dispatch_context(resolved, delivery)
+ source, text = self._prepare_turn_dispatch(resolved, turn, deliveries)
except Exception:
logger.exception(
"durable native start failed during pre-dispatch preparation for Turn=%s",
@@ -4743,6 +4719,159 @@ class SessionTurnManager:
)
return False
+ def _prepare_turn_dispatch(
+ self,
+ resolved: "MessageContext",
+ turn: dict[str, Any],
+ deliveries: list[dict[str, Any]],
+ ) -> tuple[str, str]:
+ """Hydrate a durable Turn's routing context from its initial Deliveries: its source and text."""
+
+ delivery = deliveries[0]
+ delivery_payload = self._hydrate_delivery_batch_context(resolved, deliveries)
+ resolved.platform_specific["turn_token"] = str(turn["id"])
+ resolved.platform_specific["delivery_start_attempt_id"] = str(turn["start_attempt_id"])
+ provenance = delivery_store.scheduled_delivery_provenance(delivery_payload)
+ source = SOURCE_HUMAN
+ if isinstance(provenance, dict):
+ source = SOURCE_SCHEDULED
+ native_message_id = str(provenance.get("message_id") or "").strip()
+ resolved.message_id = native_message_id or str(delivery["id"])
+ preserved = provenance.get("platform_specific")
+ if isinstance(preserved, dict):
+ if resolved.platform_specific is None:
+ resolved.platform_specific = {}
+ resolved.platform_specific.update(
+ {
+ key: value
+ for key, value in preserved.items()
+ if key not in _EXECUTION_ROUTING_KEYS
+ }
+ )
+ else:
+ native_message_id = str(
+ delivery_payload.get("native_message_id") or ""
+ ).strip()
+ resolved.message_id = (
+ native_message_id
+ if resolved.platform != "avibe" and native_message_id
+ else str(delivery["id"])
+ )
+ text = str(turn.get("dispatch_text") or "")
+ if source == SOURCE_HUMAN:
+ stored = "\n".join(str(row.get("dispatch_text") or "") for row in deliveries if row.get("dispatch_text"))
+ if stored and text.endswith(stored):
+ text = text[: -len(stored)] + _segment_dispatch_text(deliveries)
+ if source == SOURCE_SCHEDULED:
+ # Keep the raw prompt until MessageHandler parses an explicit
+ # ``subagent: prompt`` prefix. The handler prepares structured
+ # metadata; only the native write renders it as text.
+ self._restore_scheduled_dispatch_context(resolved, delivery)
+ return source, text
+
+ async def _rerun_turn(self, turn_id: str) -> bool:
+ """Rerun an accepted Vibey Turn in place, as a transcript resume of itself.
+
+ The ordinary runner holds it, so Stop, steers, and its terminal result address
+ this Turn as they would any live one (``docs/plans/vibey-restart-auto-resume.md``
+ item 2). Its input was consumed before the restart, so the rerun carries none.
+ """
+
+ with self._sqlite_engine().connect() as conn:
+ turn = delivery_store.get_turn(conn, turn_id)
+ deliveries = delivery_store.initial_deliveries_for_turn(conn, turn_id) if turn is not None else []
+ if turn is None or turn["state"] != "active" or not deliveries:
+ return False
+ session_id = str(turn["session_id"])
+ projected = self.in_flight.get(session_id)
+ if projected is not None and not projected.task.done():
+ return projected.logical_turn_id == turn_id
+ resolved = self._delivery_context(session_id)
+ lifecycle_snapshot = self.snapshot_session_lifecycle(
+ self._lifecycle_anchor_for_delivery(deliveries[0], session_id)
+ )
+ source, _text = self._prepare_turn_dispatch(resolved, turn, deliveries)
+ resolved.files = []
+ resolved.platform_specific.update(
+ {
+ "vibey_resume": True,
+ "vibey_source_turn_id": turn_id,
+ "display_text": "",
+ "display_texts": [],
+ "message_content": {},
+ }
+ )
+ self._append_accepted_agent_run_ids(
+ resolved.platform_specific, self.accepted_agent_run_ids_for_turn(turn_id)
+ )
+ execution = delivery_store.recorded_execution(turn)
+ known, agent = self._turn_vibe_agent(session_id, turn_id, execution)
+ if known is not None and agent is None:
+ # The Agent that ran the Turn no longer runs on Vibey, or nothing names it: no
+ # other Agent continues its work. It takes the restart notice instead.
+ return False
+ if agent is not None:
+ # The Turn keeps the Agent and request it was accepted with, as a live Turn
+ # does when the Session's Agent or its settings change: they reach the next Turn.
+ resolved.platform_specific.update({"vibe_agent_name": agent.name, "vibe_agent_id": agent.id})
+ if execution:
+ resolved.platform_specific["vibey_execution"] = execution
+ await self._run(
+ session_id,
+ resolved,
+ "",
+ source=source,
+ logical_turn_id=turn_id,
+ delivery_id=str(deliveries[0]["id"]),
+ durable_preallocated=True,
+ lifecycle_snapshot=lifecycle_snapshot if source == SOURCE_HUMAN else None,
+ )
+ projected = self.in_flight.get(session_id)
+ if projected is not None and projected.logical_turn_id == turn_id:
+ projected.task.add_done_callback(partial(self._after_rerun, session_id))
+ return True
+
+ def _after_rerun(self, session_id: str, task: "asyncio.Task[None]") -> None:
+ """A rerun may end before its adapter owns the command the restart left running:
+ cancelled (a Stop, the backend disabled or refreshed) or failed before dispatch. Vibey's
+ T2 then ends that command with a canceled Turn, or hands it to a Watch as a failed run's
+ is, rather than leaving it running unowned. After the adapter ran it finds nothing; a
+ shutdown leaves the command for the next start's rerun."""
+
+ if getattr(self.controller, "shutdown_requested", False) is True:
+ return
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ return
+ loop.create_task(self._settle_vibey_session(session_id), name=f"vibey-rerun-ended:{session_id}")
+
+ def _turn_vibe_agent(self, session_id: str, turn_id: str, execution: dict[str, Any]) -> tuple[Optional[bool], Any]:
+ """``(known, agent)``: the Agent that ran a Turn, while it still runs on Vibey.
+
+ Its native start records it with the rest of its request (``execution``); a Turn
+ an older release accepted names it only as the author of its committed responses,
+ and one cut before any names none (``known`` False). ``known`` is None when no
+ Agent store can tell.
+ """
+
+ agent_id = str(execution.get("agent_id") or "") or None
+ name = str(execution.get("agent_name") or "") or None
+ if not agent_id and not name:
+ with self._sqlite_engine().connect() as conn:
+ response = last_response(conn, session_id)
+ name = response.agent_name if response is not None and response.turn_id == turn_id else None
+ store = getattr(self.controller, "vibe_agent_store", None)
+ if store is None:
+ return None, None
+ if not (agent_id or name):
+ return False, None
+ # By its stable id, which survives a rename; the name only where no id was recorded.
+ agent = store.get_by_id(agent_id) if agent_id else store.get(name)
+ if agent is None or agent.backend != "vibey" or not agent.enabled or agent.archived_at:
+ return True, None
+ return True, agent
+
def begin_computer_use_request(
self,
session_id: str,
@@ -5408,6 +5537,12 @@ class SessionTurnManager:
if requeued_input:
self._publish_queue_update(session_id)
self._publish_terminal_inbox_update(session_id)
+ waiting = self._restart_recovery_wakes.get(session_id)
+ if waiting is not None:
+ # A cut Turn open for a restart retry has no runtime for a Stop to reach: its
+ # retry runs now, so T2 ends the command of a canceled Turn as live.
+ with suppress(RuntimeError): # its loop has closed
+ waiting[0].call_soon_threadsafe(waiting[1].set)
if status_changed and projected_status is not None:
from core.inbox_events import bus
@@ -5485,14 +5620,37 @@ class SessionTurnManager:
outcome: str,
failure_evidence: dict[str, Any] | None = None,
) -> dict[str, Any]:
- """Settle definitive no-write evidence at the terminal chokepoint."""
- return self._terminalize_durable_turn(
+ """Settle definitive no-write evidence at the terminal chokepoint.
+
+ A Vibey Turn rerun after a restart had its start accepted before, and nothing
+ else runs it, so an exit before its rerun dispatched fails it instead. Any other
+ accepted Turn ignores the late claim, as before.
+ """
+ result = self._terminalize_durable_turn(
turn_id,
"not_written",
settled_by=outcome,
evidence_kind="definitive_prewrite_failure",
evidence={"reason": outcome, **(failure_evidence or {})},
)
+ if result.get("changed"):
+ return result
+ with self._sqlite_engine().connect() as conn:
+ turn = delivery_store.get_turn(conn, turn_id)
+ if (
+ turn is None
+ or turn["state"] != "active"
+ or turn.get("backend") != "vibey"
+ or not delivery_store.restart_receipt(turn)
+ ):
+ return result
+ return self._terminalize_durable_turn(
+ turn_id,
+ "failed",
+ settled_by=outcome,
+ evidence_kind="rerun_failed_before_dispatch",
+ evidence={"reason": outcome, **(failure_evidence or {})},
+ )
def _reconcile_durable_runner_release(
self,
@@ -5919,6 +6077,8 @@ class SessionTurnManager:
return
if not self._durable_schema_available():
return
+ execution = (getattr(context, "platform_specific", None) or {}).get("vibey_execution")
+ execution = execution if backend == "vibey" and isinstance(execution, dict) else None
try:
identity = self._active_identity(backend, session_id, logical_turn_id)
native_turn_id = identity[1] if identity and identity[0] == logical_turn_id else None
@@ -5934,6 +6094,8 @@ class SessionTurnManager:
runtime_key=runtime_key,
runtime_turn_id=runtime_turn_id,
native_turn_id=native_turn_id,
+ # The request a Vibey Turn runs with, so a rerun after a restart keeps it.
+ execution=execution,
)
materialized: list[dict[str, Any]] = []
if bound is not None:
@@ -6799,6 +6961,74 @@ class SessionTurnManager:
# ready callback is coming to flush it — retry on our own clock.
self._schedule_lost_turn_retry(platform)
+ async def _settle_vibey_session(self, session_id: str) -> bool:
+ """Vibey's T2 for one Session, through its single recovery owner: whether it settled."""
+
+ recovery = getattr(self.controller, "vibey_recovery", None)
+ settle = getattr(recovery, "settle_session", None)
+ if not callable(settle):
+ return True
+ try:
+ return bool(await settle(session_id))
+ except Exception:
+ logger.warning("Vibey settlement failed for session=%s", session_id, exc_info=True)
+ return False
+
+ def _retry_restart_recovery(self, session_id: str, turn_id: str) -> None:
+ """Recover a cut Vibey Turn again while startup had to leave it open.
+
+ It waits on a steer receipt still to be materialized, or on T2 handing its
+ running command to a Watch before its notice. Each pass is the ordinary
+ recovery of that Session; the retry ends once the Turn reruns or settles, and
+ survives a failed pass, since it is the Turn's only one. A Stop that settles
+ the waiting Turn has T2 end its command, as the Stop would have live.
+ """
+
+ existing = self._restart_recovery_retries.get(session_id)
+ if existing is not None and not existing.done():
+ return
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ return
+
+ wake = asyncio.Event()
+
+ async def retry() -> None:
+ try:
+ for attempt in itertools.count():
+ delays = self.RESTART_RECOVERY_RETRY_DELAYS
+ try:
+ await asyncio.wait_for(
+ wake.wait(),
+ delays[attempt] if attempt < len(delays) else self.RESTART_RECOVERY_RETRY_PERIOD,
+ )
+ except asyncio.TimeoutError:
+ pass
+ wake.clear()
+ try:
+ with self._sqlite_engine().connect() as conn:
+ turn = delivery_store.get_turn(conn, turn_id)
+ if turn is None or turn["state"] not in delivery_store.TURN_OWNER_STATES:
+ if turn is not None and turn.get("terminal_outcome") == "canceled":
+ await self._settle_vibey_session(session_id)
+ return
+ projected = self.in_flight.get(session_id)
+ if projected is not None and not projected.task.done():
+ return # it reruns: its own runner owns it now
+ await self.recover_durable_delivery_state(session_id, service_restart=True)
+ except Exception:
+ logger.exception("restart recovery retry failed for Session=%s", session_id)
+ finally:
+ if self._restart_recovery_retries.get(session_id) is asyncio.current_task():
+ self._restart_recovery_retries.pop(session_id, None)
+ self._restart_recovery_wakes.pop(session_id, None)
+
+ self._restart_recovery_wakes[session_id] = (loop, wake)
+ self._restart_recovery_retries[session_id] = loop.create_task(
+ retry(), name=f"vibey-restart-recovery:{session_id}"
+ )
+
def _transport_can_deliver(self, platform: str) -> bool:
"""Whether ``platform`` can deliver right now.
@@ -6818,15 +7048,23 @@ class SessionTurnManager:
return True
async def notify_transport_ready(self, platform: str) -> int:
- """Flush the interruption reports held for one platform.
+ """Start the restart reruns and flush the interruption reports held for one platform.
- Held in memory only: the turn is already terminal, so a report that never
+ A held rerun's Turn is still open, so it needs no other record. Reports
+ are held in memory only: the turn is already terminal, so a report that never
drains (transport disabled before it connects) is dropped rather than
replayed on the next start, where it would be stale news. A send that
fails against a connected transport is retained AND retried on a bounded
backoff — see ``_schedule_lost_turn_retry`` for why nothing else would.
"""
+ for session_id, turn_id in self._reruns_awaiting_transport.pop(platform, []):
+ # A rerun held for this transport: the ordinary recovery of its Session starts it.
+ try:
+ await self.recover_durable_delivery_state(session_id, service_restart=True)
+ except Exception:
+ logger.exception("held restart rerun failed to start for Session=%s", session_id)
+ self._retry_restart_recovery(session_id, turn_id)
pending = self._pending_lost_turn_reports.pop(platform, [])
if not pending or self.controller is None:
return 0
@@ -6997,6 +7235,166 @@ class SessionTurnManager:
)
return True
+ async def _recover_steer_attempts(
+ self,
+ session_id: str | None,
+ *,
+ target_turn_id: str | None = None,
+ ) -> list[str]:
+ """Settle steer attempts a dead process left unresolved; the Sessions it changed.
+
+ ``target_turn_id`` limits it to one Turn's attempts: a cut Vibey Turn reconciles
+ its own before it reruns, so a steer the dead run accepted joins the Turn, and
+ its Run settles with the rest of the Turn's work.
+ """
+
+ recovered: list[str] = []
+ with self._sqlite_engine().connect() as conn:
+ unresolved = delivery_store.unresolved_deliveries(conn, session_id)
+ accepted_steer_attempts: list[dict[str, Any]] = []
+ steer_attempts: list[tuple[dict[str, Any], str]] = []
+ seen_attempt_ids: set[str] = set()
+ for attempt in unresolved:
+ if attempt["state"] not in {"steering", "reconciling_steer"}:
+ continue
+ attempt_id = str(attempt.get("current_attempt_id") or "")
+ attempt_target = str(attempt.get("current_target_turn_id") or "")
+ expected_native_id = str(
+ attempt.get("current_expected_native_turn_id") or ""
+ )
+ if (
+ not attempt_id
+ or attempt_id in seen_attempt_ids
+ or not attempt_target
+ or (target_turn_id is not None and attempt_target != target_turn_id)
+ ):
+ continue
+ target_turn = delivery_store.get_turn(conn, attempt_target)
+ if target_turn is None:
+ continue
+ seen_attempt_ids.add(attempt_id)
+ if (
+ attempt["state"] == "reconciling_steer"
+ and attempt.get("current_receipt_outcome") == "accepted"
+ ):
+ accepted_steer_attempts.append(attempt)
+ continue
+ if not expected_native_id:
+ continue
+ steer_attempts.append((attempt, str(target_turn["backend"])))
+
+ for attempt in accepted_steer_attempts:
+ result = await self._finish_steer(
+ str(attempt["id"]),
+ _accepted_steer_receipt(attempt),
+ context=None,
+ )
+ if result.state != "reconciling_steer":
+ recovered.append(str(attempt["session_id"]))
+
+ for attempt, backend in steer_attempts:
+ attempt_id = str(attempt["current_attempt_id"])
+ receipt = await self._reconcile_steer_attempt(
+ backend,
+ SteerReconcileRequest(
+ target_session_id=str(attempt["session_id"]),
+ expected_logical_turn_id=str(attempt["current_target_turn_id"]),
+ expected_native_turn_id=str(
+ attempt["current_expected_native_turn_id"]
+ ),
+ attempt_id=attempt_id,
+ ),
+ )
+ outcome = getattr(receipt, "outcome", SteerOutcome.UNKNOWN)
+ if (
+ str(getattr(outcome, "value", outcome)) == SteerOutcome.UNKNOWN.value
+ and attempt["state"] == "reconciling_steer"
+ ):
+ continue
+ result = await self._finish_steer(
+ str(attempt["id"]),
+ receipt,
+ context=None,
+ expected_attempt_id=attempt_id,
+ )
+ if result.state != "reconciling_steer":
+ recovered.append(str(attempt["session_id"]))
+ return recovered
+
+ async def _recover_restart_cut_vibey_turn(self, turn_id: str, attempt_id: str) -> str:
+ """Settle a Vibey Turn a restart cut: ``rerun``, ``held``, ``wait``, or ``unavailable``.
+
+ T2 has already given every open call a result. An eligible Turn reruns in place,
+ with its restart streak written in the same transaction that releases the dead
+ runtime generation (``docs/plans/vibey-restart-auto-resume.md``); one whose final
+ response was committed ends there, through its live end. The rerun delivers to
+ the Turn's surface, so it is ``held`` until that surface can, as the restart
+ notice is. A Turn with a steer still unsettled stays open for a retry; anything
+ else keeps today's notice.
+ """
+
+ from core.restart_resume import restart_resume_plan
+
+ with self._sqlite_engine().connect() as conn:
+ turn = delivery_store.get_turn(conn, turn_id)
+ if turn is None or str(turn.get("start_attempt_id") or "") != attempt_id:
+ return "unavailable"
+ plan = restart_resume_plan(conn, turn)
+ if plan != "rerun":
+ return plan
+ session_id = str(turn["session_id"])
+ try:
+ platform = str(getattr(self._delivery_context(session_id), "platform", "") or "")
+ except Exception:
+ # Setting up the rerun failed, not the Turn: it stays open and recovery retries.
+ logger.exception("restart rerun could not resolve Turn=%s's surface; retrying", turn_id)
+ return "wait"
+ if not self._transport_can_deliver(platform):
+ held = self._reruns_awaiting_transport.setdefault(platform, [])
+ if (session_id, turn_id) not in held:
+ held.append((session_id, turn_id))
+ return "held"
+ if not await asyncio.to_thread(self._rotate_for_rerun, turn_id, attempt_id, int(turn["version"])):
+ return "wait" # it changed meanwhile: the retry decides again
+ try:
+ started = await self._rerun_turn(turn_id)
+ except Exception:
+ # Setting up the rerun failed, not the Turn: it stays open and recovery retries.
+ logger.exception("restart rerun could not start Turn=%s; retrying", turn_id)
+ return "wait"
+ return "rerun" if started else "unavailable"
+
+ def _rotate_for_rerun(self, turn_id: str, attempt_id: str, decided_version: int) -> bool:
+ """The rerun's writer transaction, off the controller loop: the restart streak, and
+ the dead runtime generation released, while the Turn is still as the decision read
+ it. A concurrent recovery pass that rotated it first (and whose rerun may have bound
+ its own generation since) moved its version, so this one releases nothing."""
+
+ from core.restart_resume import PROCESS_TOKEN, restart_resume_plan, restart_streak
+
+ with self._sqlite_engine().begin() as conn:
+ reserve_write_lock(conn)
+ turn = delivery_store.get_turn(conn, turn_id)
+ if (
+ turn is None
+ or int(turn["version"]) != decided_version
+ or str(turn.get("start_attempt_id") or "") != attempt_id
+ or restart_resume_plan(conn, turn) != "rerun"
+ ):
+ return False
+ streak, response_seq = restart_streak(conn, turn)
+ return (
+ delivery_store.rotate_restart_generation(
+ conn,
+ turn_id,
+ expected_version=int(turn["version"]),
+ streak=streak,
+ response_seq=response_seq,
+ process=PROCESS_TOKEN,
+ )
+ is not None
+ )
+
async def recover_durable_delivery_state(
self,
session_id: str | None = None,
@@ -7165,6 +7563,32 @@ class SessionTurnManager:
)
for target_session, turn_id, attempt_id, backend in lost_active_turns:
+ if backend == "vibey":
+ projected = self.in_flight.get(target_session)
+ if projected is not None and projected.logical_turn_id == turn_id and not projected.task.done():
+ continue # already rerunning in this process
+ # Vibey continues its own work: a steer the dead run accepted joins the
+ # Turn first; only a Turn that no longer qualifies takes the notice below.
+ recovered.extend(
+ await self._recover_steer_attempts(target_session, target_turn_id=turn_id)
+ )
+ outcome = await self._recover_restart_cut_vibey_turn(turn_id, attempt_id)
+ if outcome in ("wait", "held"):
+ # A steer the dead run may have accepted is not settled yet, so it joins
+ # the Turn before the Turn reruns; or its surface cannot deliver yet, and
+ # its transport's ready callback starts it. The retry keeps the Turn owned.
+ self._retry_restart_recovery(target_session, turn_id)
+ continue
+ if outcome != "unavailable":
+ recovered.append(target_session)
+ continue
+ if backend == "vibey" and not await self._settle_vibey_session(target_session):
+ # Startup recovery left a running job of this Turn to its owner, and no Watch
+ # could take it yet. T2 before T4: the Turn stays open until it is handed over,
+ # so its notice names the Watch and its Runs are not failed under a running
+ # command. A Stop still settles it.
+ self._retry_restart_recovery(target_session, turn_id)
+ continue
# Read Run attribution BEFORE terminalizing: the notice below is owed
# only to a turn that has none, and terminalization retires the
# deliveries the attribution is derived from.
@@ -7210,76 +7634,7 @@ class SessionTurnManager:
if retry_turn_id:
await self._start_persisted_turn(retry_turn_id)
- with self._sqlite_engine().connect() as conn:
- unresolved = delivery_store.unresolved_deliveries(conn, session_id)
- accepted_steer_attempts: list[dict[str, Any]] = []
- steer_attempts: list[tuple[dict[str, Any], str]] = []
- seen_attempt_ids: set[str] = set()
- for attempt in unresolved:
- if attempt["state"] not in {"steering", "reconciling_steer"}:
- continue
- attempt_id = str(attempt.get("current_attempt_id") or "")
- target_turn_id = str(attempt.get("current_target_turn_id") or "")
- expected_native_id = str(
- attempt.get("current_expected_native_turn_id") or ""
- )
- if (
- not attempt_id
- or attempt_id in seen_attempt_ids
- or not target_turn_id
- ):
- continue
- target_turn = delivery_store.get_turn(conn, target_turn_id)
- if target_turn is None:
- continue
- seen_attempt_ids.add(attempt_id)
- if (
- attempt["state"] == "reconciling_steer"
- and attempt.get("current_receipt_outcome") == "accepted"
- ):
- accepted_steer_attempts.append(attempt)
- continue
- if not expected_native_id:
- continue
- steer_attempts.append((attempt, str(target_turn["backend"])))
-
- for attempt in accepted_steer_attempts:
- result = await self._finish_steer(
- str(attempt["id"]),
- _accepted_steer_receipt(attempt),
- context=None,
- )
- if result.state != "reconciling_steer":
- recovered.append(str(attempt["session_id"]))
-
- for attempt, backend in steer_attempts:
- attempt_id = str(attempt["current_attempt_id"])
- target_turn_id = str(attempt["current_target_turn_id"])
- receipt = await self._reconcile_steer_attempt(
- backend,
- SteerReconcileRequest(
- target_session_id=str(attempt["session_id"]),
- expected_logical_turn_id=target_turn_id,
- expected_native_turn_id=str(
- attempt["current_expected_native_turn_id"]
- ),
- attempt_id=attempt_id,
- ),
- )
- outcome = getattr(receipt, "outcome", SteerOutcome.UNKNOWN)
- if (
- str(getattr(outcome, "value", outcome)) == SteerOutcome.UNKNOWN.value
- and attempt["state"] == "reconciling_steer"
- ):
- continue
- result = await self._finish_steer(
- str(attempt["id"]),
- receipt,
- context=None,
- expected_attempt_id=attempt_id,
- )
- if result.state != "reconciling_steer":
- recovered.append(str(attempt["session_id"]))
+ recovered.extend(await self._recover_steer_attempts(session_id))
for target_session, turn_id in pending_interrupts:
await self._run_pending_interrupt(target_session, turn_id)
@@ -8476,7 +8831,12 @@ class SessionTurnManager:
return str((row or {}).get("native_turn_id") or "").strip() or None
async def release_for_service_shutdown(self) -> int:
- """Fail exact accepted Run owners without draining replacement work."""
+ """Fail exact accepted Run owners without draining replacement work.
+
+ A Vibey Turn nobody asked to stop is left as it is, Runs included: startup
+ recovery continues it from its committed transcript and its Runs settle
+ from the continued work (``docs/plans/vibey-restart-auto-resume.md``).
+ """
if not self._durable_schema_available():
return 0
@@ -8487,6 +8847,7 @@ class SessionTurnManager:
select(
session_turn_rows.c.id.label("turn_id"),
session_turn_rows.c.session_id,
+ session_turn_rows.c.backend,
session_turn_rows.c.control_mode,
agent_runs.c.cancel_requested,
)
@@ -8518,6 +8879,7 @@ class SessionTurnManager:
{
"turn_id": turn_id,
"session_id": str(row["session_id"]),
+ "backend": str(row["backend"] or ""),
"control_mode": row["control_mode"],
"cancel_requested": False,
},
@@ -8554,6 +8916,8 @@ class SessionTurnManager:
or owner["cancel_requested"]
or projected.cancel_settled_by == SETTLED_BY_STOPPED
)
+ if not user_stopped and owner["backend"] == "vibey":
+ continue
settled_by = SETTLED_BY_STOPPED if user_stopped else SETTLED_BY_RESTARTED
terminal = self._terminalize_durable_turn(
turn_id,
diff --git a/docs/plans/agent-core-contracts/recovery.md b/docs/plans/agent-core-contracts/recovery.md
index 6819eb141..e57cca285 100644
--- a/docs/plans/agent-core-contracts/recovery.md
+++ b/docs/plans/agent-core-contracts/recovery.md
@@ -24,9 +24,9 @@ at-least-once, and that is stated where it applies.
| ID | Invariant | Proof |
| --- | --- | --- |
| T1 | `project()` reads only committed rows; the same rows always produce the same request. | projection before and after job state changes |
-| T2 | At resume, before the first projection, and when a run ends without completing (a Stop), every tool call without a committed `tool_result` gets exactly one, chosen from the state of the job its call instance started (exited: the output; running: handover; stopped with a recorded reason: the output so far and that reason, `tools.md` §5; never ran: the interrupted result). A call without job state (`write`, `edit`) gets `[tool call interrupted; it may or may not have completed; re-read the file before continuing]`. Retries settle nothing twice, and neither do concurrent writers: the store's `append_tool_result` returns the result already committed for the call instance instead of writing a second. | crash between settlement steps, then resume twice; a Stop while several calls of one group (C-3 §2) run |
+| T2 | At resume, before the first projection, and when a run ends without completing (a Stop), every tool call without a committed `tool_result` gets exactly one, chosen from the state of the job its call instance started (exited: the output; running: handover; stopped with a recorded reason: the output so far and that reason, `tools.md` §5; never ran: the interrupted result). The resume of a Turn rerun after a restart (T4) first waits for a running job as its live call would: until it exits, until the call's foreground window passes, counted from the job's launch (`meta.created_at`), or until a Stop, which ends the command as in a live Turn; only a job still running then is handed over, and one no Watch can take stays in the foreground until it ends, as in a live Turn. Startup's own pass leaves a running job of a Turn still open to that Turn's owner, which reruns the Turn or settles this Session itself before its notice; a job whose Turn a Stop canceled, or holds a stop never settled (a Stop or replacement control, or a cancel of one of its Runs), is ended instead of handed over. The wait commits nothing and starts nothing, so a crash during it leaves the call open for the next settlement. A call without job state (`write`, `edit`) gets `[tool call interrupted; it may or may not have completed; re-read the file before continuing]`. Retries settle nothing twice, and neither do concurrent writers: the store's `append_tool_result` returns the result already committed for the call instance instead of writing a second. | crash between settlement steps, then resume twice; a Stop while several calls of one group (C-3 §2) run; a restart rerun whose job exits inside its window, outlives it, or meets a Stop |
| T3 | Inputs accepted by `steer` or `follow_up` are never dropped: they enter the context, including after a crash. They belong to the Turn that accepted them and are never re-queued as a new P3 Turn. A run that ended by design runs again for them within its Turn; after a stop, an error, or a crash they are admitted into the context (at the end of the run, or at the Session's resume) and share the Turn's outcome. | a terminating tool while a steer is queued; a crash before consumption |
-| T4 | An unsettled Turn found at startup is settled as interrupted after T2; the agent never continues it on its own (a hook `end`, an abort, or a crash all leave the same safe state). The user's next message starts a new Turn with the full context. When T2 handed the Turn's running commands to Watches, its notice names them and offers no Retry, because a resend would run them twice. | crash after an `end` hook and after a mid-turn commit |
+| T4 | An accepted Turn found unsettled at startup is continued after T2, whatever stopped the process (`vibey-restart-auto-resume.md`): it reruns in place, once its surface can deliver, keeping its identity, accepted start, Deliveries, and Runs, as a transcript resume of itself with no notice and no replayed tool call; a Turn whose final response was committed ends with it there through its live end, with no model call (Delivery below). A steer it accepted joins it first; while one is unsettled the Turn waits open. A command still running is waited for by the rerun's own T2 within its window from launch (startup leaves it to the open Turn, across repeated restarts) and is never run again; one handed to a Watch is read as the handover result. A command whose Turn a Stop canceled while it outlived the run is ended by T2, not handed to a Watch. A Turn that no longer qualifies (Stop or replacement requested, a Run cancelled or ended, Session archived, Agent off Vibey, or the third restart in a row without a committed model response) settles as interrupted with the restart notice after T2 hands its running commands to Watches (it stays open, retried, until a Watch takes them), so that notice names them and offers no Retry, because a resend would run them twice. A run that fails while the service is stopping leaves its Turn open for this. | crash after an `end` hook and after a mid-turn commit; a restart during a tool call, during a model call, and after the final commit |
## Delivery (adapter wave)
@@ -35,8 +35,9 @@ through the same emit path as the other backends: splitting, file upload, failur
are the dispatcher's. The one difference is persistence: the response row already exists, so the dispatcher writes
its display columns instead of inserting a second row (`transcript.md` §2). A crash or send failure between commit
and send loses that delivery, as it does for the other backends; the transcript, and so the context, is unaffected.
-A Turn interrupted after its final commit is settled by T4 like any other, so the user sees the interruption
-notice, and the committed reply stays in the Workbench transcript with its commit-time display.
+A Vibey Turn interrupted after its final commit reruns (T4) and ends with that reply through this same emit path:
+a reply whose display write already recorded its delivery is not sent again, and one without it is sent then. Across
+a restart, delivery of a committed final is therefore at-least-once, for the window between a send and that write.
## Shared fields
diff --git a/docs/plans/backend-failure-retry.md b/docs/plans/backend-failure-retry.md
index 334355fb5..bcce065e3 100644
--- a/docs/plans/backend-failure-retry.md
+++ b/docs/plans/backend-failure-retry.md
@@ -9,6 +9,10 @@ adapter-local policy over the same committed transcript.
- Only an attached `notify` with `event=backend_failure`, a `failure_id`, and a
durable `turn_id` can offer the action. Ordinary notifications, detached
completions, user stops, and authentication-recovery cards are unchanged.
+- A service restart does not end an accepted Vibey Turn: startup recovery
+ reruns that same Turn in place, as the transcript resume this action uses,
+ with no Delivery and no notice (`vibey-restart-auto-resume.md`). Only a Vibey
+ Turn that no longer qualifies takes the notice below.
- A service restart that terminalizes an accepted, non-restorable conversation
Turn emits the same failure-notice contract, with the original Turn ID and
backend retained across transport readiness waits and failed-send retries.
@@ -121,6 +125,8 @@ not added here.
Scenario contracts: `MESSAGE-DELIVERY-026` through `MESSAGE-DELIVERY-028`.
Restart notification binding and continuation: `MESSAGE-DELIVERY-030`.
Harness-owned notification binding and continuation: `MESSAGE-DELIVERY-320`.
+Vibey restart rerun without a notice: `MESSAGE-DELIVERY-327` through
+`MESSAGE-DELIVERY-333`.
## Review-boundary correction contract
diff --git a/docs/plans/vibey-restart-auto-resume.md b/docs/plans/vibey-restart-auto-resume.md
new file mode 100644
index 000000000..d630bcf30
--- /dev/null
+++ b/docs/plans/vibey-restart-auto-resume.md
@@ -0,0 +1,305 @@
+# Vibey continues its own work across Avibe restarts
+
+## Background
+
+A Vibey Turn cut by an Avibe restart ends today. Recovery settles its tool calls
+(T2), settles the Turn as interrupted (T4: "the agent never continues it on its
+own"), and shows a failure notice. On the Web the notice offers Retry, which
+resumes the loop from the committed transcript through a hidden
+`agent_initiated` Delivery (`backend-failure-retry.md`). On IM, and whenever T2
+handed running commands to Watches, the user has to type a continuation or the
+work stays stopped.
+
+Between 2026-10-11 and 2026-10-18 the owner's install had 18 Vibey Turns cut
+this way: 11 Harness-owned Turns settled at shutdown (`restarted` /
+`service_shutdown`), and 7 conversation Turns found at startup
+(`no_terminal_result` / `restart_runtime_missing`). One of the latter came from
+a managed `vibe restart` (shutdown intent `restart`), so the evidence kind does
+not tell a controlled restart from a crash.
+
+## Owner decisions (2026-10-18)
+
+- Every restart is continued automatically: `vibe restart`, upgrade, stop and
+ start, Desktop quit and reopen, host reboot, and a crash alike. Nothing on
+ the startup side can tell them apart reliably; the loop bound below covers a
+ restart the Turn itself causes.
+- Harness-owned Turns (Agent Runs, scheduled Tasks, Watch follow-ups) continue
+ too, and their Runs settle from the continued work, so a caller's callback
+ reports the real result instead of a restart failure.
+- No time limit: a Turn interrupted by a long downtime continues at the next
+ start. The user can Stop it like any other Turn.
+- No failure notice and no interruption copy for a continued Turn.
+
+This replaces recovery invariant T4 for Vibey (`agent-core-contracts/recovery.md`).
+
+## Mechanism: the cut Turn reruns in place
+
+The first implementation handed a cut Turn's work to a new, hidden
+continuation Turn and settled the cut Turn. Five reviewed heads in a row
+raised findings at the seams of that handover, so the mechanism changed: the
+cut Turn keeps its identity. It stays `active` with its accepted start;
+startup recovery rotates it to this process's runtime generation and reruns
+its Vibey loop with `Agent.resume` over its own accepted inputs. No Delivery
+is created, no Run moves, and no successor Turn starts. A Turn whose final
+response was committed was at first completed from that row by recovery,
+which rebuilt the end of a live Turn piece by piece; six reviewed heads raised
+findings there, so that Turn reruns too, and its own live end finishes it
+(item 8).
+
+### Inventory of the seams raised on the successor design
+
+The acceptance list for the in-place rerun. "Removed" means the seam no longer
+exists; "handled" names where the in-place design still owns it.
+
+| Seam | In-place rerun |
+| --- | --- |
+| Runs move to the continuation (one Delivery per Run, `agent_runs.delivery_id` unique, batch claim) | Removed. Runs stay on the Turn's own Deliveries; the rerun's terminal settles them like any Turn's. |
+| Steers the dead run accepted but never materialized; a steer still unsettled at startup | Handled. Recovery reconciles the Turn's steer receipts into the same Turn before it reruns, and the rerun pins them with the Turn's inputs. While one is unsettled the Turn waits open and recovery retries; the retry only recovers again, so it never hands a command over early. |
+| FIFO tie-break; a requeued continuation overtaken by same-second input | Removed. The Turn stays active, so queued input waits behind it as behind any active Turn. |
+| Job-owner ancestry (an open call owned by a settled source; a replacement successor) | Removed. The open call's Turn is the rerunning Turn: T2 defers while it is open and ends the command once a Stop or replacement canceled it. |
+| Stop or replacement during the rerun's T2 wait (pre-write cancellation instead of `handle_stop`) | Removed. The Turn is durably active, so a Stop or replacement takes the active-Turn interrupt to `handle_stop`, which ends the wait and the command. |
+| A continuation requeued, then retired (backend change) or left unclaimed, after its source settled | Removed. There is no continuation Delivery. A rerun that fails ends its Turn through the ordinary runner, with the failure notice and Retry. |
+| A Model Hub refusal at the continuation's preflight | Handled. The rerun is the accepted Turn continuing, so the refusal fails that Turn with its notice and Retry, as a refusal at any later model call of a live Turn does; the work stays in the transcript. |
+| Another restart during the T2 wait | Handled. The Turn is still accepted and active, so the next start reruns it again, within the loop bound; its startup T2 leaves the job to it. |
+| A committed final not yet delivered to IM; one already delivered | Handled by the live end of the Turn. The rerun ends with the committed row through the ordinary result emit (item 8); the dispatcher sends a final without delivery evidence once and does not resend one with it. |
+| The orphaned 👀 after a committed final | Handled by the live end of the Turn: the rerun's indicator retires the 👀 the dead process may have left, as for any rerun. |
+| The silent-final promotion (off the controller loop; a failed write) | Handled by the live end of the Turn: the dispatcher promotes as live, reading the Turn's earlier visible text from its rows when the process that showed it died. |
+| A committed final across another restart; a Stop arriving meanwhile; the successor's start | Handled by the live end of the Turn. The Turn stays active until its rerun ends it, so another restart reruns it again (within the loop bound), a Stop in the new process loses to the final as live, and the successor starts from the ordinary terminal. |
+| A rerun started before its IM transport can deliver | Handled. Recovery holds the rerun until the Turn's surface can deliver, as it holds the restart notice; the transport's ready callback starts it. |
+| Retry-loop failures; a Stop while the Turn waits open | Handled. One retry per Turn that survives a failed pass and ends when the Turn reruns or settles; a Stop that settles a waiting Turn has T2 end its command. |
+| Continuation admission recheck | Removed with the continuation. |
+| Failed silent final | Handled. A failed final keeps its own error and promotes nothing. |
+| Query plans of reads on the controller loop | Handled. Every read restart recovery performs is pinned to a bounded index `SEARCH`. |
+
+## Contract
+
+1. **Who is continued.** At startup, recovery continues every Vibey Turn whose
+ accepted work a restart cut, whichever writer recorded the cut, when all of
+ these hold:
+ - the Session is active and not system-visibility, and its Agent still runs
+ on the `vibey` backend, as does the Agent that ran the Turn (another
+ Agent never continues its work);
+ - the Turn's start was accepted (`start_receipt_outcome = accepted`), and no
+ stop was asked of it (`stop_requested`: a Stop or replacement control, or
+ a cancel of one of its live Runs), and no Run it started for has ended;
+ - the loop bound is not reached (item 6).
+ A Turn whose final response was committed qualifies by the same rules; its
+ rerun ends with that final (item 8).
+ Anything else keeps today's behavior: the Turn settles as interrupted, its
+ Runs settle without result, and the user gets today's notice.
+2. **What continues.** The same Turn. It stays `active` with its accepted
+ start; recovery clears its runtime generation (the dead process's runtime
+ and native ids) and records the restart streak in one writer transaction,
+ fenced on the Turn as the decision read it (a concurrent recovery pass that
+ rotated it first leaves this one nothing to release), then reruns the Turn through the ordinary runner with `vibey_resume`, once
+ the Turn's surface can deliver (its transport ready: recovery runs before
+ external transports connect, so the rerun waits as the restart notice does,
+ and that transport's ready callback starts it), with
+ the request it was accepted with, while its Vibe Agent still runs on Vibey:
+ its native start records that request (the Agent's id and name, the
+ effective model and reasoning effort, and the Agent's instructions) on the
+ start receipt, and the rerun finds the Agent by id, which survives a rename,
+ and runs with the recorded settings, which the message handler puts on the
+ rerun's request before the shared model validation. A Session's switch to another Agent, or
+ an edit of the Agent's settings, reaches the next Turn, as it does for a
+ live one. What a run reads from the machine when it starts (project
+ instructions, skills, the environment) is read again, as for any run. A Turn
+ an older release accepted names its Agent only as the author of its
+ committed responses, and runs with that Agent's current settings; one cut
+ before any response names none and keeps the notice. Vibey resumes its committed
+ transcript (`Agent.resume`, no input consumed, no visible `continue`, no
+ replay of a committed tool call), and the rerun's native start binds this
+ process's generation where any run's does, once its Agent exists, so steers
+ held for the Turn are dispatched when they can join it. The Turn's accepted inputs,
+ steers included, are pinned as for Retry. A steer whose acceptance receipt
+ the dead process recorded but did not materialize is reconciled into the
+ Turn first; while any steer into a Turn that would rerun is still
+ unsettled, the Turn stays open and recovery retries before it reruns. A
+ Turn that no longer qualifies settles at once, and the steer then settles
+ against it, as before this change.
+3. **Tool calls.** A rerun treats a job that survived the restart as its live
+ call would. Startup's T2 pass leaves a running job of a Turn still open to
+ the Turn owner; the rerun's own T2, before its first projection, waits for
+ it until it exits, until the call's foreground window (`bash`'s, 120 s by
+ default) passes, counted from the job's launch (`meta.created_at`), or until
+ a Stop, which ends the command as in a live Turn: the Turn is durably
+ active, so a Stop reaches the adapter's own Stop during the wait. A
+ cancellation outside a shutdown (the backend disabled or refreshed) ends the
+ command too, also before the adapter took it, and a rerun that fails before
+ its adapter has its command handed to a Watch, as a failed run's is; a
+ shutdown leaves it for the
+ next start's rerun. An exited
+ job gives its real output; only a job still running then is
+ handed to a Watch, with the ordinary handover result; a window the downtime
+ already spent hands over at once. A handover that fails is retried once,
+ and with no Watch to take it the job stays in the foreground until it ends,
+ as `bash` keeps it in a live Turn. Nothing runs twice: the wait commits and
+ starts nothing, and a committed result or handover is reused. A Turn that
+ falls back to the notice gets today's T2 first (the Turn owner asks Vibey's
+ recovery owner to settle that Session), so its notice still names its
+ Watches and offers no Retry. While no Watch can take its command yet, that
+ Turn stays open and its owner retries (T2 before T4); a Stop still settles
+ it, and then T2 ends the command at once, since settling a Turn open for
+ a restart retry (also one held for its transport or a steer) wakes the
+ retry (a command whose Turn was canceled is ended, as the Stop would have ended it live, never handed to a Watch, and
+ so is one whose Turn holds a stop the dead process recorded and never
+ settled, by the same `stop_requested` rule that keeps it from rerunning).
+4. **Runs follow the work.** Runs never leave the Turn: the rerun settles them
+ with its own outcome and output, and callbacks fire once, from that outcome.
+ At shutdown, `release_for_service_shutdown` and the scheduled-task teardown
+ leave a Vibey Turn and its Runs unsettled, and a Vibey run that fails
+ because the service is stopping (in a model call, or in a rerun's preflight
+ as the Model Hub stops) leaves its Turn open, so startup finds them as lost
+ active Turns like a conversation Turn.
+5. **Ordering.** The Turn stays the Session's active Turn, so any input queued
+ while the service was down waits behind it, and may be steered into it by
+ the existing steer path.
+6. **Loop bound.** The Turn's start receipt carries its restart streak: the
+ restarts in a row that cut it with no committed model response in between.
+ A committed response resets the streak, and a restart counts once however
+ often its recovery is retried (a failure setting the rerun up keeps the Turn
+ open for a retry). The third restart in a row falls back to item 1's
+ notice, which keeps Retry.
+7. **Presentation.** No notice, no ⚠️ reaction, no interruption copy, and no
+ second Harness prompt echo (the rerun dispatches no prompt). The session
+ shows the rerunning Turn as running.
+ The fallback notices of item 1 point to Retry where the notice shows it, or
+ to sending `continue`, instead of asking for a resend (copy fix in both
+ languages). A rerun's processing indicator runs as any Turn's does, and its
+ end also retires the 👀 the dead process may have left on its IM message
+ (there may be none, so a failed removal withholds no receipt).
+8. **A committed final.** A rerun whose Turn already holds its final response
+ (the loop's own finality rule, read after the rerun's T2 and T3) makes no
+ model call: it ends where the run that died would have, through `_settle`
+ and the ordinary result emit with that committed row. Delivery, a silent
+ final's promotion, the processing indicator, Run settlement, and the
+ successor all run in the live code. A final that carries delivery evidence
+ for its target is not sent again: the dispatcher writes a committed row's
+ display only as its delivery (on the Web that write is the delivery; on IM
+ it follows a send that returned an id), so a row written for this platform
+ is its receipt, as a native id is for an inserted row. A final without it is
+ sent once, with the images the dead run generated, which its committed
+ display carries. A silent final promotes the Turn's earlier visible text,
+ read from the Turn's rows because the process that showed it died. A Stop
+ arriving in the new process loses to the final, as it does live.
+
+## Ownership and touch points
+
+- `core/session_turns.py`: `recover_durable_delivery_state` reconciles a lost
+ Vibey Turn's steer receipts (`_recover_steer_attempts`), then
+ `_recover_restart_cut_vibey_turn` reruns it in place (`_rerun_turn`, which
+ shares the dispatch preparation of `_start_persisted_turn`), holds it until
+ its transport can deliver (`notify_transport_ready` starts it), keeps it open
+ for a retry, or leaves it to today's settlement. `_retry_restart_recovery`
+ recovers a waiting Turn again. `release_for_service_shutdown` skips a Vibey
+ Turn nobody asked to stop.
+- `core/message_dispatcher.py` with `core/message_mirror.py`: item 8's delivery
+ evidence (`delivered_committed_row`) and a rerun's silent-final result
+ (`resumed_turn_text`), both bounded reads on the controller loop.
+- `core/restart_resume.py`: `restart_resume_plan` (item 1, the steer wait, and
+ item 6's streak).
+- `storage/message_deliveries.py`: `rotate_restart_generation` (the writer
+ transaction of item 2) and `turn_has_unresolved_steer`; `bind_native_start`
+ keeps the restart streak when the rerun binds its generation.
+- `storage/agent_transcript.py`: `last_response` and `committed_final`, bounded
+ reads of the context tail, and `last_turn_text`, the silent final's result,
+ bounded by the Turn's start. A final follows the loop's own rule: no tool
+ calls, not the `error`/`aborted` partial a failed stream commits, and no
+ input after it.
+- `modules/agents/vibey/agent.py`: a rerun takes a committed final
+ (`_take_committed_final`) and settles from it.
+- `modules/agents/vibey/agent.py`, `recovery.py`, `tools.py` and agent-core
+ `settle_open_calls` / `bash.settle_in_foreground`: item 3's T2 split
+ (startup defers a running job of an open Turn; the rerun's T2 waits; T2 ends
+ the command of a canceled Turn; `VibeyRecovery.settle_session` hands over
+ before a fallback notice). Stopping the service stops the Model Hub gateway
+ before Turns end, so a run in a model call fails during shutdown; a run that
+ ends without completing while the service is stopping leaves its Turn open
+ instead of settling it.
+- `core/scheduled_tasks.py`: no change needed. Its shutdown cancellation only
+ covers executions still in flight, and a Turn-owned Run has already handed
+ its execution to the Turn; `recover_processing` skips Delivery-linked Runs.
+- Docs: `agent-core-contracts/recovery.md` T4, `backend-failure-retry.md`.
+ Public docs (`avibe-docs` `agents/vibey.mdx` Restart bullet, both languages)
+ change in a follow-up in that repository.
+
+## Validation
+
+Scenario IDs in `tests/scenarios/message_delivery/catalog.yaml`:
+
+- `MESSAGE-DELIVERY-327`: startup recovery with a cut conversation Turn: the
+ same Turn reruns as a transcript resume, no Delivery, no notice, once its
+ transport can deliver.
+- `MESSAGE-DELIVERY-328`: cut Harness Turn: shutdown leaves the Turn and its
+ Run; the rerun settles the Run from its own result, no owed restart notice;
+ a Run steered into the Turn whose receipt was never materialized settles the
+ same way, after a retry when startup could not materialize it.
+- `MESSAGE-DELIVERY-329`: ineligible cases each fall back to today's settlement
+ and notice: Stop requested, archived Session, Agent switched off Vibey, the
+ third restart in a row without a model response (a committed response
+ resets the streak). A start never accepted stays on the unknown-start path
+ it takes today.
+- `MESSAGE-DELIVERY-330`: a command running at the restart: startup leaves it
+ to the rerun, also across another restart, which gives a job exiting inside
+ its window (from launch) its real output, hands over one whose window the
+ downtime spent, and ends it on a Stop; a committed handover is reused; the
+ command never starts again.
+- `MESSAGE-DELIVERY-331`: input queued during downtime runs after the rerun.
+- `MESSAGE-DELIVERY-332`: a run that fails because the service is stopping
+ leaves its Turn open.
+- `MESSAGE-DELIVERY-333`: a committed final: the Turn reruns, also across
+ another restart, and the rerun ends with it and no model call; a final not
+ yet delivered to IM is sent once, with its images; one already delivered is
+ not resent; a silent final promotes the narration the dead process showed.
+- `MESSAGE-DELIVERY-322` keeps the Watch-naming notice without Retry for the
+ fallback, and a Stop while it waits for that Watch ends the command.
+- Local Incus: restart the master environment during a long Vibey conversation
+ Turn and during a Vibey Agent Run with a callback; both continue without a
+ notice, and the callback reports the continued result.
+
+## Known by design
+
+- The response being generated at the restart is generated again: its partial
+ output was never committed.
+- A `write` or `edit` cut mid-call gets the interrupted result and the model is
+ told to re-read the file, as after any interruption.
+- While the rerun waits for a surviving command it shows as running without
+ new activity, as a live foreground command does; it starts its model call
+ after the wait. A Stop then ends the command and the Turn before any model
+ call.
+- The restart path is at-least-once for a committed final: one the dead process
+ sent to IM just before the dispatcher recorded its delivery is sent again by
+ the rerun. A rare duplicate is the accepted cost; a silently lost answer is
+ not.
+- A Stop requested before the restart keeps the notice even when the final was
+ committed (item 1 applies to every Turn); the answer stays in the
+ transcript.
+- A rerun has a live Turn's reliability, and no durable retry beyond it. A
+ transient failure in its run or its end is handled as in a live Turn:
+ - removing the 👀 the dead process left is one attempt, like a live Turn's
+ own removal;
+ - a failed read or write of a silent final's promotion is logged and the
+ Turn settles, as a live promotion's failure is (a silent final's Run
+ result is empty either way);
+ - a failure in the rerun's own T2 or T3 before its run fails the Turn with
+ the notice and Retry, as at the start of any live run. A command whose
+ result was not committed keeps its job's output for the next resume.
+- A Turn whose initial Run already settled from its result, in the moment
+ before the Turn's own terminal write, cannot rerun (the runner refuses a
+ Turn whose initial Run ended). It keeps today's restart settlement, as on
+ every backend: the Run keeps its result and owes no notice, and the Turn is
+ recorded as interrupted.
+- A message-mode processing indicator (Telegram's "processing" message) is
+ deleted once a run's native start is accepted, and a rerun requires an
+ accepted start, so the dead process has already deleted it. Only a process
+ that died inside that delete call leaves it. That is the same on every
+ backend, and no ack message id is recorded durably to clean it up.
+- A rerun whose transport never becomes ready stays open, as a scheduled Run
+ waits for its transport; a Stop ends it.
+- A rerun's final that cites sources is displayed again from its message, so
+ images the dead run generated are not attached to it; they stay in the
+ Turn's tool results.
+- A Stop is checked before each handover of a surviving command, as the live
+ `bash` call checks it; one that arrives while a handover succeeds leaves the
+ command with its Watch, as it does live, and the Watch can be stopped.
+- Native backends (Claude Code, Codex, OpenCode) keep today's behavior.
diff --git a/modules/agents/vibey/agent.py b/modules/agents/vibey/agent.py
index 4edc723b4..cc5fd3dd2 100644
--- a/modules/agents/vibey/agent.py
+++ b/modules/agents/vibey/agent.py
@@ -15,9 +15,12 @@ One Turn is one ``Agent.run`` (plan section 4). The adapter
the route's Model Hub limits;
* before any run of a Session in this process, under the Session writer lock,
settles open tool calls (T2) and admits accepted inputs that were never
- consumed (T3); at startup it settles every Session's open calls eagerly. The
- Turn a restart interrupted is settled by the Turn owner (T4; ``vibey`` is a
- process-bound runtime).
+ consumed (T3); at startup it settles every Session's open calls eagerly,
+ except a running command of a Turn not yet settled. The Turn owner then
+ reruns the Turn a restart interrupted in place, as a transcript resume of
+ itself (T4; ``vibey`` is a process-bound runtime), whose T2 waits for that
+ command as its live call would, or settles the Turn after having it handed to
+ a Watch; a run the service stop cut leaves its Turn open for that.
"""
from __future__ import annotations
@@ -112,6 +115,7 @@ from storage.agent_transcript import (
RESPONSE_TYPES,
SQLiteTranscriptStore,
call_instance_result,
+ committed_final,
final_outcome,
render_text,
text_parts,
@@ -201,6 +205,9 @@ class _Run:
replied: dict[str, bool] = field(default_factory=dict)
# The images each final delivers with its reply, as committed with its row.
reply_images: dict[str, tuple[HostedImage, ...]] = field(default_factory=dict)
+ # A rerun's final as its row committed it, undelivered: that display carries the images
+ # the run that died generated, which this run does not have.
+ committed_display: Optional[str] = None
# Steer attempts ``Agent.steer`` accepted into this run: reconcile's in-process evidence.
accepted_steers: set[str] = field(default_factory=set)
# Automatic transcript-only continuations already spent by this Turn. The
@@ -328,13 +335,39 @@ class VibeyAgent(BaseAgent):
# Rows this Turn writes are the Turn's Agent's, from its request snapshot, even if the
# Session's selected Agent changes while it runs.
self.store.bind_agent(session_id, request.vibe_agent_name, turn_id=turn.turn_id)
- await self._resume(runtime)
+ # A transcript resume (a Turn rerun after a restart) gives a job that survived the restart
+ # the foreground wait its live call would have, so its real output becomes the call's result.
+ resuming = (context.platform_specific or {}).get("vibey_resume") is True
+ try:
+ await self._resume(runtime, stop=turn.stop_event if resuming else None)
+ except asyncio.CancelledError:
+ if resuming and getattr(self.controller, "shutdown_requested", False) is not True:
+ # Cancelled outside a shutdown (the backend disabled or refreshed): the work
+ # stops, so the command the wait was for ends with it instead of running on
+ # unowned. A shutdown leaves it for the next start's rerun.
+ turn.stop_event.set()
+ await self._settle_open_calls(runtime, stop=turn.stop_event)
+ raise
+ if resuming and await self._take_committed_final(turn):
+ # The run that died committed the Turn's final: this one ends where that one
+ # would have, with no model call. A Stop now loses to it, as it does live.
+ self._accept_native_start(request, session_id)
+ indicator = getattr(self.controller, "processing_indicator", None)
+ if indicator is not None:
+ await indicator.delete_ack_message(request)
+ await self._settle(turn)
+ self._maybe_backfill_session_title(request, session_id)
+ return
if turn.stop_requested:
return await self._settle_stopped(turn)
cwd = request.working_path or runtime.cwd
try:
turn.router = await self._preflight(turn)
except Exception as error:
+ if self._service_stopping(turn):
+ # Stopping the service (its Model Hub gateway first) failed the route, not
+ # the Turn: leave it open, as a run cut below is, for the restart.
+ await asyncio.Event().wait()
turn.settled = True
await self._fail_preflight(request, error)
return
@@ -351,10 +384,9 @@ class VibeyAgent(BaseAgent):
message = await self._render_input(session_id, request.message, request.files, request.input_metadata)
if turn.stop_requested:
return await self._settle_stopped(turn)
- # Native acceptance materializes the input row the loop consumes.
- self.bind_agent_session_id(request, session_id)
- mark_backend_dispatch_attempted(context)
- self.mark_runtime_turn_started(context)
+ # Native acceptance materializes the input row the loop consumes, and binds this
+ # process's generation: steers held for the Turn are dispatched once its Agent exists.
+ self._accept_native_start(request, session_id)
indicator = getattr(self.controller, "processing_indicator", None)
if indicator is not None:
# The ack message goes once the native write is accepted, as for the other backends.
@@ -405,11 +437,64 @@ class VibeyAgent(BaseAgent):
transcript_resume = True
continue
agent_input = await self._continuing_input(turn)
+ if self._cut_by_service_stop(turn):
+ # Stopping the service (its Model Hub gateway first) ended this run, not
+ # anything the Turn did. Leave the Turn open: startup recovery runs T2 and
+ # continues it from the committed transcript (vibey-restart-auto-resume.md).
+ await asyncio.Event().wait()
if turn.reason not in _COMPLETED:
await self._settle_left_open(runtime)
await self._settle(turn)
self._maybe_backfill_session_title(request, session_id)
+ async def _take_committed_final(self, run: _Run) -> bool:
+ """Whether the Turn's transcript already holds its final response, which the run then takes.
+
+ Its replied decision is the one its row's type was committed by (``final_outcome``).
+ """
+ found = await asyncio.to_thread(self._committed_final, run.session_id, run.turn_id)
+ if found is None:
+ return False
+ row_id, message_type, message, display = found
+ run.final_row, run.reason = row_id, "completed"
+ run.responses[row_id] = message
+ run.replied[row_id] = message_type != "error"
+ run.committed_display = display
+ return True
+
+ def _committed_final(
+ self, session_id: str, turn_id: str
+ ) -> Optional[tuple[str, str, AssistantMessage, Optional[str]]]:
+ """``(row_id, type, message, committed display or None once delivered)`` of the Turn's final."""
+ with self._engine.connect() as conn:
+ final = committed_final(conn, session_id, turn_id)
+ row = (
+ conn.execute(
+ select(messages.c.content_json, messages.c.content_text).where(messages.c.id == final.row_id)
+ ).first()
+ if final is not None
+ else None
+ )
+ if final is None or row is None:
+ return None
+ try:
+ content = json.loads(row.content_json or "{}")
+ message = message_from_dict(content["model"]["message"])
+ except (TypeError, ValueError, KeyError):
+ logger.warning("Vibey could not read the committed final %s of Turn %s", final.row_id, turn_id)
+ return None
+ if not isinstance(message, AssistantMessage):
+ return None
+ return final.row_id, final.message_type, message, None if "kind" in content else str(row.content_text or "")
+
+ def _cut_by_service_stop(self, run: _Run) -> bool:
+ """Whether a run that did not complete ended while the service was stopping."""
+ return bool(run.reason not in _COMPLETED and not run.final_row and self._service_stopping(run))
+
+ def _service_stopping(self, run: _Run) -> bool:
+ """Whether the service is stopping under a run nobody asked to stop."""
+ return not run.stop_requested and getattr(self.controller, "shutdown_requested", False) is True
+
def _resume_turn_inputs(self, session_id: str, source_turn_id: Any) -> tuple[int, ...]:
"""Committed inputs of the failed task, including steers across its retry ancestry."""
if not isinstance(source_turn_id, str) or not source_turn_id:
@@ -481,6 +566,21 @@ class VibeyAgent(BaseAgent):
return False
return time.monotonic() - run.continuation_started_at < policy.max_elapsed_s
+ def _accept_native_start(self, request: AgentRequest, session_id: str) -> None:
+ # The native start records the request this Turn runs with, which a restart rerun keeps.
+ if request.context.platform_specific is None:
+ request.context.platform_specific = {}
+ request.context.platform_specific["vibey_execution"] = {
+ "agent_id": request.vibe_agent_id,
+ "agent_name": request.vibe_agent_name,
+ "model": request.vibe_agent_model,
+ "reasoning_effort": request.vibe_agent_reasoning_effort,
+ "system_prompt": request.vibe_agent_system_prompt,
+ }
+ self.bind_agent_session_id(request, session_id)
+ mark_backend_dispatch_attempted(request.context)
+ self.mark_runtime_turn_started(request.context)
+
async def _settle_stopped(self, turn: _Run) -> None:
"""A Stop before the loop started: the Turn settles as stopped, nothing dispatched."""
turn.reason = "aborted"
@@ -535,29 +635,34 @@ class VibeyAgent(BaseAgent):
if rt.run is not None and rt.run.request.session_key == session_key
}
- async def recover_runtime_state(self) -> list[str]:
+ async def recover_runtime_state(self, session_id: Optional[str] = None) -> list[str]:
"""One settlement pass outside a Turn: T2, T3 admission, then J5; return the unsettled Sessions.
Candidates come from the rows: Sessions whose context can hold an open call, and
Sessions with accepted inputs no run consumed. Each runs the same resume a Turn
runs (T2 and T3, no model call). A Session a run holds is skipped: its own resume
- settles it. J5 pruning follows a pass that settled everything. Never reports
- success over a failure; ``VibeyRecovery`` owns the retries. The first pass of the
- process also starts the media sweep.
+ settles it. So is a Session whose open call has a running job and an unsettled
+ Turn: the Turn owner either continues that Turn, whose own resume waits for the
+ job, or settles it after asking for this Session alone (``session_id``), which
+ hands the job over as before (``vibey-restart-auto-resume.md``). J5 pruning follows
+ a pass that settled everything. Never reports success over a failure;
+ ``VibeyRecovery`` owns the retries. The first pass of the process also starts the
+ media sweep.
"""
if self._media_sweep is None:
self._media_sweep = asyncio.create_task(self._sweep_media(), name="vibey-agent-media-sweep")
- candidates = await asyncio.to_thread(self._recovery_candidates)
+ explicit = session_id is not None
+ candidates = [session_id] if session_id else await asyncio.to_thread(self._recovery_candidates)
unsettled: list[str] = []
- for session_id in candidates:
- async with self._held(session_id, wait=False) as runtime:
+ for candidate in candidates:
+ async with self._held(candidate, wait=False) as runtime:
if runtime is None:
continue
try:
- await self._resume(runtime)
+ await self._resume(runtime, defer_continued=not explicit)
except Exception:
- logger.exception("Vibey could not settle Session %s outside a Turn", session_id)
- unsettled.append(session_id)
+ logger.exception("Vibey could not settle Session %s outside a Turn", candidate)
+ unsettled.append(candidate)
if not unsettled:
await self._prune_settled_jobs()
return unsettled
@@ -830,6 +935,8 @@ class VibeyAgent(BaseAgent):
body, citations = self._display_with_citations(
final, final=True, replied=bool(replied), images=run.reply_images.get(run.final_row, ())
)
+ if run.committed_display is not None and citations is None:
+ body = run.committed_display
if failed and not body.strip():
# A silent final whose run failed after the commit: like a final without text
# of its own (``_display_source``), its row carries the explanation.
@@ -935,11 +1042,21 @@ class VibeyAgent(BaseAgent):
# --- resume (recovery.md T2, T3) -------------------------------------------
- async def _resume(self, runtime: _SessionRuntime) -> None:
+ async def _resume(
+ self,
+ runtime: _SessionRuntime,
+ *,
+ stop: Optional[asyncio.Event] = None,
+ defer_continued: bool = False,
+ ) -> None:
+ """T2, then T3. ``stop``: the continuing Turn's own resume, which waits for a running job
+ (``_settle_open_calls``). ``defer_continued``: recovery leaves alone a Session the Turn owner
+ has yet to continue or settle."""
session_id = runtime.session_id
# T2 before every run: idempotent, and a Turn that waited behind an aborted one
# finds the call that abort left open on the same, still-held runtime.
- await self._settle_open_calls(runtime)
+ if not await self._settle_open_calls(runtime, stop=stop, defer_continued=defer_continued):
+ return
if not runtime.cwd:
runtime.cwd = await asyncio.to_thread(self._session_workdir, session_id)
for message_id, text_value, files, delivery in await asyncio.to_thread(self._unconsumed_inputs, session_id):
@@ -977,12 +1094,24 @@ class VibeyAgent(BaseAgent):
logger.exception("Vibey could not settle the calls a run left open in Session %s", runtime.session_id)
self._schedule_recovery()
- async def _settle_open_calls(self, runtime: _SessionRuntime) -> None:
- """T2: one committed result for every open tool call, chosen from its job's state."""
+ async def _settle_open_calls(
+ self,
+ runtime: _SessionRuntime,
+ *,
+ stop: Optional[asyncio.Event] = None,
+ defer_continued: bool = False,
+ ) -> bool:
+ """T2: one committed result for every open tool call, chosen from its job's state.
+
+ With ``stop``, a job still running is first waited for as its live call would wait
+ (its foreground window from the job's launch, or until ``stop``), so only a job still
+ running then is handed to a Watch. With ``defer_continued``, a running job whose Turn
+ is not yet settled is left to that Turn's owner and nothing is written: ``False``.
+ """
session_id = runtime.session_id
open_calls = open_tool_calls(await self.store.load(session_id))
if not open_calls:
- return
+ return True
suite = self._tools()
def evidence() -> dict[CallInstance, str]:
@@ -1002,6 +1131,25 @@ class VibeyAgent(BaseAgent):
# the Turn writing it now (or none, at startup) or the Session's current selection.
# Only the last response can hold open calls, so they share one owner.
owner_agent, owner_turn = await asyncio.to_thread(self._response_owner, open_calls[0][0].row_id)
+ work = await asyncio.to_thread(self._work_state, owner_turn)
+ if (
+ defer_continued
+ and work == "open"
+ and any(suite.jobs.status(job_id).state == "running" for job_id in job_ids.values())
+ ):
+ return False
+ if work == "stopped" and stop is None:
+ # A Stop settled the Turn while its command outlived the run: it ends the command,
+ # as it ends a live foreground command, rather than leaving it to a Watch.
+ stop = asyncio.Event()
+ stop.set()
+ wait_running = None
+ if stop is not None and suite.wait_recovered is not None:
+ wait_recovered = suite.wait_recovered
+
+ async def wait_running(call: Any, job_id: str) -> Any:
+ return await wait_recovered(call, job_id, stop)
+
with self.store.writing_as(session_id, owner_agent, turn_id=owner_turn):
await settle_open_calls(
session_id=session_id,
@@ -1009,7 +1157,28 @@ class VibeyAgent(BaseAgent):
jobs=suite.jobs,
job_ids=job_ids,
render_result=suite.render_recovered,
+ wait_running=wait_running,
)
+ return True
+
+ def _work_state(self, turn_id: Optional[str]) -> str:
+ """Where the Turn owning an open call stands: ``open``, ``stopped``, or ``settled``.
+
+ A Turn a restart cut stays open and reruns in place, so its call's Turn is the one
+ that continues it. ``stopped``: a Stop or a replacement canceled that Turn, or a stop
+ asked of it is not yet settled (``stop_requested``; it never reruns).
+ """
+ from core.restart_resume import stop_requested
+
+ if not turn_id:
+ return "settled"
+ with self._engine.connect() as conn:
+ turn = delivery_store.get_turn(conn, turn_id)
+ if turn is not None and turn["state"] in delivery_store.TURN_OWNER_STATES:
+ return "stopped" if stop_requested(conn, turn) else "open"
+ if turn is None:
+ return "settled"
+ return "stopped" if turn.get("terminal_outcome") == "canceled" else "settled"
def _response_owner(self, row_id: str) -> tuple[Optional[str], Optional[str]]:
with self._engine.connect() as conn:
diff --git a/modules/agents/vibey/recovery.py b/modules/agents/vibey/recovery.py
index c7c7465d7..3f7306215 100644
--- a/modules/agents/vibey/recovery.py
+++ b/modules/agents/vibey/recovery.py
@@ -51,6 +51,23 @@ class VibeyRecovery:
self._task = asyncio.create_task(self._retry(), name="vibey-agent-recovery")
return unsettled
+ async def settle_session(self, session_id: str) -> bool:
+ """Settle one Session now, a running job handed to its Watch: ``True`` once it settled.
+
+ The Turn owner asks before it settles a Turn a restart cut without continuing it, so
+ that Turn's notice names the Watches (T2 before T4). Serialized with every pass; a
+ failure is retried like any unsettled Session.
+ """
+ async with self._lock:
+ try:
+ unsettled = await self._controller.agent_service.agents["vibey"].recover_runtime_state(session_id)
+ except Exception:
+ logger.exception("Vibey could not settle Session %s", session_id)
+ unsettled = [session_id]
+ if unsettled:
+ self.ensure()
+ return not unsettled
+
def ensure(self) -> None:
"""A Turn could not settle something (a returned input it failed to admit): retry it."""
if not self.retrying:
diff --git a/modules/agents/vibey/tools.py b/modules/agents/vibey/tools.py
index 0f2bd583e..68903f155 100644
--- a/modules/agents/vibey/tools.py
+++ b/modules/agents/vibey/tools.py
@@ -3,8 +3,9 @@
``ToolSuite`` is the adapter's whole dependency on ``core.agent_core.tools``:
the job host, the tools built over the loop's tracking wrapper (``Agent.jobs``,
so an abort kills foreground commands), the bash renderer recovery uses for open
-calls (``recovery.md`` T2), the lookup from a call instance to its job, and
-job pruning.
+calls (``recovery.md`` T2), the foreground wait a Turn continued after a restart
+gives a surviving job, the lookup from a call instance to its job, and job
+pruning.
The job host lives in Watch's ``agent_jobs_dir()``, where ``vibe stop`` and the
job-Watch sweep look, and hands a job over through Watch's own
@@ -38,7 +39,7 @@ from core.agent_core.messages import (
ToolResultMessage,
message_from_dict,
)
-from core.agent_core.tools.base import CallInstance, JobHost, Tool, ToolContext, ToolResult, ToolSpec
+from core.agent_core.tools.base import CallInstance, JobHost, JobStatus, Tool, ToolContext, ToolResult, ToolSpec
from core.agent_core.tools.hosted import MAX_IMAGES, hosted_tool_spec, image_generate_spec, size_form
from core.agent_core.tools.read import MAX_INLINE_IMAGE_BYTES, ImageUnreadable, read_image
from core.agent_core.tools.truncate import format_size
@@ -59,6 +60,9 @@ class ToolSuite:
#: The job a call instance started, if any.
find_job: Callable[[CallInstance], Optional[str]]
prune: Optional[Callable[[CallSettled], list[str]]] = None
+ #: ``(call, job_id, stop)``: a surviving job settled as its live call would, from its launch, or until
+ #: ``stop``: its status and Watch (``settle_in_foreground``); ``None`` hands a running job over at once.
+ wait_recovered: Optional[Callable[[Any, str, asyncio.Event], Awaitable[tuple[JobStatus, Optional[str]]]]] = None
#: The shell commands run in, as the environment block names it (C-7 tools.md section 8).
shell: str = field(default_factory=lambda: default_shell().describe())
@@ -72,9 +76,10 @@ def local_tool_suite(jobs_dir: Optional[str] = None, *, route: Optional[HandOver
Every path that creates a job Watch (``watch=true``, the foreground-window hand-over,
and T2 recovery) goes through the host's ``on_hand_over``, which takes the Watch's
- Agent and authorization from ``route``: the job's owning Turn.
+ Agent and authorization from ``route``: the job's owning Turn. ``bash`` and the
+ recovered wait use the same default foreground window.
"""
- from core.agent_core.tools.bash import recovered_result
+ from core.agent_core.tools.bash import recovered_result, settle_in_foreground
from core.agent_core.tools.coding import create_coding_tools
from core.agent_core.tools.jobs import LocalJobHost
from core.watches import ManagedWatchStore, agent_jobs_dir
@@ -102,6 +107,9 @@ def local_tool_suite(jobs_dir: Optional[str] = None, *, route: Optional[HandOver
async def render(call: Any, job_id: str, status: Any, watch_id: Optional[str]) -> Any:
return await recovered_result(host, job_id, status, watch_id)
+ async def wait_recovered(call: Any, job_id: str, stop: asyncio.Event) -> tuple[JobStatus, Optional[str]]:
+ return await settle_in_foreground(host, call, job_id, stop=stop)
+
def prune(call_settled: CallSettled) -> list[str]:
# J5: a job's files stay until its call has a durable result and no Watch still manages it.
watches = ManagedWatchStore()
@@ -114,6 +122,7 @@ def local_tool_suite(jobs_dir: Optional[str] = None, *, route: Optional[HandOver
find_job=host.find_job,
prune=prune,
shell=host.shell.describe(),
+ wait_recovered=wait_recovered,
)
diff --git a/storage/agent_transcript.py b/storage/agent_transcript.py
index 667df4dc0..e91ed769e 100644
--- a/storage/agent_transcript.py
+++ b/storage/agent_transcript.py
@@ -91,6 +91,7 @@ from storage.agent_session_rows import reserve_write_lock
from storage.message_deliveries import TURN_OWNER_STATES
from storage.models import agent_events, agent_sessions, message_deliveries, messages, scopes, session_turns
from storage.vibey_recovery import VIBEY_RECOVERY_INPUT_PREDICATE
+from vibe.message_types import PROMOTED_FROM_TYPES
INPUT_TYPES = ("user", "harness", "agent_initiated", "annotation")
# A final response is ``result``, or ``error`` when ``final_outcome`` says it failed:
@@ -757,6 +758,141 @@ def recovered_watch_ids(conn: Connection, session_id: str, turn_id: str) -> list
return [str(json.loads(raw)["details"]["watch_id"]) for raw in rows]
+# A response that stopped this way is never final: the loop commits the partial a failed
+# stream carried with ``final=False`` (``Agent._commit_model_message``).
+_UNFINISHED_STOP_REASONS = frozenset({"error", "aborted"})
+
+
+@dataclass(frozen=True)
+class CommittedResponse:
+ """A response row in a Session's own context: the facts restart recovery reads."""
+
+ row_id: str
+ context_seq: int
+ message_type: str
+ turn_id: Optional[str]
+ text: str
+ has_tool_calls: bool
+ stop_reason: str
+ agent_name: Optional[str] = None
+
+
+def last_response(conn: Connection, session_id: str) -> Optional[CommittedResponse]:
+ """The last model response the Session's own rows committed, by one bounded index read."""
+ row = conn.execute(
+ select(
+ messages.c.id,
+ messages.c.context_seq,
+ messages.c.type,
+ messages.c.content_text,
+ messages.c.content_json,
+ messages.c.metadata_json,
+ messages.c.author_name,
+ )
+ .where(
+ messages.c.session_id == session_id,
+ messages.c.context_seq.is_not(None),
+ messages.c.type.in_(RESPONSE_TYPES),
+ )
+ .order_by(messages.c.context_seq.desc())
+ .limit(1)
+ ).mappings().first()
+ if row is None:
+ return None
+ try:
+ model_message = json.loads(row["content_json"] or "{}")["model"]["message"]
+ blocks, stop_reason = model_message["content"], str(model_message.get("stop_reason") or "")
+ except (TypeError, ValueError, KeyError, AttributeError):
+ blocks, stop_reason = [], ""
+ try:
+ metadata = json.loads(row["metadata_json"] or "{}")
+ except (TypeError, ValueError):
+ metadata = {}
+ turn_id = metadata.get("turn_id") if isinstance(metadata, dict) else None
+ return CommittedResponse(
+ row_id=str(row["id"]),
+ context_seq=int(row["context_seq"]),
+ message_type=str(row["type"]),
+ turn_id=str(turn_id) if turn_id else None,
+ text=str(row["content_text"] or ""),
+ has_tool_calls=any(
+ isinstance(block, dict) and block.get("type") == "tool_call" for block in blocks or ()
+ ),
+ stop_reason=stop_reason,
+ agent_name=str(row["author_name"]) if row["author_name"] else None,
+ )
+
+
+def committed_final(conn: Connection, session_id: str, turn_id: str) -> Optional[CommittedResponse]:
+ """The final response ``turn_id`` committed, when the Turn's work ended with it.
+
+ Read after T2 and T3, with the loop's finality rule (``Agent._commit_model_message``):
+ a Turn has work left unless its last committed response has no tool calls, did not
+ stop with ``error`` or ``aborted`` (the partial a failed stream commits), and no
+ input follows it, in the context or still waiting to enter it (an accepted steer no
+ run consumed). Every read is bounded by the Session's context tail or by the Turn's
+ own Deliveries.
+ """
+ response = last_response(conn, session_id)
+ if (
+ response is None
+ or response.turn_id != turn_id
+ or response.has_tool_calls
+ or response.stop_reason in _UNFINISHED_STOP_REASONS
+ # A reply pending input superseded is visible, never final.
+ or response.message_type == "output"
+ ):
+ return None
+ later_input = conn.execute(
+ select(messages.c.id)
+ .where(
+ messages.c.session_id == session_id,
+ messages.c.context_seq > response.context_seq,
+ messages.c.type.in_(INPUT_TYPES),
+ )
+ .limit(1)
+ ).first()
+ if later_input is not None:
+ return None
+ unconsumed = conn.execute(
+ text(
+ "select messages.id from message_deliveries "
+ "cross join messages on messages.id = message_deliveries.message_id "
+ "where message_deliveries.turn_id = :turn_id and message_deliveries.state = 'accepted' "
+ f"and {VIBEY_RECOVERY_INPUT_PREDICATE} limit 1"
+ ),
+ {"turn_id": turn_id},
+ ).first()
+ return None if unconsumed is not None else response
+
+
+def last_turn_text(conn: Connection, session_id: str, *, since: str) -> Optional[tuple[str, str]]:
+ """``(row_id, text)`` of the last visible text the Session's agent wrote since ``since``.
+
+ What a Turn that ends on a silent final leaves as its result
+ (``promote_turn_text_to_result``): its narration (``interim``) or a superseded
+ reply (``output``). A Session runs one Turn at a time, so with ``since`` the
+ Turn's start these are the Turn's own rows; each read is an index-keyed
+ ``SEARCH`` bounded by that window.
+ """
+ latest: Optional[tuple[str, str, str]] = None
+ for message_type in PROMOTED_FROM_TYPES:
+ row = conn.execute(
+ select(messages.c.created_at, messages.c.id, messages.c.content_text)
+ .where(
+ messages.c.session_id == session_id,
+ messages.c.type == message_type,
+ messages.c.created_at >= since,
+ messages.c.author == "agent",
+ )
+ .order_by(messages.c.created_at.desc(), messages.c.id.desc())
+ .limit(1)
+ ).first()
+ if row is not None and str(row.content_text or "").strip() and (latest is None or (row[0], row[1]) > latest[:2]):
+ latest = (str(row.created_at), str(row.id), str(row.content_text))
+ return (latest[1], latest[2]) if latest else None
+
+
def fork_link(conn: Connection, session_id: str) -> Optional[tuple[str, int]]:
"""``(source_session_id, anchor_seq)`` when the Session inherits a context."""
raw = conn.execute(
diff --git a/storage/message_deliveries.py b/storage/message_deliveries.py
index 47a717fe3..17f1a24fc 100644
--- a/storage/message_deliveries.py
+++ b/storage/message_deliveries.py
@@ -1182,7 +1182,13 @@ def bind_native_start(
runtime_key: str | None,
runtime_turn_id: str | None,
native_turn_id: str | None,
+ execution: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
+ """Accept a Turn's native start; ``execution`` records the request a Vibey Turn runs with.
+
+ A Turn rerun after a restart keeps its restart streak and recorded execution across
+ its rebind (``vibey-restart-auto-resume.md``).
+ """
turn = get_turn(conn, turn_id)
if turn is None:
return None
@@ -1206,12 +1212,88 @@ def bind_native_start(
"runtime_turn_id": runtime_turn_id or turn.get("runtime_turn_id"),
"native_turn_id": native_turn_id or turn.get("native_turn_id"),
"start_receipt_outcome": "accepted",
- "start_receipt_json": _canonical_json({"kind": "native_start"}),
+ "start_receipt_json": _canonical_json(
+ {
+ "kind": "native_start",
+ **_carried_receipt(turn),
+ **({"vibey_execution": dict(execution)} if execution else {}),
+ }
+ ),
"started_at": turn.get("started_at") or turn_now_iso(),
},
)
+#: Start-receipt keys of a Turn rerun in place after restarts: the restarts in a row
+#: that cut it, and the context sequence of the last model response then committed.
+RESTART_STREAK_KEYS = ("restart_streak", "restart_response_seq", "restart_process")
+
+
+def _start_receipt(turn: dict[str, Any]) -> dict[str, Any]:
+ try:
+ receipt = json.loads(str(turn.get("start_receipt_json") or "{}"))
+ except (TypeError, ValueError):
+ return {}
+ return receipt if isinstance(receipt, dict) else {}
+
+
+def restart_receipt(turn: dict[str, Any]) -> dict[str, Any]:
+ """The restart streak a Turn's start receipt carries, if it was rerun after a restart."""
+ receipt = _start_receipt(turn)
+ return {key: receipt[key] for key in RESTART_STREAK_KEYS if key in receipt}
+
+
+def recorded_execution(turn: dict[str, Any]) -> dict[str, Any]:
+ """The request a Vibey Turn's native start recorded (its Agent, model, effort, and
+ instructions), or ``{}``."""
+ execution = _start_receipt(turn).get("vibey_execution")
+ return dict(execution) if isinstance(execution, dict) else {}
+
+
+def _carried_receipt(turn: dict[str, Any]) -> dict[str, Any]:
+ receipt = _start_receipt(turn)
+ return {key: receipt[key] for key in (*RESTART_STREAK_KEYS, "vibey_execution") if key in receipt}
+
+
+def rotate_restart_generation(
+ conn: Connection,
+ turn_id: str,
+ *,
+ expected_version: int,
+ streak: int,
+ response_seq: int | None,
+ process: str | None = None,
+) -> dict[str, Any] | None:
+ """Release an accepted Turn from the runtime generation a restart killed, to rerun it.
+
+ The Turn keeps its accepted start; the dead process's runtime and native ids are
+ cleared so the rerun's native start binds this process's, and the start receipt
+ records the restart streak (``vibey-restart-auto-resume.md`` item 6).
+ """
+ turn = get_turn(conn, turn_id)
+ if (
+ turn is None
+ or turn["state"] != "active"
+ or turn.get("start_receipt_outcome") != "accepted"
+ or int(turn["version"]) != expected_version
+ ):
+ return None
+ receipt = _start_receipt(turn)
+ receipt.update({"restart_streak": streak, "restart_response_seq": response_seq, "restart_process": process})
+ return cas_turn(
+ conn,
+ turn_id,
+ expected_version=expected_version,
+ expected_states=("active",),
+ values={
+ "runtime_key": None,
+ "runtime_turn_id": None,
+ "native_turn_id": None,
+ "start_receipt_json": _canonical_json(receipt),
+ },
+ )
+
+
def rebind_restored_native_generation(
conn: Connection,
turn_id: str,
@@ -2569,6 +2651,19 @@ def pending_control_for_turn(conn: Connection, turn_id: str) -> dict[str, Any] |
)
+def turn_has_unresolved_steer(conn: Connection, turn_id: str) -> bool:
+ """Whether a steer into the Turn still waits for its receipt to be settled."""
+ return (
+ conn.execute(
+ select(message_deliveries.c.id)
+ .where(message_deliveries.c.current_target_turn_id == turn_id)
+ .where(message_deliveries.c.state.in_(("steering", "reconciling_steer")))
+ .limit(1)
+ ).first()
+ is not None
+ )
+
+
def pending_steer_for_turn(conn: Connection, turn_id: str) -> dict[str, Any] | None:
return _one(
conn,
diff --git a/tests/agent_core/fakes.py b/tests/agent_core/fakes.py
index 0d3b5011e..d865c5fd4 100644
--- a/tests/agent_core/fakes.py
+++ b/tests/agent_core/fakes.py
@@ -261,6 +261,10 @@ class FakeJobHost:
def status(self, job_id) -> JobStatus:
return self.states.get(job_id, JobStatus("gone"))
+ def meta(self, job_id) -> dict:
+ """No launch record: a surviving job's foreground window counts as already spent."""
+ return {}
+
async def wait(self, job_id, *, deadline_s) -> JobStatus:
while self.status(job_id).state == "running":
self.changed.clear()
diff --git a/tests/agent_core/tools/test_bash.py b/tests/agent_core/tools/test_bash.py
index 78709af25..5ff04206b 100644
--- a/tests/agent_core/tools/test_bash.py
+++ b/tests/agent_core/tools/test_bash.py
@@ -16,7 +16,7 @@ import pytest
from core.agent_core.agent.jobs import TrackingJobHost
from core.agent_core.cancel import CancelToken
import core.agent_core.tools.bash as bash_module
-from core.agent_core.tools.bash import BashTool, settle_bash_call
+from core.agent_core.tools.bash import BashTool, settle_bash_call, settle_in_foreground
import core.agent_core.tools.jobs as jobs_module
from core.agent_core.tools.jobs import LocalJobHost
from core.agent_core.tools.shell import Shell
@@ -627,3 +627,32 @@ async def test_a_timeout_holds_across_handover_and_a_restart(tmp_path, make_ctx)
settled = await settle_bash_call(restarted, instance())
assert result_text(settled) == "begun\n\n\nCommand timed out after 60 seconds"
assert not os.path.exists(os.path.join(restarted.job_dir(job_id), "exit"))
+
+
+async def test_a_stop_during_a_failed_recovered_handover_ends_the_command_before_another_attempt():
+ """MESSAGE-DELIVERY-330: a restart rerun's wait checks its Stop before each handover, as the
+ live call does, so a Stop that arrives while the first attempt fails ends the command
+ instead of a second attempt leaving it running as a Watch."""
+ from core.agent_core.messages import ToolCallBlock
+ from core.agent_core.tools.base import JobStatus
+ from tests.agent_core.fakes import FakeJobHost
+
+ jobs = FakeJobHost()
+ jobs.states["job_1"] = JobStatus("running")
+ stop, attempts = asyncio.Event(), []
+ hand_over = jobs.hand_over
+
+ async def stopped_while_locked(job_id):
+ attempts.append(job_id)
+ if len(attempts) == 1:
+ stop.set()
+ raise RuntimeError("the Watch store is locked")
+ return await hand_over(job_id)
+
+ jobs.hand_over = stopped_while_locked
+ call = ToolCallBlock(id="call_bash", name="bash", arguments={"command": "sleep 60"})
+
+ status, watch_id = await settle_in_foreground(jobs, call, "job_1", stop=stop)
+
+ assert attempts == ["job_1"] and watch_id is None and jobs.watches == {}
+ assert jobs.killed == ["job_1"] and status.state != "running"
diff --git a/tests/backend_failure_retry_helpers.py b/tests/backend_failure_retry_helpers.py
index 9326e9284..1099697ec 100644
--- a/tests/backend_failure_retry_helpers.py
+++ b/tests/backend_failure_retry_helpers.py
@@ -1,5 +1,7 @@
"""Real durable failed-Turn fixtures shared by retry boundary tests."""
+import asyncio
+
from storage import message_deliveries as deliveries, messages_service
@@ -108,3 +110,75 @@ def reserve_failure_retry(conn, notice):
message_kind="quick_reply",
),
)
+
+
+def previous_process(step):
+ """What the previous process committed for the Turn before the restart, then startup T2.
+
+ ``model``: the input consumed, the first model call cut before anything committed;
+ ``tool``: a ``write`` call cut mid-call (T2 gives it the interrupted result);
+ ``handover``: a bash command still running (T2 hands it to a Watch);
+ ``partial``: the text a failed stream carried, committed as a non-final response;
+ ``running``: a bash command left running, its call still open (startup T2 has not run);
+ ``final``: the Turn's final reply;
+ ``silent_final``: delivered narration, then a final with nothing to show;
+ ``failed_final``: delivered narration, then a final that fails with nothing to show.
+ """
+ from core.agent_core.agent.recovery import settle_open_calls
+ from core.agent_core.messages import ToolCallBlock, UserMessage, text
+ from core.agent_core.tools.base import CallInstance, JobStatus, ToolResult
+ from storage.agent_transcript import SQLiteTranscriptStore
+ from tests.agent_core.fakes import FakeJobHost, assistant
+
+ def write(engine, session_id, turn_id):
+ with engine.connect() as conn:
+ input_id = deliveries.delivery_for_turn(conn, turn_id)["message_id"]
+ store, jobs = SQLiteTranscriptStore(engine), FakeJobHost()
+ jobs.states["job_1"] = JobStatus("running")
+ call = (
+ ToolCallBlock(id="call_bash", name="bash", arguments={"command": "sleep 60; echo done >> s6.txt"})
+ if step in {"handover", "running"}
+ else ToolCallBlock(id="call_write", name="write", arguments={"path": "notes.md", "content": "x"})
+ )
+
+ async def handed_over(_call, _job_id, _status, watch_id):
+ return ToolResult((text(f"Command is still running and is now Watch {watch_id}."),))
+
+ async def previous_process_then_t2():
+ await store.consume_input(session_id, input_id, UserMessage((text("run it"),)))
+ if step == "model":
+ return
+ if step == "final":
+ await store.append_response(session_id, assistant("全部完成"), final=True)
+ return
+ if step in {"silent_final", "failed_final"}:
+ # The narration the dispatcher delivered and persisted, then the silent final.
+ with engine.begin() as conn:
+ messages_service.append(
+ conn, scope_id=None, session_id=session_id, platform="avibe", author="agent",
+ source="agent", message_type="interim",
+ text="日志读完了。\n缓存被两个写者共享。\n锁已加上。\n已验证。",
+ )
+ # A final whose reply is all ``<silent>`` replies and shows nothing; one that does
+ # not reply at all (an empty response) failed.
+ await store.append_response(
+ session_id, assistant(""), final=True, replied=step == "silent_final"
+ )
+ return
+ if step == "partial":
+ await store.append_response(session_id, assistant("先读完日志", stop_reason="error"), final=False)
+ return
+ response = await store.append_response(session_id, assistant("", calls=(call,)), final=False)
+ if step == "running":
+ return
+ instance = CallInstance(session_id, response.row_id, response.context_seq, call.id)
+ await settle_open_calls(
+ session_id=session_id, store=store, jobs=jobs,
+ job_ids={instance: "job_1"} if step == "handover" else {}, render_result=handed_over,
+ )
+
+ asyncio.run(previous_process_then_t2())
+ if step == "handover":
+ assert jobs.watches == {"job_1": "watch_job_1"}
+
+ return write
diff --git a/tests/scenarios/message_delivery/catalog.yaml b/tests/scenarios/message_delivery/catalog.yaml
index 8ba7e9c98..e58a90f41 100644
--- a/tests/scenarios/message_delivery/catalog.yaml
+++ b/tests/scenarios/message_delivery/catalog.yaml
@@ -374,11 +374,13 @@ scenarios:
layer: contract
test: tests/test_opencode_steer_superseded_reply.py::test_steer_keeps_superseded_completed_reply
- id: MESSAGE-DELIVERY-322
- name: A restart notice names the Watch that continues the interrupted command and offers no Retry
+ name: A restart notice names the Watch that continues the interrupted command and offers no Retry; a Stop while it waits for that Watch ends the command
status: covered
kind: recovery
layer: contract
- test: tests/test_backend_failure_retry.py::test_restart_notice_names_the_watch_running_the_command_and_offers_no_retry
+ test:
+ - tests/test_backend_failure_retry.py::test_restart_notice_names_the_watch_running_the_command_and_offers_no_retry
+ - tests/test_backend_failure_retry.py::test_a_restart_notice_waits_until_its_running_command_is_handed_over
- id: MESSAGE-DELIVERY-323
name: A scheduled Harness P1 input queues behind a Turn a person's input is part of and runs as its own Turn
status: covered
@@ -419,5 +421,67 @@ scenarios:
test:
- tests/test_vibey_agent.py::test_a_short_reply_a_steer_superseded_stays_a_visible_reply
- tests/test_agent_transcript.py::test_a_reply_pending_input_superseded_commits_as_a_visible_reply
+ - id: MESSAGE-DELIVERY-327
+ name: Startup reruns a Vibey Turn a restart cut in place from its transcript, once its surface can deliver, without a notice
+ status: covered
+ kind: recovery
+ layer: contract
+ test:
+ - tests/test_backend_failure_retry.py::test_restart_reruns_a_cut_vibey_turn_in_place_without_a_notice
+ - tests/test_backend_failure_retry.py::test_a_rerun_that_exits_before_dispatching_fails_its_turn
+ - tests/test_backend_failure_retry.py::test_a_rerun_whose_setup_fails_is_retried_not_settled
+ - tests/test_backend_failure_retry.py::test_concurrent_recovery_passes_rerun_a_cut_turn_once
+ - tests/test_message_handler_typing.py::MessageHandlerTypingTests::test_a_restart_rerun_runs_with_the_request_its_turn_was_accepted_with
+ - tests/test_backend_failure_retry.py::test_a_cut_vibey_turn_that_no_longer_qualifies_keeps_the_restart_notice
+ - tests/test_processing_indicator_reaction.py::QueuedReactionLifecycleTests::test_a_restart_rerun_retires_the_running_reaction_its_dead_process_may_have_left
+ - tests/test_agent_transcript.py::test_the_restart_reads_of_a_turns_context_tail_stay_bounded
+ - id: MESSAGE-DELIVERY-328
+ name: A Vibey Agent Run cut by a restart, steered-in Runs included (the Turn waiting open until their receipts materialize), is left at shutdown and settles once from the rerun's result, the answer after its command
+ status: covered
+ kind: recovery
+ layer: contract
+ test:
+ - tests/test_harness_failure_retry.py::test_a_vibey_agent_run_cut_by_a_restart_settles_from_the_continued_work
+ - tests/test_vibey_agent.py::test_a_restart_rerun_waits_for_its_surviving_command_as_the_live_call_would
+ - id: MESSAGE-DELIVERY-329
+ name: A cut Vibey Turn that no longer qualifies (Stop, Agent off Vibey, archived, the Turn's own Agent no longer on Vibey, the third restart in a row without a model response) keeps the restart notice; a committed response resets the streak
+ status: covered
+ kind: recovery
+ layer: contract
+ test: tests/test_backend_failure_retry.py::test_a_cut_vibey_turn_that_no_longer_qualifies_keeps_the_restart_notice
+ - id: MESSAGE-DELIVERY-330
+ name: A command still running at restart is left to the rerun (across repeated restarts), which waits for it within its foreground window from launch, hands it over only after, ends it on a Stop, and never starts it again
+ status: covered
+ kind: recovery
+ layer: contract
+ test:
+ - tests/test_backend_failure_retry.py::test_restart_reruns_a_cut_vibey_turn_in_place_without_a_notice
+ - tests/test_backend_failure_retry.py::test_a_turn_waiting_for_its_steer_keeps_its_running_command_for_the_rerun
+ - tests/test_backend_failure_retry.py::test_a_stop_ends_the_command_of_a_turn_held_for_its_rerun_at_once
+ - tests/test_backend_failure_retry.py::test_a_rerun_cancelled_before_its_adapter_owns_the_command_still_ends_it
+ - tests/test_vibey_agent.py::test_a_restart_rerun_waits_for_its_surviving_command_as_the_live_call_would
+ - tests/agent_core/tools/test_bash.py::test_a_stop_during_a_failed_recovered_handover_ends_the_command_before_another_attempt
+ - id: MESSAGE-DELIVERY-331
+ name: Input queued while the service was down runs after the rerun of the Turn a restart cut
+ status: covered
+ kind: concurrency
+ layer: contract
+ test: tests/test_backend_failure_retry.py::test_restart_rerun_runs_before_input_queued_while_the_service_was_down
+ - id: MESSAGE-DELIVERY-332
+ name: A Vibey run that fails because the service is stopping (a model call, or a rerun's preflight) leaves its Turn open for its restart rerun
+ status: covered
+ kind: recovery
+ layer: contract
+ test: tests/test_vibey_agent.py::test_a_run_the_service_stop_cut_leaves_its_turn_open_for_its_restart_rerun
+ - id: MESSAGE-DELIVERY-333
+ name: The rerun of a Turn whose final was committed ends with it through the live end, with no model call; a final not yet delivered is sent once, one already delivered is not sent again, a silent final promotes the Turn's earlier visible text, and the Turn stays open across another restart until then
+ status: covered
+ kind: recovery
+ layer: contract
+ test:
+ - tests/test_vibey_agent.py::test_a_restart_rerun_ends_with_the_final_its_cut_run_committed
+ - tests/test_backend_failure_retry.py::test_restart_reruns_a_cut_vibey_turn_in_place_without_a_notice
+ - tests/test_backend_failure_retry.py::test_a_cut_vibey_turn_that_no_longer_qualifies_keeps_the_restart_notice
+ - tests/test_agent_transcript.py::test_the_restart_reads_of_a_turns_context_tail_stay_bounded
next_priority:
- - MESSAGE-DELIVERY-327
+ - MESSAGE-DELIVERY-334
diff --git a/tests/test_agent_transcript.py b/tests/test_agent_transcript.py
index ddcd45b6a..0c8f48044 100644
--- a/tests/test_agent_transcript.py
+++ b/tests/test_agent_transcript.py
@@ -10,6 +10,7 @@ from __future__ import annotations
import asyncio
import json
+import re
import sqlite3
import threading
@@ -538,6 +539,92 @@ async def test_a_reply_pending_input_superseded_commits_as_a_visible_reply(engin
assert [entry.row_id for entry in await store.load("ses_main") if entry.kind == "response"] == [reply.row_id]
+async def test_the_restart_reads_of_a_turns_context_tail_stay_bounded(engine) -> None:
+ """MESSAGE-DELIVERY-327 / MESSAGE-DELIVERY-333: restart recovery decides each cut Turn
+ (``restart_resume_plan``, its restart streak included) and releases its runtime generation to
+ rerun it; the rerun finds the final it committed, and its end asks whether that final was
+ delivered and what a silent final leaves as the result, the last two on the controller loop.
+ Every read is an indexed SEARCH keyed by the Turn, a row id, the Session's context tail, or the
+ Turn's own time window: no scan, and a temporary sort only over the Turn's own Deliveries, never
+ over the growing history."""
+ from sqlalchemy import event
+
+ from core.message_mirror import delivered_committed_row, resumed_turn_text
+ from core.restart_resume import restart_resume_plan, restart_streak
+ from modules.im.base import MessageContext
+ from storage.agent_transcript import committed_final, last_turn_text
+ from storage.db import get_cached_sqlite_engine
+
+ with engine.begin() as conn:
+ home = _scope(conn, "C-home")
+ _session(conn, "ses_main", home)
+ first = _accepted_turn(conn, "ses_main", home, "看一下目录", "turn_main")
+ _session(conn, "ses_cut", home)
+ cut = _accepted_turn(conn, "ses_cut", home, "跑一下测试", "turn_cut")
+ store = SQLiteTranscriptStore(engine)
+ await store.consume_input("ses_main", first, _user("看一下目录"))
+ final = await store.append_response("ses_main", _assistant("两个文件。"), final=True, turn_id="turn_main")
+ await store.consume_input("ses_cut", cut, _user("跑一下测试"))
+ await store.append_response("ses_cut", _assistant("先跑", call_id="call_1"), final=False, turn_id="turn_cut")
+ statements: list[tuple[str, object]] = []
+
+ def capture(_conn, _cursor, statement, parameters, _context, _executemany):
+ statements.append((statement, parameters))
+
+ rerun = MessageContext(
+ user_id="u1", channel_id="C-home", platform="slack",
+ platform_specific={"vibey_resume": True, "agent_session_id": "ses_main", "turn_token": "turn_main"},
+ )
+ cached = get_cached_sqlite_engine()
+ event.listen(engine, "before_cursor_execute", capture)
+ event.listen(cached, "before_cursor_execute", capture)
+ try:
+ with engine.connect() as conn:
+ # Both Turns rerun: one ends with the final it committed, one continues.
+ assert restart_resume_plan(conn, message_deliveries.get_turn(conn, "turn_main")) == "rerun"
+ assert committed_final(conn, "ses_main", "turn_main").row_id == final.row_id
+ cut_turn = message_deliveries.get_turn(conn, "turn_cut")
+ assert restart_resume_plan(conn, cut_turn) == "rerun"
+ assert committed_final(conn, "ses_cut", "turn_cut") is None
+ # A silent final's result: the Turn's last narration or superseded reply.
+ last_turn_text(conn, "ses_main", since="2026-01-01T00:00:00.000000Z")
+ # The rerun's end, on the loop: the final's delivery, and a silent final's result.
+ assert delivered_committed_row(rerun, final.row_id, outward=True) is None
+ assert resumed_turn_text(rerun) is None
+ with engine.begin() as conn:
+ # The rerun's writer transaction: the streak, and the released generation.
+ streak, response_seq = restart_streak(conn, cut_turn)
+ assert message_deliveries.rotate_restart_generation(
+ conn, "turn_cut", expected_version=int(cut_turn["version"]), streak=streak, response_seq=response_seq
+ )
+ finally:
+ event.remove(engine, "before_cursor_execute", capture)
+ event.remove(cached, "before_cursor_execute", capture)
+
+ raw = sqlite3.connect(str(get_sqlite_state_path()))
+ try:
+ plans = [[row[-1] for row in raw.execute("EXPLAIN QUERY PLAN " + sql, params)] for sql, params in statements]
+ finally:
+ raw.close()
+ turn_keys = ("(turn_id=?)", "(current_target_turn_id=?)")
+ bounded = re.compile(
+ r"SEARCH \w+ USING (?:COVERING )?INDEX \w+ \((?:id=\?|delivery_id=\?|turn_id=\?|current_target_turn_id=\?"
+ r"|session_id=\? AND context_seq>\?|session_id=\? AND type=\? AND created_at>\?)\)"
+ )
+ for (sql, _params), plan in zip(statements, plans):
+ searches = [step for step in plan if step.startswith(("SEARCH", "SCAN"))]
+ if not searches and sql.lstrip().upper().startswith("UPDATE"):
+ continue # a CAS write keyed by the row id, whose plan names no table read
+ assert searches and all(bounded.fullmatch(step) for step in searches), (sql, plan)
+ if any("TEMP B-TREE" in step for step in plan):
+ assert all(step.endswith(turn_keys) for step in searches), (sql, plan)
+ # The steer the Turn may still wait for is found by the Turn it targets.
+ assert any(
+ "current_target_turn_id" in sql and "steering" in str(params) and bounded.fullmatch(plan[0])
+ for (sql, params), plan in zip(statements, plans)
+ )
+
+
async def test_a_call_instance_is_settled_once_whoever_writes(engine) -> None:
with engine.begin() as conn:
home = _scope(conn, "C-home")
diff --git a/tests/test_backend_failure_retry.py b/tests/test_backend_failure_retry.py
index 398f9670a..75354cee1 100644
--- a/tests/test_backend_failure_retry.py
+++ b/tests/test_backend_failure_retry.py
@@ -3,29 +3,58 @@ from __future__ import annotations
import asyncio
import json
from concurrent.futures import ThreadPoolExecutor
+from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
-from sqlalchemy import update
+from sqlalchemy import select, update
from core.session_turns import DeliveryRequest
from modules.agents.catalog import AGENT_BACKENDS
from storage import message_deliveries, messages_service
from storage.agent_session_rows import reserve_write_lock
from storage.db import create_sqlite_engine
-from storage.models import agent_sessions, messages, session_turns
+from storage.models import (
+ agent_events,
+ agent_sessions,
+ message_deliveries as message_deliveries_table,
+ messages,
+ session_turns,
+)
+from tests.backend_failure_retry_helpers import previous_process as _previous_process
from tests.backend_failure_retry_helpers import reserve_failure_retry, seed_failed_notice
from tests.test_session_delivery_fsm import _context, _fsm_schema_template, managers # noqa: F401
from tests.test_ui_session_stream import _accepted_dispatch, _make_session, isolated_state # noqa: F401
from tests.ui_server_test_helpers import csrf_headers
-def _restart_failure_notice(tmp_path, *, backend, delivery="ready", before_restart=None):
- """Recover an accepted Turn through the real notice dispatcher and storage.
+def _with_vibey_recovery(controller, engine, tmp_path, jobs):
+ """The registered Vibey adapter over ``jobs`` and its single recovery owner, as at startup."""
+ from core.agent_core.messages import text
+ from core.agent_core.tools.base import ToolResult
+ from core.agent_core.tools.bash import settle_in_foreground
+ from modules.agents.vibey import VibeyAgent
+ from modules.agents.vibey.recovery import VibeyRecovery
+ from modules.agents.vibey.tools import ToolSuite
- ``before_restart(engine, session_id, turn_id)`` writes what the previous process
- and the startup steps before Turn recovery left behind.
- """
+ async def handed_over(_call, _job_id, _status, watch_id):
+ return ToolResult((text(f"Command is still running and is now Watch {watch_id}."),))
+
+ suite = ToolSuite(
+ jobs=jobs, create_tools=lambda *_args: [], render_recovered=handed_over,
+ find_job=lambda call: "job_1" if call.tool_call_id == "call_bash" else None,
+ wait_recovered=lambda call, job_id, stop: settle_in_foreground(jobs, call, job_id, stop=stop),
+ )
+ controller.agent_service.agents["vibey"] = VibeyAgent(
+ controller, engine=engine, tool_suite=suite, state_dir=tmp_path / "agent_core"
+ )
+ controller.vibey_recovery = VibeyRecovery(controller)
+
+
+def _accepted_turn(tmp_path, *, backend, delivery="ready", native_message_id=None, jobs=None):
+ """One accepted Turn through the real Delivery owner, notice dispatcher, and storage.
+
+ With ``jobs``, startup also runs Vibey's own recovery pass first, over those jobs."""
from core.message_dispatcher import ConsolidatedMessageDispatcher
from core.session_turns import SessionTurnManager
from tests.scenario_harness.message_delivery import MessageDeliveryController
@@ -46,39 +75,76 @@ def _restart_failure_notice(tmp_path, *, backend, delivery="ready", before_resta
controller.session_turns = manager
dispatcher = ConsolidatedMessageDispatcher(controller)
controller.emit_agent_message = dispatcher.emit_agent_message
+ if jobs is not None:
+ _with_vibey_recovery(controller, engine, tmp_path, jobs)
if delivery == "send_failed":
controller.im_client.send_message = AsyncMock(side_effect=[OSError("transport offline"), "msg-recovered"])
- starts = []
+ starts, contexts = [], []
async def accepted(sid, ctx, text, **_kwargs):
starts.append((sid, text))
+ contexts.append(ctx)
_bind_test_native_start(engine, ctx)
manager._run = accepted
original = asyncio.run(manager.deliver(
- DeliveryRequest(session_id=session_id, priority="p3", content="完成剩余工作,不要重复提交"),
+ DeliveryRequest(
+ session_id=session_id, priority="p3", content="完成剩余工作,不要重复提交",
+ **({"platform": "slack", "native_message_id": native_message_id} if native_message_id else {}),
+ ),
context=context(session_id),
))
starts.clear()
+ contexts.clear()
+ return SimpleNamespace(
+ manager=manager, engine=engine, session_id=session_id, turn_id=str(original.turn_id),
+ context=context, starts=starts, contexts=contexts, delivery=delivery,
+ )
+
+
+def _restart(turn, before_restart=None):
+ """``before_restart(engine, session_id, turn_id)`` writes what the previous process
+ and the startup steps before Turn recovery left behind."""
+ import secrets
+
+ from core import restart_resume
+
if before_restart is not None:
- before_restart(engine, session_id, str(original.turn_id))
- manager._active_identity = lambda *_args: None
- manager._transport_can_deliver = lambda _platform: delivery != "unready"
- asyncio.run(manager.recover_durable_delivery_state(session_id, service_restart=True))
- assert starts == []
- if delivery != "ready":
- with engine.connect() as conn:
- assert not any(
- row["type"] == "notify"
- for row in messages_service.list_session_messages(conn, session_id=session_id)["messages"]
- )
- assert asyncio.run(manager.notify_transport_ready("avibe")) == 1
+ before_restart(turn.engine, turn.session_id, turn.turn_id)
+ restart_resume.PROCESS_TOKEN = secrets.token_hex(8) # a new process starts
+ turn.manager._active_identity = lambda *_args: None
+ turn.manager._transport_can_deliver = lambda _platform: turn.delivery != "unready"
+ recovery = getattr(turn.manager.controller, "vibey_recovery", None)
+
+ async def startup():
+ # The controller's order: Vibey's T2 pass, then Turn recovery of every Session.
+ if recovery is not None:
+ assert await recovery.start() == []
+ await turn.manager.recover_durable_delivery_state(service_restart=True)
+
+ asyncio.run(startup())
+
+
+def _notices(engine, session_id):
with engine.connect() as conn:
- turn = message_deliveries.get_turn(conn, str(original.turn_id))
- notices = [
+ return [
row for row in messages_service.list_session_messages(conn, session_id=session_id)["messages"]
if row["type"] == "notify"
]
+
+
+def _restart_failure_notice(tmp_path, *, backend, delivery="ready", before_restart=None, jobs=None):
+ """Recover an accepted Turn through the real notice dispatcher and storage."""
+ cut = _accepted_turn(tmp_path, backend=backend, delivery=delivery, jobs=jobs)
+ manager, engine, session_id = cut.manager, cut.engine, cut.session_id
+ _restart(cut, before_restart)
+ assert cut.starts == []
+ if delivery != "ready":
+ assert _notices(engine, session_id) == []
+ assert asyncio.run(manager.notify_transport_ready("avibe")) == 1
+ with engine.connect() as conn:
+ turn = message_deliveries.get_turn(conn, cut.turn_id)
+ notices = _notices(engine, session_id)
assert turn["terminal_evidence_kind"] == "restart_runtime_missing"
assert turn["start_receipt_outcome"] == "accepted"
assert len(notices) == 1
@@ -149,42 +215,546 @@ def test_restart_notice_retries_through_web_and_controller(isolated_state, tmp_p
assert reloaded["content"]["failure_retry"]["state"] == "accepted"
-def _hand_over_the_open_command(engine, session_id, turn_id):
- """The restart left the Turn's bash command running; startup recovery made it a Watch (T2)."""
- from core.agent_core.agent.recovery import settle_open_calls
- from core.agent_core.messages import ToolCallBlock, UserMessage, text
- from core.agent_core.tools.base import CallInstance, JobStatus, ToolResult
- from storage.agent_transcript import SQLiteTranscriptStore
- from tests.agent_core.fakes import FakeJobHost, assistant
+def _request_stop(engine, _session_id, turn_id):
+ """The user pressed Stop as the service went down."""
+ with engine.begin() as conn:
+ turn = message_deliveries.get_turn(conn, turn_id)
+ assert message_deliveries.cas_turn(
+ conn, turn_id, expected_version=int(turn["version"]), expected_states=("active",),
+ values={"control_mode": "stop_only"},
+ )
+
+
+def _then(*steps):
+ def run(engine, session_id, turn_id):
+ for step in steps:
+ step(engine, session_id, turn_id)
+ return run
+
+
+def _deliveries(engine, session_id):
with engine.connect() as conn:
- input_id = message_deliveries.delivery_for_turn(conn, turn_id)["message_id"]
- store, jobs = SQLiteTranscriptStore(engine), FakeJobHost()
+ return conn.execute(
+ select(message_deliveries_table.c.id).where(message_deliveries_table.c.session_id == session_id)
+ ).scalars().all()
+
+
+def _respond(engine, session_id, turn_id):
+ """The rerun answered: one more model response (a tool call and its result) in the Turn."""
+ from core.agent_core.messages import ToolCallBlock, ToolResultMessage, text
+ from storage.agent_transcript import SQLiteTranscriptStore
+ from tests.agent_core.fakes import assistant
+
+ store = SQLiteTranscriptStore(engine)
+
+ async def respond():
+ call = ToolCallBlock(id="call_read", name="read", arguments={"path": "notes.md"})
+ await store.append_response(session_id, assistant("", calls=(call,)), final=False, turn_id=turn_id)
+ await store.append_tool_result(
+ session_id, ToolResultMessage(tool_call_id=call.id, tool_name=call.name, content=(text("x"),)), details={}
+ )
+
+ asyncio.run(respond())
+
+
[email protected]("step", ["model", "tool", "handover", "partial", "running", "final"])
+def test_restart_reruns_a_cut_vibey_turn_in_place_without_a_notice(isolated_state, tmp_path, step):
+ """MESSAGE-DELIVERY-327 / MESSAGE-DELIVERY-330 / MESSAGE-DELIVERY-333: the same Turn reruns as a
+ transcript resume of itself, with the Agent it ran with: no Delivery, no successor Turn, no
+ notice. A committed handover does not stop it, a command still running is left to it, not
+ handed over at startup, a failed stream's partial text is not a final reply, and a committed
+ final reruns too, for its own end to deliver it. The Agent and request are the ones its
+ native start recorded, the Agent found by id, even when it was cut before its first response or
+ renamed since."""
+ from core.agent_core.tools.base import JobStatus
+ from tests.agent_core.fakes import FakeJobHost
+
+ jobs = None
+ if step == "running":
+ jobs = FakeJobHost()
+ jobs.states["job_1"] = JobStatus("running")
+ from core.vibe_agents import VibeAgent
+
+ cut = _accepted_turn(tmp_path, backend="vibey", jobs=jobs)
+ before = _deliveries(cut.engine, cut.session_id)
+ agents = {
+ f"agt_{name}": VibeAgent(id=f"agt_{name}", name=name, normalized_name=name, backend="vibey")
+ for name in ("worker", "reviewer")
+ }
+ cut.manager.controller.vibe_agent_store = SimpleNamespace(
+ get=lambda name: next((agent for agent in agents.values() if agent.name == name), None),
+ get_by_id=agents.get,
+ )
+ # The Vibey run's native start, as its adapter accepts it: it names the Agent it runs with.
+ cut.manager.controller._session_id_from_context = lambda ctx: (ctx.platform_specific or {}).get(
+ "agent_session_id"
+ )
+ accepted = cut.context(cut.session_id)
+ execution = {
+ "agent_id": "agt_worker", "agent_name": "worker", "model": "model-accepted",
+ "reasoning_effort": "high", "system_prompt": "Review carefully.",
+ }
+ accepted.platform_specific.update(
+ {"turn_token": cut.turn_id, "agent_session_id": cut.session_id, "vibey_execution": execution}
+ )
+ cut.manager.on_native_start(
+ accepted, backend="vibey", runtime_key=f"runtime:{cut.turn_id}", runtime_turn_id=f"runtime-turn:{cut.turn_id}"
+ )
+
+ def switch_agent(engine, session_id, _turn_id):
+ # The Session moved to another Vibey Agent while the Turn ran: that reaches the next Turn.
+ # The Turn's own Agent was renamed meanwhile; its id still finds it.
+ with engine.begin() as conn:
+ conn.execute(update(agent_sessions).where(agent_sessions.c.id == session_id).values(agent_name="reviewer"))
+ agents["agt_worker"] = VibeAgent(id="agt_worker", name="builder", normalized_name="builder", backend="vibey")
+
+ rotate, rotated_on_loop = message_deliveries.rotate_restart_generation, []
+
+ def rotate_where(*args, **kwargs):
+ try:
+ asyncio.get_running_loop()
+ rotated_on_loop.append(True)
+ except RuntimeError:
+ rotated_on_loop.append(False)
+ return rotate(*args, **kwargs)
+
+ message_deliveries.rotate_restart_generation = rotate_where
+ try:
+ _restart(cut, _then(_previous_process(step), switch_agent))
+ finally:
+ message_deliveries.rotate_restart_generation = rotate
+ # Its writer transaction takes SQLite's write lock: off the controller loop.
+ assert rotated_on_loop == [False]
+
+ if jobs is not None:
+ assert jobs.watches == {}
+ with cut.engine.connect() as conn:
+ assert not conn.execute(
+ select(agent_events.c.id).where(
+ agent_events.c.session_id == cut.session_id, agent_events.c.event_type == "tool_result"
+ )
+ ).first()
+ # Started once, as a transcript resume of the cut Turn itself: no input, nothing replayed.
+ assert cut.starts == [(cut.session_id, "")]
+ [started] = cut.contexts
+ assert started.platform_specific["turn_token"] == cut.turn_id
+ assert started.platform_specific["vibey_resume"] is True
+ assert started.platform_specific["vibey_source_turn_id"] == cut.turn_id
+ assert started.platform_specific["delivery_id"] == before[0]
+ assert (started.platform_specific["vibe_agent_name"], started.platform_specific["vibe_agent_id"]) == (
+ "builder", "agt_worker",
+ )
+ # With the request it was accepted with: Agent settings edited since reach the next Turn.
+ assert started.platform_specific["vibey_execution"] == execution
+ assert _deliveries(cut.engine, cut.session_id) == before
+ with cut.engine.connect() as conn:
+ turn = message_deliveries.get_turn(conn, cut.turn_id)
+ users = [
+ row for row in messages_service.list_session_messages(conn, session_id=cut.session_id)["messages"]
+ if row["type"] == "user"
+ ]
+ assert (turn["state"], turn["start_receipt_outcome"]) == ("active", "accepted")
+ assert json.loads(turn["start_receipt_json"])["restart_streak"] == 1
+ assert _notices(cut.engine, cut.session_id) == []
+ assert [row["text"] for row in users] == ["完成剩余工作,不要重复提交"]
+
+
+def _switch_off_vibey(engine, session_id, _turn_id):
+ with engine.begin() as conn:
+ conn.execute(update(agent_sessions).where(agent_sessions.c.id == session_id).values(agent_backend="claude"))
+
+
+def _archive(engine, session_id, _turn_id):
+ with engine.begin() as conn:
+ conn.execute(update(agent_sessions).where(agent_sessions.c.id == session_id).values(status="archived"))
+
+
+ "case",
+ [
+ "qualifies", "unready", "final_again", "stop", "switched", "archived", "agent_gone", "agent_unknown",
+ "bound", "reset",
+ ],
+)
+def test_a_cut_vibey_turn_that_no_longer_qualifies_keeps_the_restart_notice(isolated_state, tmp_path, case):
+ """MESSAGE-DELIVERY-329: Stop, an Agent off Vibey, an archived Session, the Agent that ran the
+ Turn no longer on Vibey or named nowhere (another Agent never continues its work), or the third
+ restart in a row with no committed model response keep today's settlement and notice. A response
+ the rerun committed resets that streak. A Turn whose final was committed stays open until its
+ rerun ends it, so another restart before then reruns it again (MESSAGE-DELIVERY-333). A rerun
+ waits for the surface it delivers to, as its notice would (MESSAGE-DELIVERY-327)."""
+ from core.vibe_agents import VibeAgent
+
+ cut = _accepted_turn(tmp_path, backend="vibey", delivery="unready" if case == "unready" else "ready")
+ if case in {"agent_gone", "agent_unknown"}:
+ # The Turn ran with ``worker``, archived since; the Session now selects ``reviewer``.
+ agents = {
+ "agt_worker": VibeAgent(
+ id="agt_worker", name="worker", normalized_name="worker", backend="vibey",
+ archived_at="2026-10-18T00:00:00Z",
+ ),
+ "agt_reviewer": VibeAgent(id="agt_reviewer", name="reviewer", normalized_name="reviewer", backend="vibey"),
+ }
+ cut.manager.controller.vibe_agent_store = SimpleNamespace(
+ get=lambda name: next((agent for agent in agents.values() if agent.name == name), None),
+ get_by_id=agents.get,
+ )
+ before = {
+ "stop": _then(_previous_process("tool"), _request_stop),
+ "switched": _then(_previous_process("tool"), _switch_off_vibey),
+ "archived": _then(_previous_process("tool"), _archive),
+ # An older release accepted it (no recorded request) and it was cut before any response.
+ "agent_unknown": _previous_process("model"),
+ "final_again": _previous_process("final"),
+ }.get(case, _previous_process("tool"))
+ _restart(cut, before)
+ if case in {"bound", "reset"}:
+ # The rerun dies before the model answers, twice; the third restart is the bound.
+ _restart(cut, _respond if case == "reset" else None)
+ _restart(cut)
+ if case == "final_again":
+ # The process dies again before the rerun ended the Turn.
+ _restart(cut)
+ if case == "unready":
+ # Held, open, until its transport can deliver; the ready callback starts it.
+ assert cut.starts == [] and _notices(cut.engine, cut.session_id) == []
+ cut.manager._transport_can_deliver = lambda _platform: True
+ assert asyncio.run(cut.manager.notify_transport_ready("avibe")) == 0
+
+ reran = case in {"qualifies", "unready", "final_again", "reset"}
+ assert len(cut.starts) == {"qualifies": 1, "unready": 1, "final_again": 2, "reset": 3, "bound": 2}.get(case, 0)
+ with cut.engine.connect() as conn:
+ turn = message_deliveries.get_turn(conn, cut.turn_id)
+ if reran:
+ assert turn["state"] == "active"
+ assert json.loads(turn["start_receipt_json"])["restart_streak"] == (2 if case in {"reset", "final_again"} else 1)
+ assert _notices(cut.engine, cut.session_id) == []
+ return
+ assert (turn["terminal_outcome"], turn["settled_by"]) == ("failed", "no_terminal_result")
+ assert turn["terminal_evidence_kind"] == "restart_runtime_missing"
+ if case != "archived":
+ [notice] = _notices(cut.engine, cut.session_id)
+ assert notice["metadata"]["turn_id"] == cut.turn_id
+
+
[email protected]("then", ["rerun", "stopped"])
+def test_a_turn_waiting_for_its_steer_keeps_its_running_command_for_the_rerun(
+ isolated_state, tmp_path, monkeypatch, then
+):
+ """MESSAGE-DELIVERY-330: while the Turn waits open for a steer receipt startup could not
+ materialize, its running command stays with it: the retry only recovers again, and the rerun
+ gets the command, with no handover in between. A Turn a Stop was asked of never reruns, so it
+ does not wait for the steer: it settles with the notice at once and its command ends
+ (MESSAGE-DELIVERY-329)."""
+ from core.agent_core.tools.base import JobStatus
+ from core.session_turns import SessionTurnManager
+ from tests.agent_core.fakes import FakeJobHost
+
+ jobs = FakeJobHost()
+ jobs.states["job_1"] = JobStatus("running")
+ monkeypatch.setattr(SessionTurnManager, "RESTART_RECOVERY_RETRY_DELAYS", (0.05,))
+ monkeypatch.setattr(SessionTurnManager, "RESTART_RECOVERY_RETRY_PERIOD", 0.05)
+ cut = _accepted_turn(tmp_path, backend="vibey", jobs=jobs)
+ _previous_process("running")(cut.engine, cut.session_id, cut.turn_id)
+ with cut.engine.begin() as conn:
+ # A steer the dead run accepted, its receipt recorded and the Delivery not materialized.
+ steer = message_deliveries.insert_delivery(
+ conn, delivery_id=message_deliveries.new_delivery_id(), session_id=cut.session_id, priority="p1",
+ state="reserved", dispatch_text="也看一下告警",
+ snapshot=message_deliveries.message_snapshot(
+ scope_id=None, session_id=cut.session_id, platform="avibe", author="user", source="user",
+ text="也看一下告警",
+ ),
+ )
+ turn = message_deliveries.get_turn(conn, cut.turn_id)
+ opened = message_deliveries.open_steer_attempt(
+ conn, steer["id"], expected_version=int(steer["version"]), turn_id=cut.turn_id,
+ attempt_id="atm_steer", expected_native_turn_id=turn["native_turn_id"],
+ )
+ assert message_deliveries.mark_attempt_receipt(
+ conn, steer["id"], expected_version=int(opened["version"]), outcome="accepted", receipt={},
+ )
+ finish, lost = cut.manager._finish_steer, []
+
+ async def lost_once(*args, **kwargs):
+ if not lost:
+ lost.append(args[0])
+ return SimpleNamespace(state="reconciling_steer") # its CAS was lost
+ return await finish(*args, **kwargs)
+
+ cut.manager._finish_steer = lost_once
+ cut.manager._active_identity = lambda *_args: None
+ if then == "stopped":
+ _request_stop(cut.engine, cut.session_id, cut.turn_id)
+
+ async def startup():
+ assert await cut.manager.controller.vibey_recovery.start() == []
+ await cut.manager.recover_durable_delivery_state(service_restart=True)
+ assert cut.starts == [] and lost and jobs.watches == {}
+ if then == "stopped":
+ with cut.engine.connect() as conn:
+ turn = message_deliveries.get_turn(conn, cut.turn_id)
+ assert (turn["state"], turn["terminal_evidence_kind"]) == ("terminal", "restart_runtime_missing")
+ assert jobs.killed == ["job_1"] and len(_notices(cut.engine, cut.session_id)) == 1
+ return
+ for _ in range(100):
+ if cut.starts:
+ break
+ await asyncio.sleep(0.02)
+
+ asyncio.run(startup())
+ if then == "stopped":
+ assert cut.starts == [] and jobs.watches == {}
+ return
+
+ assert cut.starts == [(cut.session_id, "")]
+ assert jobs.watches == {}
+ with cut.engine.connect() as conn:
+ assert message_deliveries.get_delivery(conn, steer["id"])["turn_id"] == cut.turn_id
+
+
+def test_a_stop_ends_the_command_of_a_turn_held_for_its_rerun_at_once(isolated_state, tmp_path):
+ """MESSAGE-DELIVERY-330: a cut Turn held open for its rerun (its transport not ready yet) has
+ no runtime for a Stop to reach, so the Stop settles it directly; its retry wakes at once and
+ T2 ends the surviving command, as the Stop would have live, without waiting out its delay."""
+ from core.agent_core.tools.base import JobStatus
+ from tests.agent_core.fakes import FakeJobHost
+
+ jobs = FakeJobHost()
jobs.states["job_1"] = JobStatus("running")
- call = ToolCallBlock(id="call_bash", name="bash", arguments={"command": "sleep 60; echo done >> s6.txt"})
+ cut = _accepted_turn(tmp_path, backend="vibey", delivery="unready", jobs=jobs)
+ _previous_process("running")(cut.engine, cut.session_id, cut.turn_id)
+ cut.manager._active_identity = lambda *_args: None
+ cut.manager._transport_can_deliver = lambda _platform: False
+
+ async def startup_then_stop():
+ assert await cut.manager.controller.vibey_recovery.start() == []
+ await cut.manager.recover_durable_delivery_state(service_restart=True)
+ # Held: the command stays with the Turn.
+ assert cut.starts == [] and jobs.killed == [] and jobs.watches == {}
+ cut.manager._terminalize_durable_turn(
+ cut.turn_id, "canceled", settled_by="adapter_not_active", evidence_kind="stop_not_active"
+ )
+ for _ in range(50):
+ if jobs.killed:
+ break
+ await asyncio.sleep(0.02)
- async def handed_over(_call, _job_id, _status, watch_id):
- return ToolResult((text(f"Command is still running and is now Watch {watch_id}."),))
+ asyncio.run(startup_then_stop())
- async def previous_process_then_t2():
- await store.consume_input(session_id, input_id, UserMessage((text("run it"),)))
- response = await store.append_response(session_id, assistant("", calls=(call,)), final=False)
- instance = CallInstance(session_id, response.row_id, response.context_seq, "call_bash")
- await settle_open_calls(
- session_id=session_id, store=store, jobs=jobs, job_ids={instance: "job_1"}, render_result=handed_over
+ assert jobs.killed == ["job_1"] and jobs.watches == {} and cut.starts == []
+
+
+def test_concurrent_recovery_passes_rerun_a_cut_turn_once(isolated_state, tmp_path):
+ """MESSAGE-DELIVERY-327: two recovery passes over the same cut Turn (a held rerun's transport
+ callback and its retry) both decide to rerun it; one rotates and reruns it, and the other,
+ whose rotation lands after that rerun bound its generation, releases nothing and starts
+ nothing."""
+ import threading
+
+ cut = _accepted_turn(tmp_path, backend="vibey")
+ _previous_process("tool")(cut.engine, cut.session_id, cut.turn_id)
+ bound, calls, lock = threading.Event(), [], threading.Lock()
+ rotate = cut.manager._rotate_for_rerun
+
+ def second_after_the_first_bound(*args):
+ with lock:
+ calls.append(args)
+ later = len(calls) > 1
+ if later:
+ assert bound.wait(5) # the other pass's rerun has bound its generation
+ return rotate(*args)
+
+ async def accepted(sid, ctx, text, **_kwargs):
+ # The rerun's native start binds this process's generation, as Vibey's does.
+ cut.starts.append((sid, text))
+ cut.manager.on_native_start(
+ ctx, backend="vibey", runtime_key=f"runtime:{sid}", runtime_turn_id=f"runtime-turn:{sid}"
)
+ bound.set()
- asyncio.run(previous_process_then_t2())
- assert jobs.watches == {"job_1": "watch_job_1"}
+ cut.manager.controller._session_id_from_context = lambda ctx: (ctx.platform_specific or {}).get(
+ "agent_session_id"
+ )
+ cut.manager._rotate_for_rerun = second_after_the_first_bound
+ cut.manager._run = accepted
+ cut.manager._active_identity = lambda *_args: None
+ cut.manager._transport_can_deliver = lambda _platform: True
+
+ async def two_passes():
+ await asyncio.gather(
+ *(cut.manager.recover_durable_delivery_state(cut.session_id, service_restart=True) for _ in range(2))
+ )
+
+ asyncio.run(two_passes())
+
+ assert len(calls) == 2 and cut.starts == [(cut.session_id, "")]
+ with cut.engine.connect() as conn:
+ turn = message_deliveries.get_turn(conn, cut.turn_id)
+ assert turn["state"] == "active" and turn["runtime_key"] == f"runtime:{cut.session_id}"
+
+
+def test_a_rerun_whose_setup_fails_is_retried_not_settled(isolated_state, tmp_path, monkeypatch):
+ """MESSAGE-DELIVERY-327: a transient failure setting the rerun up (its routing context, here)
+ is not the Turn's: the Turn stays open, recovery retries and reruns it, and the restart counts
+ once in its streak."""
+ from core.session_turns import SessionTurnManager
+
+ monkeypatch.setattr(SessionTurnManager, "RESTART_RECOVERY_RETRY_DELAYS", (0.05,))
+ monkeypatch.setattr(SessionTurnManager, "RESTART_RECOVERY_RETRY_PERIOD", 0.05)
+ cut = _accepted_turn(tmp_path, backend="vibey")
+ _previous_process("tool")(cut.engine, cut.session_id, cut.turn_id)
+ build, failed = cut.manager._delivery_context, []
+
+ def locked_once(session_id):
+ if not failed:
+ failed.append(session_id)
+ raise RuntimeError("database is locked")
+ return build(session_id)
+
+ cut.manager._delivery_context = locked_once
+ cut.manager._active_identity = lambda *_args: None
+
+ async def startup():
+ await cut.manager.recover_durable_delivery_state(service_restart=True)
+ assert cut.starts == [] and failed and _notices(cut.engine, cut.session_id) == []
+ for _ in range(100):
+ if cut.starts:
+ break
+ await asyncio.sleep(0.02)
+
+ asyncio.run(startup())
+
+ assert cut.starts == [(cut.session_id, "")]
+ with cut.engine.connect() as conn:
+ turn = message_deliveries.get_turn(conn, cut.turn_id)
+ assert turn["state"] == "active" and json.loads(turn["start_receipt_json"])["restart_streak"] == 1
+ assert _notices(cut.engine, cut.session_id) == []
+
+
[email protected]("ended", ["cancelled", "failed"])
+def test_a_rerun_cancelled_before_its_adapter_owns_the_command_still_ends_it(isolated_state, tmp_path, ended):
+ """MESSAGE-DELIVERY-330: the rerun is published before Vibey takes the command its restart
+ left running; a backend disable that cancels it in that interval still ends the command with
+ the canceled Turn, and a failure there (a pre-turn step raising, which the runner turns into a
+ failed Turn) hands the command to a Watch as a failed run's is: never left running unowned."""
+ from core.agent_core.tools.base import JobStatus
+ from core.run_settlement import SETTLED_BY_BACKEND_DISABLED
+ from core.session_turns import Turn
+ from tests.agent_core.fakes import FakeJobHost
+ from tests.test_internal_server import _bind_test_native_start
+
+ jobs = FakeJobHost()
+ jobs.states["job_1"] = JobStatus("running")
+ cut = _accepted_turn(tmp_path, backend="vibey", jobs=jobs)
+
+ release = asyncio.Event()
+
+ async def fails_before_its_adapter(turn_id):
+ await release.wait()
+ # The runner catches the failure and settles the Turn; its task ends normally.
+ cut.manager._terminalize_durable_turn(
+ turn_id, "failed", settled_by="no_terminal_result", evidence_kind="test_pre_turn_failure"
+ )
+
+ async def published_not_yet_adapted(sid, ctx, text, **_kwargs):
+ # ``_run`` publishes the in-flight Turn; the adapter has not started yet.
+ cut.starts.append((sid, text))
+ _bind_test_native_start(cut.engine, ctx)
+ turn_id = ctx.platform_specific["turn_token"]
+ cut.manager.in_flight[sid] = Turn(
+ task=asyncio.ensure_future(
+ fails_before_its_adapter(turn_id) if ended == "failed" else asyncio.Event().wait()
+ ),
+ context=ctx, logical_turn_id=turn_id,
+ )
+
+ cut.manager._run = published_not_yet_adapted
+ _previous_process("running")(cut.engine, cut.session_id, cut.turn_id)
+ cut.manager._active_identity = lambda *_args: None
+
+ async def startup():
+ assert await cut.manager.controller.vibey_recovery.start() == []
+ await cut.manager.recover_durable_delivery_state(service_restart=True)
+ assert cut.starts == [(cut.session_id, "")] and jobs.killed == []
+ if ended == "failed":
+ release.set()
+ else:
+ # The backend is disabled: its Turns are settled and their tasks cancelled.
+ cut.manager._terminalize_durable_turn(
+ cut.turn_id, "canceled", settled_by=SETTLED_BY_BACKEND_DISABLED, evidence_kind="backend_disabled"
+ )
+ cut.manager.in_flight.pop(cut.session_id).task.cancel()
+ for _ in range(100):
+ if jobs.killed or jobs.watches:
+ break
+ await asyncio.sleep(0.02)
+
+ asyncio.run(startup())
+
+ if ended == "failed":
+ assert jobs.killed == [] and jobs.watches == {"job_1": "watch_job_1"}
+ else:
+ assert jobs.killed == ["job_1"] and jobs.watches == {}
+
+
+def test_a_rerun_that_exits_before_dispatching_fails_its_turn(isolated_state, tmp_path):
+ """MESSAGE-DELIVERY-327: the rerun's runner finds no way to dispatch (its definitive no-write
+ exit). The Turn's start was accepted before the restart and nothing else runs it, so it fails
+ instead of staying open with no runner."""
+ from core.run_settlement import SETTLED_BY_NO_TERMINAL_RESULT
+
+ cut = _accepted_turn(tmp_path, backend="vibey")
+
+ async def exits_before_dispatch(sid, ctx, text, **_kwargs):
+ cut.starts.append((sid, text))
+ cut.manager._settle_durable_prewrite_failure(
+ ctx.platform_specific["turn_token"], outcome=SETTLED_BY_NO_TERMINAL_RESULT
+ )
+
+ cut.manager._run = exits_before_dispatch
+ _restart(cut, _previous_process("tool"))
+
+ assert cut.starts == [(cut.session_id, "")]
+ with cut.engine.connect() as conn:
+ turn = message_deliveries.get_turn(conn, cut.turn_id)
+ assert (turn["state"], turn["terminal_outcome"]) == ("terminal", "failed")
+ assert turn["terminal_evidence_kind"] == "rerun_failed_before_dispatch"
+
+
+def test_restart_rerun_runs_before_input_queued_while_the_service_was_down(isolated_state, tmp_path):
+ """MESSAGE-DELIVERY-331: the Turn stays the Session's active Turn, so the queued input waits
+ behind its rerun, then runs."""
+ cut = _accepted_turn(tmp_path, backend="vibey")
+
+ def queue_input(engine, session_id, _turn_id):
+ with engine.begin() as conn:
+ message_deliveries.enqueue_queued(conn, scope_id=None, session_id=session_id, text="然后再看看日志")
+
+ _restart(cut, _then(_previous_process("tool"), queue_input))
+
+ assert cut.starts == [(cut.session_id, "")]
+ with cut.engine.connect() as conn:
+ assert [row["text"] for row in message_deliveries.list_queued(conn, cut.session_id)] == ["然后再看看日志"]
+ assert asyncio.run(cut.manager.terminalize_turn(cut.turn_id))
+ assert cut.starts == [(cut.session_id, ""), (cut.session_id, "然后再看看日志")]
def test_restart_notice_names_the_watch_running_the_command_and_offers_no_retry(isolated_state, tmp_path):
"""MESSAGE-DELIVERY-322: resending a command that still runs as a Watch would run its effects twice."""
from vibe.ui_server import app
+ from core.agent_core.tools.base import JobStatus
+ from tests.agent_core.fakes import FakeJobHost
+
+ jobs = FakeJobHost()
+ jobs.states["job_1"] = JobStatus("running")
+ # Startup's pass leaves the command to the Turn's owner, which hands it over before its notice.
session_id, notice = _restart_failure_notice(
- tmp_path, backend="vibey", before_restart=_hand_over_the_open_command
+ tmp_path, backend="vibey", jobs=jobs, before_restart=_then(_previous_process("running"), _switch_off_vibey)
)
+ assert jobs.watches == {"job_1": "watch_job_1"}
assert "watch_job_1" in notice["text"] and "重新发送" not in notice["text"]
assert notice["metadata"]["watch_ids"] == ["watch_job_1"]
@@ -198,6 +768,64 @@ def test_restart_notice_names_the_watch_running_the_command_and_offers_no_retry(
dispatch.assert_not_awaited()
[email protected]("then", ["handed_over", "stopped"])
+def test_a_restart_notice_waits_until_its_running_command_is_handed_over(isolated_state, tmp_path, monkeypatch, then):
+ """MESSAGE-DELIVERY-322: with no Watch to take the command yet, the Turn stays open (T2 before
+ T4): no notice without the Watch, nothing failed under a running command; the retry hands it
+ over and the notice then names it. A Stop that settles the waiting Turn ends the command."""
+ from core.agent_core.tools.base import JobStatus
+ from core.session_turns import SessionTurnManager
+ from tests.agent_core.fakes import FakeJobHost
+
+ jobs = FakeJobHost()
+ jobs.states["job_1"] = JobStatus("running")
+ attempts = []
+ hand_over = jobs.hand_over
+
+ async def locked_once(job_id):
+ attempts.append(job_id)
+ if len(attempts) == 1 or then == "stopped":
+ raise RuntimeError("the Watch store is locked")
+ return await hand_over(job_id)
+
+ jobs.hand_over = locked_once
+ monkeypatch.setattr(SessionTurnManager, "RESTART_RECOVERY_RETRY_DELAYS", (0.05,))
+ monkeypatch.setattr(SessionTurnManager, "RESTART_RECOVERY_RETRY_PERIOD", 0.05)
+ cut = _accepted_turn(tmp_path, backend="vibey", jobs=jobs)
+ _then(_previous_process("running"), _switch_off_vibey)(cut.engine, cut.session_id, cut.turn_id)
+ cut.manager._active_identity = lambda *_args: None
+
+ async def startup():
+ assert await cut.manager.controller.vibey_recovery.start() == []
+ await cut.manager.recover_durable_delivery_state(cut.session_id, service_restart=True)
+ with cut.engine.connect() as conn:
+ assert message_deliveries.get_turn(conn, cut.turn_id)["state"] == "active"
+ assert _notices(cut.engine, cut.session_id) == []
+ if then == "stopped":
+ # The user stops the Turn while it waits; no runtime holds it, so the Stop settles it.
+ with cut.engine.begin() as conn:
+ message_deliveries.terminalize_turn(
+ conn, cut.turn_id, outcome="canceled", settled_by="adapter_not_active",
+ evidence_kind="stop_not_active",
+ )
+ for _ in range(100):
+ if _notices(cut.engine, cut.session_id) or jobs.killed:
+ break
+ await asyncio.sleep(0.02)
+
+ asyncio.run(startup())
+
+ if then == "stopped":
+ # Ended as the Stop would end it, never left running as a Watch, and no notice owed.
+ assert jobs.killed == ["job_1"] and jobs.watches == {}
+ assert _notices(cut.engine, cut.session_id) == []
+ return
+ [notice] = _notices(cut.engine, cut.session_id)
+ assert notice["metadata"]["watch_ids"] == ["watch_job_1"] and jobs.watches == {"job_1": "watch_job_1"}
+ with cut.engine.connect() as conn:
+ assert message_deliveries.get_turn(conn, cut.turn_id)["settled_by"] == "no_terminal_result"
+
+
@pytest.mark.parametrize("change", ["newer", "busy", "unlinked"])
def test_restart_notice_keeps_retry_boundary_guards(isolated_state, tmp_path, change):
from vibe.ui_server import app
diff --git a/tests/test_harness_failure_retry.py b/tests/test_harness_failure_retry.py
index 83c7d8296..292ff33a1 100644
--- a/tests/test_harness_failure_retry.py
+++ b/tests/test_harness_failure_retry.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
+from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import httpx
@@ -12,12 +13,12 @@ from sqlalchemy import select
from core import internal_server
from core.message_dispatcher import ConsolidatedMessageDispatcher
from core.scheduled_tasks import ScheduledTaskService, TaskExecutionStore
-from core.session_turns import DeliveryRequest, SessionTurnManager
+from core.session_turns import DeliveryRequest, SessionTurnManager, capture_scheduled_provenance
from storage import message_deliveries, messages_service
from storage.background import attach_agent_run_delivery_in_connection
from storage.db import create_sqlite_engine
from storage.models import agent_runs
-from tests.backend_failure_retry_helpers import seed_failed_notice
+from tests.backend_failure_retry_helpers import previous_process as _previous_process, seed_failed_notice
from tests.scenario_harness.message_delivery import MessageDeliveryController
from tests.test_harness_failure_visibility import _drain_service
from tests.test_internal_server import _bind_test_native_start, _build_controller_double
@@ -213,3 +214,180 @@ def test_harness_notice_keeps_retry_boundary_guards(isolated_state, tmp_path, ch
)
assert response.status_code == 409
dispatch.assert_not_awaited()
+
+
[email protected]("step", ["tool", "steered", "steer_retried"])
+def test_a_vibey_agent_run_cut_by_a_restart_settles_from_the_continued_work(
+ isolated_state, tmp_path, monkeypatch, step
+):
+ """MESSAGE-DELIVERY-328: shutdown leaves the Run with its Turn, startup reruns that Turn in
+ place, and the Run settles once, from the rerun's result, owing no notice. A Run whose steer
+ the dead run accepted, its receipt not yet materialized, joins the Turn and settles the same
+ way; when the receipt cannot be materialized at startup, the Turn waits open for the retry."""
+ from core.session_turns import Turn
+
+ _scope_id, session_id = _make_session(tmp_path, agent_backend="vibey")
+ engine = create_sqlite_engine()
+ controller = MessageDeliveryController(platform="avibe")
+ controller.config.language = "zh"
+ request_store = TaskExecutionStore()
+ service = _drain_service(tmp_path, controller, request_store.sqlite_backend, request_store)
+ service._t = ScheduledTaskService._t.__get__(service, ScheduledTaskService)
+ controller.scheduled_task_service = service
+ controller.emit_agent_message = ConsolidatedMessageDispatcher(controller).emit_agent_message
+
+ def context(sid):
+ value = _context(sid)
+ value.platform_specific["agent_session_target"]["agent_backend"] = "vibey"
+ return value
+
+ manager = SessionTurnManager(controller, build_context=context)
+ manager._engine = engine
+ controller.session_turns = manager
+ controller._session_id_from_context = lambda ctx: str(
+ (ctx.platform_specific or {}).get("agent_session_id") or ""
+ )
+ run = request_store.enqueue_agent_run(
+ message="整理日志并汇报", session_id=session_id, agent_name="worker", agent_backend="vibey",
+ callback_session_id="ses_caller",
+ )
+
+ steered = request_store.enqueue_agent_run(
+ message="也看一下告警", session_id=session_id, agent_name="worker", agent_backend="vibey",
+ )
+ assert request_store.claim(steered.id) is not None
+
+ def callbacks():
+ return [(row["status"], row["result_text"]) for row in request_store.list_pending_callbacks() if row["id"] == run.id]
+
+ assert request_store.claim(run.id) is not None
+ started = []
+
+ async def accept(_sid, ctx, _text, **_kwargs):
+ started.append(ctx)
+ _bind_test_native_start(engine, ctx)
+
+ manager._run = accept
+
+ harness_context = context(session_id)
+ harness_context.platform_specific.update(
+ {"task_execution_id": run.id, "task_trigger_kind": "agent_run", "callback_session_id": "ses_caller"}
+ )
+
+ async def scenario():
+ original = await manager.deliver(
+ DeliveryRequest(
+ session_id=session_id, priority="p3", content=run.prompt,
+ source="harness", author="harness", message_type="harness",
+ metadata={"scheduled_provenance": capture_scheduled_provenance(harness_context)},
+ ),
+ context=context(session_id),
+ )
+ with engine.begin() as conn:
+ assert attach_agent_run_delivery_in_connection(
+ conn, run.id, session_id=session_id, delivery_id=original.delivery_id,
+ )
+ if step in {"steered", "steer_retried"}:
+ # A second Run steered into the Turn: the run accepted it, then the process
+ # died after the receipt and before the Delivery was materialized.
+ steer_id = message_deliveries.new_delivery_id()
+ steer = message_deliveries.insert_delivery(
+ conn, delivery_id=steer_id, session_id=session_id, priority="p1", state="reserved",
+ snapshot=message_deliveries.message_snapshot(
+ scope_id=None, session_id=session_id, platform="avibe", author="harness",
+ source="harness", message_type="harness", text="也看一下告警",
+ ),
+ dispatch_text="也看一下告警",
+ )
+ assert attach_agent_run_delivery_in_connection(
+ conn, steered.id, session_id=session_id, delivery_id=steer_id,
+ )
+ cut = message_deliveries.get_turn(conn, original.turn_id)
+ opened = message_deliveries.open_steer_attempt(
+ conn, steer_id, expected_version=int(steer["version"]), turn_id=original.turn_id,
+ attempt_id="atm_steer", expected_native_turn_id=cut["native_turn_id"],
+ )
+ assert message_deliveries.mark_attempt_receipt(
+ conn, steer_id, expected_version=int(opened["version"]), outcome="accepted", receipt={},
+ )
+ # Service shutdown while the Turn runs: nobody asked it to stop.
+ running = asyncio.create_task(asyncio.Event().wait())
+ manager.in_flight[session_id] = Turn(
+ task=running, context=started[-1], logical_turn_id=original.turn_id,
+ )
+ assert await manager.release_for_service_shutdown() == 0
+ assert not running.cancelled()
+ with engine.connect() as conn:
+ assert message_deliveries.get_turn(conn, original.turn_id)["state"] == "active"
+ assert request_store.get_run(run.id)["status"] == "running"
+ running.cancel()
+ manager.in_flight.clear()
+ return original
+
+ original = asyncio.run(scenario())
+ # The next start: T2 settled the cut call, then Turn recovery.
+ _previous_process("tool")(engine, session_id, original.turn_id)
+ manager._active_identity = lambda *_args: None
+ if step == "steer_retried":
+ # Startup cannot materialize the receipt (its CAS is lost); the retry can.
+ finish = manager._finish_steer
+ lost = []
+
+ async def lost_once(*args, **kwargs):
+ if not lost:
+ lost.append(args[0])
+ return SimpleNamespace(state="reconciling_steer")
+ return await finish(*args, **kwargs)
+
+ manager._finish_steer = lost_once
+ monkeypatch.setattr(SessionTurnManager, "RESTART_RECOVERY_RETRY_DELAYS", (0.05, 0.05))
+
+ async def startup():
+ await manager.recover_durable_delivery_state(session_id, service_restart=True)
+ if step == "steer_retried":
+ # Nothing completed or reran without the steer: the Turn stays open.
+ with engine.connect() as conn:
+ assert message_deliveries.get_turn(conn, original.turn_id)["state"] == "active"
+ assert len(started) == 1 and lost
+ # The first retry pass fails outright; the retries go on.
+ recover, failed = manager.recover_durable_delivery_state, []
+
+ async def fails_once(*args, **kwargs):
+ if not failed:
+ failed.append(True)
+ raise RuntimeError("database is locked")
+ return await recover(*args, **kwargs)
+
+ manager.recover_durable_delivery_state = fails_once
+ for _ in range(100):
+ if len(started) > 1:
+ break
+ await asyncio.sleep(0.02)
+
+ asyncio.run(startup())
+
+ stored = request_store.get_run(run.id)
+ with engine.connect() as conn:
+ source = message_deliveries.get_turn(conn, original.turn_id)
+ assert source["state"] == "active"
+ continued = started[-1]
+ # The cut Turn itself reruns as a transcript resume, with the Run's Harness provenance.
+ assert continued.platform_specific["turn_token"] == original.turn_id
+ assert continued.platform_specific["vibey_resume"] is True
+ assert continued.platform_specific["vibey_source_turn_id"] == original.turn_id
+ assert continued.platform_specific["task_execution_id"] == run.id
+ assert stored["status"] == "running" and callbacks() == []
+ assert stored["delivery_id"] == original.delivery_id
+ owned = {run.id} | ({steered.id} if step != "tool" else set())
+ assert set(continued.platform_specific["accepted_agent_run_ids"]) == owned
+ # The rerun's own result settles the Runs, exactly once.
+ asyncio.run(controller.emit_agent_message(continued, "result", "日志已整理完毕"))
+ stored = request_store.get_run(run.id)
+ assert stored["status"] == "succeeded"
+ assert stored["result_text"] == "日志已整理完毕"
+ if step != "tool":
+ assert request_store.get_run(steered.id)["status"] == "succeeded"
+ # The caller's callback reports the work's own result, once.
+ assert callbacks() == [("succeeded", stored["result_text"])]
+ assert request_store.sqlite_backend.owed_failure_notice(run.id) is None
+ assert "interrupt_reason" not in (stored.get("metadata") or {})
diff --git a/tests/test_message_handler_typing.py b/tests/test_message_handler_typing.py
index d963eaac9..b10a3255f 100644
--- a/tests/test_message_handler_typing.py
+++ b/tests/test_message_handler_typing.py
@@ -731,6 +731,36 @@ class MessageHandlerTypingTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(request.message, "reviewer: inspect the change")
self.assertIsNone(request.subagent_name)
+ async def test_a_restart_rerun_runs_with_the_request_its_turn_was_accepted_with(self):
+ """MESSAGE-DELIVERY-327: the rerun's request carries the model, effort, and instructions its
+ native start recorded, before the shared model validation, so an Agent edited (or its
+ model cleared) while the service was down reaches only the next Turn."""
+ controller = _StubController(platform="slack", ack_mode="reaction", typing_result=True)
+ agent = controller.resolve_vibe_agent_for_context(None)
+ agent.backend, agent.model, agent.reasoning_effort = "vibey", None, None
+ controller.resolve_agent_for_context = lambda _context: "vibey"
+ controller.resolve_vibe_agent_for_context = lambda *_args, **_kwargs: agent
+ handler = MessageHandler(controller)
+ handler.set_session_handler(_StubSessionHandler())
+ context = MessageContext(
+ user_id="U1", channel_id="C1", message_id="m1", platform="slack",
+ platform_specific={
+ "vibey_resume": True,
+ "vibey_execution": {
+ "agent_id": "agent-default", "agent_name": "default", "model": "team-model",
+ "reasoning_effort": "high", "system_prompt": "Run the steps in order.",
+ },
+ },
+ )
+
+ await handler.handle_user_message(context, "")
+
+ _, request = controller.agent_service.requests[0]
+ self.assertEqual(
+ (request.vibe_agent_model, request.vibe_agent_reasoning_effort, request.vibe_agent_system_prompt),
+ ("team-model", "high", "Run the steps in order."),
+ )
+
async def test_scope_model_and_reasoning_override_vibe_agent_defaults(self):
controller = _StubController(platform="slack", ack_mode="reaction", typing_result=True)
controller.settings_manager.routing = type(
diff --git a/tests/test_processing_indicator_reaction.py b/tests/test_processing_indicator_reaction.py
index 0c7ef76f8..0595f1367 100644
--- a/tests/test_processing_indicator_reaction.py
+++ b/tests/test_processing_indicator_reaction.py
@@ -110,6 +110,35 @@ class QueuedReactionLifecycleTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(handle.ack_reaction_emoji, ACK_REACTION_EMOJI)
self.assertEqual(im.calls, [("add", "m1", ACK_REACTION_EMOJI)])
+ async def test_a_restart_rerun_retires_the_running_reaction_its_dead_process_may_have_left(self):
+ """MESSAGE-DELIVERY-327: the rerun runs its own indicator as any Turn does; its end also
+ retires the 👀 the dead process may have left. There may be none, so a failed removal
+ does not withhold its terminal receipt."""
+ from core.processing_indicator import STOPPED_REACTION_EMOJI
+
+ cases = [
+ # Reaction mode: its own 👀 (adding one already there is a no-op) is the one it retires.
+ (True, True, [("add", "m1", ACK_REACTION_EMOJI), ("remove", "m1", ACK_REACTION_EMOJI)]),
+ # Typing mode: the stale 👀 is removed, then the Stop's receipt lands.
+ (False, True, [("remove", "m1", ACK_REACTION_EMOJI), ("add", "m1", STOPPED_REACTION_EMOJI)]),
+ # Typing mode, no 👀 ever added: the removal fails, and the receipt still lands.
+ (False, False, [("remove", "m1", ACK_REACTION_EMOJI), ("add", "m1", STOPPED_REACTION_EMOJI)]),
+ ]
+ for selected, removable, expected in cases:
+ with self.subTest(reaction_indicator_selected=selected, stale_reaction=removable):
+ im = _FakeIM(remove_ok=removable)
+ svc = _svc(im)
+ context = _ctx()
+ context.platform_specific = {"vibey_resume": True}
+ # As ``start()`` builds it: the receipt goes on the message the Turn answers.
+ handle = ProcessingIndicatorHandle(
+ context=context, reaction_indicator_selected=selected, terminal_reaction_message_id="m1"
+ )
+
+ await svc.promote_reaction_to_running(handle)
+ await svc.finish(handle, terminal_emoji=None if selected else STOPPED_REACTION_EMOJI)
+ self.assertEqual(im.calls, expected)
+
async def test_promote_is_idempotent(self):
im = _FakeIM()
svc = _svc(im)
diff --git a/tests/test_vibey_agent.py b/tests/test_vibey_agent.py
index 9ce441437..500dccf1f 100644
--- a/tests/test_vibey_agent.py
+++ b/tests/test_vibey_agent.py
@@ -208,6 +208,7 @@ class _Controller:
self.terminal_texts: list[Optional[str]] = []
self.released: list[str] = []
self.started: list[Optional[str]] = []
+ self.native_agents: list[Optional[str]] = []
self.hub_calls: list[str] = []
self.hub_protocol = "anthropic"
# The Model Hub model definition's limits (C-9 takes W, L_in, and O from them).
@@ -285,6 +286,7 @@ class _Controller:
# Outside a run (a crash simulation), the identity a previous process's adapter bound.
native = self.agent.steering_native_turn_id(target) or f"vibey:previous-process:{turn_id}"
self.started.append(native)
+ self.native_agents.append((context.platform_specific or {}).get("vibey_execution"))
with self.engine.begin() as conn:
turn = message_deliveries.get_turn(conn, turn_id)
assert message_deliveries.bind_native_start(
@@ -302,6 +304,16 @@ def _turn(context) -> str:
return str((context.platform_specific or {}).get("turn_token") or "")
+def _owner_settled(engine, *requests) -> None:
+ """The Turn owner settled these Turns without continuing them, as after a restart notice."""
+ with engine.begin() as conn:
+ conn.execute(
+ update(session_turns)
+ .where(session_turns.c.id.in_([_turn(request.context) for request in requests]))
+ .values(state="terminal", terminal_outcome="failed", terminal_at=NOW)
+ )
+
+
def _context(platform: str, session_id: str, *, turn_id: str, delivery_id: str) -> MessageContext:
return MessageContext(
user_id="u1",
@@ -2603,6 +2615,8 @@ async def test_startup_hands_an_orphaned_foreground_job_to_its_watch(engine, ses
call = ToolCallBlock(id="call_bash", name="bash", arguments={"command": "pytest -q"})
await harness.agent.store.append_response(SESSION, assistant("", calls=(call,)), final=False)
jobs.states["job_1"] = JobStatus("running")
+ # The Turn owner settled the Turn without continuing it, so the job is startup's to hand over.
+ _owner_settled(engine, request)
# A pass that failed before it settled anything runs again in full.
harness.new_agent()
@@ -2627,6 +2641,307 @@ async def test_startup_hands_an_orphaned_foreground_job_to_its_watch(engine, ses
assert harness.provider.requests == []
[email protected]("cut_in", ["model_call", "preflight"])
+async def test_a_run_the_service_stop_cut_leaves_its_turn_open_for_its_restart_rerun(
+ engine, session, tmp_path, published, cut_in
+) -> None:
+ """MESSAGE-DELIVERY-332: stopping the service (its gateway first) fails the run, in a model
+ call or in a rerun's preflight; the Turn is the restart's to continue, so it neither fails
+ nor shows a notice."""
+ from core.agent_core.ai.provider import ProviderError
+
+ harness = _Harness(engine, tmp_path, "telegram", [[ProviderError("server", "gateway closed", False)]])
+ harness.controller.shutdown_requested = True
+ request = harness.request("keep working")
+ if cut_in == "preflight":
+ # A rerun of a Turn the previous process accepted; the Model Hub stops with the service.
+ harness.controller._native_start(request.context)
+ request.context.platform_specific.update(
+ {"vibey_resume": True, "vibey_source_turn_id": _turn(request.context)}
+ )
+
+ async def stopped(*_args, **_kwargs):
+ raise RuntimeError("the Model Hub gateway is stopped")
+
+ harness.controller.model_hub_runtime.resolve = stopped
+
+ running = asyncio.create_task(harness.agent.handle_message(request))
+ for _ in range(50):
+ await asyncio.sleep(0.01)
+ if harness.provider.requests and not running.done():
+ break
+ await asyncio.sleep(0.05)
+
+ assert not running.done()
+ assert harness.controller.terminals == [] and harness.rows("notify") == []
+ assert harness.controller.im_client.sent == []
+ with engine.connect() as conn:
+ assert message_deliveries.get_turn(conn, _turn(request.context))["state"] == "active"
+ running.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await running
+
+
+ ("platform", "case"), [("telegram", "undelivered"), ("telegram", "delivered"), ("avibe", "silent")]
+)
+async def test_a_restart_rerun_ends_with_the_final_its_cut_run_committed(
+ engine, session, tmp_path, published, platform, case
+) -> None:
+ """MESSAGE-DELIVERY-333: the run a restart cut had committed the Turn's final, so the rerun
+ ends with that row through the live end, with no model call. A final that process never sent
+ to IM is delivered once, with the image that run generated; one whose delivery the dispatcher
+ recorded is not sent again; a silent final leaves the narration that process showed as the
+ Turn's result."""
+ from dataclasses import replace
+
+ from core.message_output import MessageOutput, terminal_output_for
+
+ answer = "The cache is safe: one writer holds the lock."
+ narration = "Found it.\nThe cache is shared.\nTwo writers race.\nThe lock went in #12.\nFixed and verified."
+ harness = _Harness(engine, tmp_path, platform, [])
+ # The previous process: the Turn's final committed, then the restart cut the run.
+ request = harness.request("is the cache safe?")
+ cut_turn = _turn(request.context)
+ harness.controller._native_start(request.context)
+ harness.agent.store.bind_agent(SESSION, None, turn_id=cut_turn)
+ await harness.agent.store.consume_input(
+ SESSION, request.context.platform_specific["delivery_id"], UserMessage((text("is the cache safe?"),))
+ )
+ if case == "silent":
+ shown = await harness.agent.store.append_response(SESSION, assistant(narration), final=False)
+ await harness.controller.emit_agent_message(
+ request.context, "assistant", narration,
+ output=MessageOutput(completes_turn=False, completes_run=False, persisted_row_id=shown.row_id),
+ )
+ [shown] = harness.rows("interim")
+ image = tmp_path / "lighthouse.jpg"
+ image.write_bytes(_JPEG)
+ final = await harness.agent.store.append_response(
+ SESSION, assistant("<silent>done</silent>" if case == "silent" else answer), final=True,
+ images=(HostedImage(path=str(image), mime="image/jpeg"),) if case == "undelivered" else (),
+ )
+ if case == "delivered":
+ # It was delivered before the process died, but the Turn was not yet settled.
+ await harness.agent.emit_result_message(
+ request.context, answer, request=request,
+ output=replace(terminal_output_for(request), persisted_row_id=final.row_id),
+ )
+ assert harness.controller.im_client.sent == [answer]
+ harness.controller.im_client.sent.clear()
+ with engine.begin() as conn:
+ # The Turn owner releases the dead runtime generation to rerun the Turn in place.
+ turn = message_deliveries.get_turn(conn, cut_turn)
+ assert message_deliveries.rotate_restart_generation(
+ conn, cut_turn, expected_version=int(turn["version"]), streak=1, response_seq=final.context_seq
+ )
+ # A new process: a new adapter, and a dispatcher that has seen none of the Turn.
+ harness.controller.message_dispatcher = ConsolidatedMessageDispatcher(harness.controller)
+ harness.new_agent()
+ context = _context(
+ platform, SESSION, turn_id=cut_turn, delivery_id=request.context.platform_specific["delivery_id"]
+ )
+ context.platform_specific.update({"vibey_resume": True, "vibey_source_turn_id": cut_turn})
+ resume = AgentRequest(**{**request.__dict__, "context": context, "message": "", "user_message": ""})
+
+ await harness.agent.handle_message(resume)
+
+ assert harness.provider.requests == [] and harness.controller.hub_calls == []
+ assert harness.controller.terminals[-1] == {"turn": cut_turn, "is_error": False, "settled_by": "terminal_result"}
+ [result] = harness.rows("result")
+ if case == "silent":
+ assert (result["id"], result["content_text"]) == (shown["id"], narration)
+ assert harness.rows("interim") == []
+ return
+ assert result["id"] == final.row_id and result["content_text"].startswith(answer)
+ sent, uploads = harness.controller.im_client.sent, harness.controller.im_client.uploads
+ if case == "delivered":
+ assert sent == [] and uploads == []
+ return
+ # Sent once, with the image its run generated.
+ assert len(sent) == 1 and sent[0].startswith(answer)
+ assert uploads == [str(image.resolve())]
+
+
+ "case",
+ [
+ "exits", "window_passed", "watch_unavailable", "stop", "cancelled", "cancelled_at_shutdown", "stopped_left_open",
+ "stop_pending", "cancel_pending",
+ ],
+)
+async def test_a_restart_rerun_waits_for_its_surviving_command_as_the_live_call_would(
+ engine, session, tmp_path, published, monkeypatch, case
+) -> None:
+ """MESSAGE-DELIVERY-330: startup leaves a running command to the Turn it reruns, and so does a
+ second startup before that rerun ran; the rerun waits for it within the call's foreground
+ window, counted from its launch, before it binds its native start, so its real output is the
+ result the reply (and a Run, MESSAGE-DELIVERY-328) settles from. A window the downtime spent
+ hands it over; with no Watch to take it, it stays in the foreground until it ends; a Stop ends
+ the wait and the command, as does one that settles the Turn while it is left open, or one the
+ dead process recorded and never settled (a Stop, or a cancel of the Turn's Run), and so does a cancellation outside a shutdown (the
+ backend disabled or refreshed); a shutdown leaves the command for the next start. It never
+ starts again."""
+ import core.watches as watches_module
+ from core.scheduled_tasks import TaskExecutionStore
+ from core.watches import ManagedWatchStore
+
+ suite = local_tool_suite(str(tmp_path / "jobs"))
+ harness = _Harness(engine, tmp_path, "avibe", [[Done(assistant("步骤三完成"))]], suite=suite)
+ # The previous process: an accepted Turn of an Agent Run, cut while its command ran.
+ request = harness.request("run the three steps")
+ cut_turn = _turn(request.context)
+ harness.controller._native_start(request.context)
+ await harness.agent.store.consume_input(
+ SESSION, request.context.platform_specific["delivery_id"], UserMessage((text("run the three steps"),))
+ )
+ command = "sleep 1; printf step-two-done" if case in {"exits", "watch_unavailable"} else "sleep 30"
+ call = ToolCallBlock(id="call_bash", name="bash", arguments={"command": command})
+ response = await harness.agent.store.append_response(SESSION, assistant("", calls=(call,)), final=False)
+ job_id = await _real_job(suite, command, call=_instance(response, "call_bash"), cwd=tmp_path)
+ requests = TaskExecutionStore()
+ run = requests.enqueue_agent_run(message="run the three steps", session_id=SESSION, agent_name="vibey")
+ assert requests.claim(run.id) is not None
+ try:
+ if case == "watch_unavailable":
+
+ async def no_watch(meta, **_kwargs):
+ raise RuntimeError("the Watch store is locked")
+
+ monkeypatch.setattr(watches_module, "hand_over_job", no_watch)
+ if case in {"window_passed", "watch_unavailable"}:
+ # The job launched three minutes ago: the downtime spent its 120 s window.
+ from datetime import datetime, timedelta
+
+ meta_path = Path(suite.jobs.job_dir(job_id)) / "meta.json"
+ meta = json.loads(meta_path.read_text())
+ launched = datetime.fromisoformat(meta["created_at"].replace("Z", "+00:00")) - timedelta(minutes=3)
+ meta["created_at"] = launched.isoformat(timespec="milliseconds").replace("+00:00", "Z")
+ meta_path.write_text(json.dumps(meta))
+ if case == "stop_pending":
+ # The user pressed Stop as the service went down; the Stop never settled the Turn,
+ # which will not rerun. Startup's T2 ends the command, as that Stop would have.
+ with engine.begin() as conn:
+ turn = message_deliveries.get_turn(conn, cut_turn)
+ assert message_deliveries.cas_turn(
+ conn, cut_turn, expected_version=int(turn["version"]), expected_states=("active",),
+ values={"control_mode": "stop_only"},
+ )
+ if case == "cancel_pending":
+ # The caller canceled the Turn's Run as the service went down, before its Stop reached
+ # the Turn: the same.
+ from storage.background import attach_agent_run_delivery_in_connection
+
+ with engine.begin() as conn:
+ assert attach_agent_run_delivery_in_connection(
+ conn, run.id, session_id=SESSION, delivery_id=request.context.platform_specific["delivery_id"]
+ )
+ assert requests.cancel_run(run.id)
+ harness.new_agent()
+ assert await harness.agent.recover_runtime_state() == []
+ assert ManagedWatchStore().find_job_watch(job_id) is None
+ if case in {"stopped_left_open", "stop_pending", "cancel_pending"}:
+ if case == "stopped_left_open":
+ # The Turn stays open (its fallback waits for a Watch) and a Stop settles it with no
+ # runtime holding it: the next T2 ends the command, as the Stop would have.
+ with engine.begin() as conn:
+ message_deliveries.terminalize_turn(
+ conn, cut_turn, outcome="canceled", settled_by="adapter_not_active",
+ evidence_kind="stop_not_active",
+ )
+ assert await harness.agent.recover_runtime_state() == []
+ [result] = [entry for entry in await harness.context_rows() if entry.kind == "tool_result"]
+ assert "Stopped by the user" in result.message.content[0].text
+ assert suite.jobs.status(job_id).state != "running" and ManagedWatchStore().find_job_watch(job_id) is None
+ return
+ with engine.begin() as conn:
+ # The Turn owner releases the dead runtime generation to rerun the Turn in place.
+ turn = message_deliveries.get_turn(conn, cut_turn)
+ assert message_deliveries.rotate_restart_generation(
+ conn, cut_turn, expected_version=int(turn["version"]), streak=1, response_seq=response.context_seq
+ )
+ native_starts = list(harness.controller.started)
+ # Another restart before the rerun ran: the command is still the Turn's.
+ harness.new_agent()
+ assert await harness.agent.recover_runtime_state() == []
+ assert ManagedWatchStore().find_job_watch(job_id) is None
+ context = _context(
+ "avibe", SESSION, turn_id=cut_turn, delivery_id=request.context.platform_specific["delivery_id"]
+ )
+ accepted = {
+ "agent_id": "agt_worker", "agent_name": "worker", "model": "model-accepted",
+ "reasoning_effort": None, "system_prompt": "Run the steps in order.",
+ }
+ context.platform_specific.update(
+ {
+ "vibey_resume": True, "vibey_source_turn_id": cut_turn, "accepted_agent_run_ids": [run.id],
+ # Recorded by the Turn's native start; the Agent's model was edited since.
+ "vibey_execution": accepted,
+ }
+ )
+ # The rerun's request, as the message handler builds it from that record.
+ resume = AgentRequest(
+ **{
+ **request.__dict__, "context": context, "message": "", "user_message": "",
+ "vibe_agent_name": "worker", "vibe_agent_id": "agt_worker", "vibe_agent_model": "model-accepted",
+ "vibe_agent_reasoning_effort": None, "vibe_agent_system_prompt": "Run the steps in order.",
+ }
+ )
+
+ resumed = asyncio.create_task(harness.agent.handle_message(resume))
+ if case in {"cancelled", "cancelled_at_shutdown"}:
+ await _until(lambda: harness.agent._run_for(SESSION, cut_turn) is not None, "the rerun never ran")
+ await asyncio.sleep(0.3)
+ harness.controller.shutdown_requested = case == "cancelled_at_shutdown"
+ # The Turn task is cancelled directly, without ``handle_stop``.
+ resumed.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(resumed, timeout=20)
+ results = [entry for entry in await harness.context_rows() if entry.kind == "tool_result"]
+ if case == "cancelled":
+ assert "Stopped by the user" in results[0].message.content[0].text
+ assert suite.jobs.status(job_id).state != "running"
+ else:
+ assert results == [] and suite.jobs.status(job_id).state == "running"
+ assert ManagedWatchStore().find_job_watch(job_id) is None and harness.provider.requests == []
+ return
+ if case == "stop":
+ await _until(lambda: harness.agent._run_for(SESSION, cut_turn) is not None, "the rerun never ran")
+ await asyncio.sleep(0.3)
+ assert not resumed.done()
+ # Its native start, which releases the steers held for the Turn, waits for the Agent
+ # that takes them; the durably active Turn still routes this Stop to ``handle_stop``.
+ assert harness.controller.started == native_starts
+ assert await harness.agent.handle_stop(resume)
+ await asyncio.wait_for(resumed, timeout=20)
+
+ [result] = [entry for entry in await harness.context_rows() if entry.kind == "tool_result"]
+ body = result.message.content[0].text
+ watch_id = ManagedWatchStore().find_job_watch(job_id)
+ if case in {"exits", "watch_unavailable"}:
+ # It runs with the request its Turn was accepted with, and its native start records
+ # that request again for a later restart's rerun.
+ assert harness.controller.hub_calls[-1] == "model-accepted"
+ assert harness.controller.native_agents[-1] == accepted
+ assert "step-two-done" in body and watch_id is None
+ assert "step-two-done" in json.dumps(message_to_dict(harness.provider.requests[0].messages[-1]))
+ assert harness.controller.im_client.sent[-1] == "步骤三完成"
+ stored = requests.get_run(run.id)
+ assert (stored["status"], stored["result_text"]) == ("succeeded", "步骤三完成")
+ elif case == "window_passed":
+ assert watch_id and f"now Watch {watch_id}" in body
+ assert suite.jobs.status(job_id).state == "running"
+ else:
+ # Stopped before the rerun's model call: none was made.
+ assert "Stopped by the user" in body and watch_id is None
+ assert suite.jobs.status(job_id).state != "running" and harness.provider.requests == []
+ # The rerun consumed nothing: the Turn's one input, and no hidden row.
+ assert len(harness.rows("user")) == 1 and harness.rows("agent_initiated") == []
+ finally:
+ if suite.jobs.status(job_id).state == "running":
+ await suite.jobs.kill(job_id)
+
+
async def test_startup_recovers_a_session_routed_to_another_agent_since(engine, session, tmp_path, published) -> None:
jobs = FakeJobHost()
@@ -2655,7 +2970,8 @@ async def test_startup_recovers_a_session_routed_to_another_agent_since(engine,
)
harness.new_agent()
- await harness.agent.recover_runtime_state()
+ # Startup leaves the job to the Turn owner, whose notice for a Session off Vibey asks for it first.
+ await harness.agent.recover_runtime_state(SESSION)
assert jobs.watches == {"job_1": "watch_job_1"}
assert (await harness.context_rows())[-1].kind == "tool_result"
@@ -4053,7 +4369,7 @@ async def test_resume_settles_a_real_bash_job_through_the_real_renderer(
await _until(lambda: suite.jobs.status(job_id).state == "exited", "the command never exited")
harness.new_agent()
- await harness.agent.recover_runtime_state()
+ await harness.agent.recover_runtime_state(SESSION)
result = (await harness.context_rows())[-1]
assert result.kind == "tool_result"
@@ -4118,7 +4434,7 @@ async def test_resume_settles_a_reused_call_id_from_its_own_instance_whatever_th
try:
agent = harness.new_agent()
agent._tool_suite = None
- await agent.recover_runtime_state()
+ await agent.recover_runtime_state(SESSION)
result = (await harness.context_rows())[-1]
watch = _job_watch(current)
@@ -4199,6 +4515,13 @@ async def test_recovery_retries_a_session_it_could_not_settle(
_hand_over_failing_once(monkeypatch)
harness, running, _exited = await _two_sessions_with_open_calls(engine, tmp_path, local_tool_suite())
+ with engine.begin() as conn:
+ # SESSION's Turn owner settled its Turn without continuing it: the job is recovery's.
+ conn.execute(
+ update(session_turns)
+ .where(session_turns.c.session_id == SESSION)
+ .values(state="terminal", terminal_outcome="failed", terminal_at=NOW)
+ )
recovery = _controller_with(harness.new_agent()).vibey_recovery
try:
# The first pass settles ses_b and reports SESSION, never success over a failure.
@@ -4418,10 +4741,11 @@ async def test_startup_recovery_hands_a_job_over_with_its_turns_authorization(
job_id = await _remote_turn_with_open_call(harness, tmp_path, snapshot=snapshot, author_id=author_id)
suite = harness.agent._tool_suite
try:
- # The process restarted: no live request, only the Turn's durable rows.
+ # The process restarted: no live request, only the Turn's durable rows; the Turn owner
+ # settles this Session before its notice.
agent = harness.new_agent()
agent._tool_suite = None
- assert await agent.recover_runtime_state() == []
+ assert await agent.recover_runtime_state(SESSION) == []
watch = _job_watch(job_id)
assert watch is not None and watch.agent_name == "vibey"
diff --git a/tests/test_vibey_resume_ownership.py b/tests/test_vibey_resume_ownership.py
index c77a4a5fb..9ed0d2af3 100644
--- a/tests/test_vibey_resume_ownership.py
+++ b/tests/test_vibey_resume_ownership.py
@@ -207,7 +207,7 @@ async def test_retry_events_and_recovered_job_belong_to_retry_turn_not_last_inpu
await harness.agent.store.append_payload(SESSION, "agent_state", {"version": 1, "state": {"resumed": True}})
jobs.states["job_retry"] = JobStatus("running")
harness.new_agent()
- assert await harness.agent.recover_runtime_state() == []
+ assert await harness.agent.recover_runtime_state(SESSION) == []
with engine.connect() as conn:
events = conn.execute(select(agent_events).where(agent_events.c.session_id == SESSION)).mappings().all()
diff --git a/ui/src/components/workbench/FailureRetry.test.tsx b/ui/src/components/workbench/FailureRetry.test.tsx
index 1c1eb6ef7..d16bed716 100644
--- a/ui/src/components/workbench/FailureRetry.test.tsx
+++ b/ui/src/components/workbench/FailureRetry.test.tsx
@@ -32,8 +32,8 @@ function mount(props: Partial<Parameters<typeof FailureRetry>[0]> = {}) {
describe('failed-turn retry action', () => {
it.each([
- '⚠️ 本轮执行被中断——Avibe 服务在它运行期间重启。如果这项工作仍然需要完成,请重新发送请求。',
- '[Avibe Harness] 一项后台任务已随 Avibe 重启停止。如需继续,请发送“继续”给 Agent。',
+ '⚠️ 本轮执行被中断——Avibe 服务在它运行期间重启。如需继续,请点击“重试”(如有显示),或发送“继续”。',
+ '[Avibe Harness] 一项后台任务已随 Avibe 重启停止。如需继续,请点击“重试”(如有显示),或发送“继续”给 Agent。',
])('MESSAGE-DELIVERY-320: offers retry for a linked restart notice: %s', async (text) => {
const onRetry = mount({
message: {
@@ -127,7 +127,7 @@ describe('failed-turn retry action', () => {
{
message: {
...notice,
- text: '[Avibe Harness] 一项后台任务已随 Avibe 重启停止。如需继续,请发送“继续”给 Agent。',
+ text: '[Avibe Harness] 一项后台任务已随 Avibe 重启停止。如需继续,请点击“重试”(如有显示),或发送“继续”给 Agent。',
metadata: { event: 'backend_failure', failure_id: 'turn:unbound', replayed: true },
},
},
diff --git a/vibe/i18n/en.json b/vibe/i18n/en.json
index 933db74d0..3c201dc1b 100644
--- a/vibe/i18n/en.json
+++ b/vibe/i18n/en.json
@@ -215,8 +215,8 @@
"watchCircuitRepairFailed": "[Avibe Harness] Watch \"{name}\" remains paused because its circuit-repair Agent Run failed.",
"watchFollowUpFailed": "[Avibe Harness] Watch \"{name}\" follow-up Agent processing failed.",
"interrupted": "[Avibe Harness] Scheduled work \"{name}\" was interrupted ({reason}). Nothing is wrong with the definition itself.",
- "restartStopped": "[Avibe Harness] Background task \"{name}\" stopped when Avibe restarted. To continue, send \"continue\" to the Agent.",
- "restartStoppedUnnamed": "[Avibe Harness] A background task stopped when Avibe restarted. To continue, send \"continue\" to the Agent.",
+ "restartStopped": "[Avibe Harness] Background task \"{name}\" stopped when Avibe restarted. To continue, choose Retry if it is shown, or send \"continue\" to the Agent.",
+ "restartStoppedUnnamed": "[Avibe Harness] A background task stopped when Avibe restarted. To continue, choose Retry if it is shown, or send \"continue\" to the Agent.",
"commandLine": "Command: {command}",
"error": "Error: {error}",
"commandExit": "Exit code: {exitCode}",
@@ -1289,7 +1289,7 @@
"interrupted": {
"backendRefresh": "⚠️ This turn was interrupted — its Agent runtime had to be replaced while it was still running, for example by a credential migration. Send the request again if the work still needs doing.",
"backendDisabled": "⏹ This turn was stopped because {agent} was turned off. Turn it on again in Settings, then send the request again if the work still needs doing.",
- "serviceRestart": "⚠️ This turn was interrupted — the Avibe service restarted while it was still running. Send the request again if the work still needs doing.",
+ "serviceRestart": "⚠️ This turn was interrupted — the Avibe service restarted while it was still running. To continue, choose Retry if it is shown, or send \"continue\".",
"serviceRestartWatch": "⚠️ This turn was interrupted — the Avibe service restarted while it was still running. Its running command continues as Watch {watch}, and the result will follow here.",
"serviceRestartWatches": "⚠️ This turn was interrupted — the Avibe service restarted while it was still running. Its running commands continue as Watches {watches}, and their results will follow here."
}
diff --git a/vibe/i18n/zh.json b/vibe/i18n/zh.json
index 8c120f176..2cd264957 100644
--- a/vibe/i18n/zh.json
+++ b/vibe/i18n/zh.json
@@ -215,8 +215,8 @@
"watchCircuitRepairFailed": "[Avibe Harness] Watch「{name}」的熔断修复 Agent Run 失败,Watch 将保持暂停。",
"watchFollowUpFailed": "[Avibe Harness] Watch「{name}」的后续 Agent 处理失败。",
"interrupted": "[Avibe Harness] 后台任务「{name}」被中断({reason}),任务本身没有问题。",
- "restartStopped": "[Avibe Harness] 后台任务「{name}」已随 Avibe 重启停止。如需继续,请发送“继续”给 Agent。",
- "restartStoppedUnnamed": "[Avibe Harness] 一项后台任务已随 Avibe 重启停止。如需继续,请发送“继续”给 Agent。",
+ "restartStopped": "[Avibe Harness] 后台任务「{name}」已随 Avibe 重启停止。如需继续,请点击“重试”(如有显示),或发送“继续”给 Agent。",
+ "restartStoppedUnnamed": "[Avibe Harness] 一项后台任务已随 Avibe 重启停止。如需继续,请点击“重试”(如有显示),或发送“继续”给 Agent。",
"commandLine": "命令:{command}",
"error": "错误:{error}",
"commandExit": "退出码:{exitCode}",
@@ -1289,7 +1289,7 @@
"interrupted": {
"backendRefresh": "⚠️ 本轮执行被中断——它运行期间 Agent 运行时不得不被替换(例如凭据迁移)。如果这项工作仍需完成,请重新发送请求。",
"backendDisabled": "⏹ 本轮执行已停止,因为 {agent} 已被关闭。请在设置中重新开启它;如果这项工作仍需完成,再重新发送请求。",
- "serviceRestart": "⚠️ 本轮执行被中断——Avibe 服务在它运行期间重启。如果这项工作仍然需要完成,请重新发送请求。",
+ "serviceRestart": "⚠️ 本轮执行被中断——Avibe 服务在它运行期间重启。如需继续,请点击“重试”(如有显示),或发送“继续”。",
"serviceRestartWatch": "⚠️ 本轮执行被中断——Avibe 服务在它运行期间重启。正在运行的命令已作为 Watch {watch} 继续执行,结果会发到这里。",
"serviceRestartWatches": "⚠️ 本轮执行被中断——Avibe 服务在它运行期间重启。正在运行的命令已作为 Watch {watches} 继续执行,结果会发到这里。"
}