Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(turns): commit a terminal emit's Turn write beside its signal, after the IM surfaces (#2740)

Call Whitec967ceaf7c20
diff --git a/core/controller.py b/core/controller.py
index b90761cf5..d7b0a160b 100644
--- a/core/controller.py
+++ b/core/controller.py
@@ -1970,6 +1970,9 @@ class Controller:
**({"citations": citations} if citations else {}),
**({"hosted": hosted} if hosted else {}),
)
+ # The dispatcher's terminal commit already ran this write beside the Turn's
+ # signal (#2714), which makes this call a no-op. It still covers a dispatcher
+ # that did not reach that commit.
manager = getattr(self, "session_turns", None)
complete = getattr(manager, "on_terminal_delivery_complete", None)
if callable(complete):
diff --git a/core/message_context.py b/core/message_context.py
index 2d35d2c8f..0d5275056 100644
--- a/core/message_context.py
+++ b/core/message_context.py
@@ -172,6 +172,13 @@ def build_context_turn_sink_key(
return base
+# The Turn evidence a terminal emit's commit handed to its write (#2714). The
+# dispatcher sets it on the Turn's sink beside that write; ``dispatch_turn`` moves
+# it onto the dispatch context before popping the sink, so the runner can commit
+# the same evidence when the write itself failed.
+TERMINAL_COMMIT_KEY = "terminal_commit"
+
+
def resolve_turn_sink_key(controller: object, context: MessageContext) -> str:
"""Resolve ``context``'s turn-sink key, preferring ``controller``'s own accessor.
diff --git a/core/message_dispatcher.py b/core/message_dispatcher.py
index fde3dc878..d1f06ad6f 100644
--- a/core/message_dispatcher.py
+++ b/core/message_dispatcher.py
@@ -13,10 +13,11 @@ import re
import tempfile
import time
import unicodedata
+from contextlib import asynccontextmanager
from dataclasses import dataclass, replace
from datetime import datetime, timezone
from pathlib import Path
-from typing import Any, Callable, Mapping, Optional, Sequence
+from typing import Any, AsyncIterator, Callable, Mapping, Optional, Sequence
from urllib.parse import urljoin
from modules.agents.catalog import AGENT_BACKENDS
@@ -28,7 +29,7 @@ from core.delivery_evidence import STAGE_PERSIST, STAGE_SEND, STAGE_STREAM, Deli
from core.delivery_target import routed_delivery_context
from core.citations import CitationBundle, materialize_citations
from core import failure_notices
-from core.message_context import resolve_turn_sink_key
+from core.message_context import TERMINAL_COMMIT_KEY, resolve_turn_sink_key
from core.message_mirror import (
agent_message_exists,
persist_agent_message,
@@ -239,48 +240,65 @@ async def _stream_chunk(
if completes_turn is None:
completes_turn = kind == "result"
if completes_turn:
- # The result is the turn's final answer — release the streaming dispatch
- # so it can close the SSE stream right after this chunk. Unlike chunk
- # forwarding above, the COMPLETION signal IS turn-token-gated (mirrors
- # ``Controller.mark_turn_complete`` / ``_is_active_turn``): a late ``result``
- # from a SUPERSEDED or OLDER turn (stopped / timed-out / a scheduled-watch run
- # that carries no token) resolves the CURRENT turn's sink by session key, and
- # setting its ``done_event`` would pop ``in_flight`` / publish ``turn.end`` /
- # flush the queue while the active backend is still running (Codex P1/P2).
- # When the live sink HAS a token, only a result with the MATCHING token may
- # complete it — a different OR absent token is stale. Fail-open only when the
- # sink itself is tokenless. The reused-receiver Claude case keeps completing
- # because its result emit adopts the live turn's token (see ClaudeAgent).
- if not emit_matches_active_turn(sink, context):
- return
- # Record WHY the waiter is being released, before releasing it. This is the
- # honest settlement: a real terminal result arrived, so ``dispatch_turn``
- # must leave the run's terminal state to the out-of-band writer. The
- # fallback releasers use ``setdefault``, so this value always wins over them.
- #
- # An acknowledged Stop is NOT one of those fallbacks and must not be
- # overwritten (Codex P2). ``settle_bound_turn_sink`` bails out when ``done`` is
- # already set, so reaching here with ``stopped`` recorded means the user's stop
- # won the race and was already reported to them; replacing the reason would
- # make ``_execute_agent_run`` skip the guarded ``canceled`` settlement and let
- # the outcome depend on which coroutine happened to resume first. The run's
- # terminal STATUS is still first-writer-wins in the store, so a result whose row
- # write already landed keeps its ``succeeded`` — this only stops the reason from
- # silently disagreeing with what the user was told.
- #
- # A contained backend teardown is the same case for the same reason: the
- # service retired the runtime itself and the release already named that as
- # the settlement, so a straggler result must not relabel an infrastructure
- # interruption as a healthy terminal result. Only these two are protected --
- # NOT all of ``SETTLEMENTS_WITHOUT_RESULT`` -- because
- # ``SETTLED_BY_NO_TERMINAL_RESULT`` is the pessimistic default a fallback
- # releaser writes, and upgrading THAT when a real result lands is the whole
- # point of this line.
- if sink.get("settled_by") not in (SETTLED_BY_STOPPED, SETTLED_BY_BACKEND_REFRESH, SETTLED_BY_BACKEND_DISABLED):
- sink["settled_by"] = SETTLED_BY_TERMINAL_RESULT
- done = sink.get("done_event")
- if done is not None:
- done.set()
+ _complete_turn_sink(sink, context)
+
+
+def _complete_turn_sink(sink: dict, context) -> None:
+ """Release the turn sink's waiter for a terminal result, recording why."""
+
+ # The result is the turn's final answer — release the streaming dispatch
+ # so it can close the SSE stream right after this chunk. Unlike chunk
+ # forwarding in ``_stream_chunk``, the COMPLETION signal IS turn-token-gated (mirrors
+ # ``Controller.mark_turn_complete`` / ``_is_active_turn``): a late ``result``
+ # from a SUPERSEDED or OLDER turn (stopped / timed-out / a scheduled-watch run
+ # that carries no token) resolves the CURRENT turn's sink by session key, and
+ # setting its ``done_event`` would pop ``in_flight`` / publish ``turn.end`` /
+ # flush the queue while the active backend is still running (Codex P1/P2).
+ # When the live sink HAS a token, only a result with the MATCHING token may
+ # complete it — a different OR absent token is stale. Fail-open only when the
+ # sink itself is tokenless. The reused-receiver Claude case keeps completing
+ # because its result emit adopts the live turn's token (see ClaudeAgent).
+ if not emit_matches_active_turn(sink, context):
+ return
+ # Record WHY the waiter is being released, before releasing it. This is the
+ # honest settlement: a real terminal result arrived, so ``dispatch_turn``
+ # must leave the run's terminal state to the out-of-band writer. The
+ # fallback releasers use ``setdefault``, so this value always wins over them.
+ #
+ # An acknowledged Stop is NOT one of those fallbacks and must not be
+ # overwritten (Codex P2). ``settle_bound_turn_sink`` bails out when ``done`` is
+ # already set, so reaching here with ``stopped`` recorded means the user's stop
+ # won the race and was already reported to them; replacing the reason would
+ # make ``_execute_agent_run`` skip the guarded ``canceled`` settlement and let
+ # the outcome depend on which coroutine happened to resume first. The run's
+ # terminal STATUS is still first-writer-wins in the store, so a result whose row
+ # write already landed keeps its ``succeeded`` — this only stops the reason from
+ # silently disagreeing with what the user was told.
+ #
+ # A contained backend teardown is the same case for the same reason: the
+ # service retired the runtime itself and the release already named that as
+ # the settlement, so a straggler result must not relabel an infrastructure
+ # interruption as a healthy terminal result. Only these two are protected --
+ # NOT all of ``SETTLEMENTS_WITHOUT_RESULT`` -- because
+ # ``SETTLED_BY_NO_TERMINAL_RESULT`` is the pessimistic default a fallback
+ # releaser writes, and upgrading THAT when a real result lands is the whole
+ # point of this line.
+ if sink.get("settled_by") not in (SETTLED_BY_STOPPED, SETTLED_BY_BACKEND_REFRESH, SETTLED_BY_BACKEND_DISABLED):
+ sink["settled_by"] = SETTLED_BY_TERMINAL_RESULT
+ done = sink.get("done_event")
+ if done is not None:
+ done.set()
+
+
+def _complete_streamed_turn(controller, context) -> None:
+ """The completion half of ``_stream_chunk``, for a terminal emit's Turn commit."""
+
+ get_sink = getattr(controller, "get_turn_sink", None)
+ if not callable(get_sink):
+ return
+ sink = get_sink(resolve_turn_sink_key(controller, context))
+ if sink is not None:
+ _complete_turn_sink(sink, context)
_WECHAT_TEXT_LIMIT = 1900
@@ -304,6 +322,29 @@ _INTERIM_MIN_WIDTH = 500
_RESULT_DOCUMENT_NAME = "result.md"
+@dataclass
+class _TerminalCommit:
+ """A terminal emit's Turn commit, armed once its durable steps have passed (#2714).
+
+ ``signal`` releases the Turn's waiter. ``after_signal`` is what the emit runs
+ between that signal and its return; whatever it raises is held until the
+ commit has run, so a failure there cannot separate the signal from the
+ Turn's evidence write.
+ """
+
+ output_semantics: MessageOutput
+ signal: Optional[Callable[[], None]] = None
+ after_signal: Optional[Callable[[], None]] = None
+
+ def arm(
+ self,
+ signal: Callable[[], None],
+ after_signal: Optional[Callable[[], None]] = None,
+ ) -> None:
+ self.signal = signal
+ self.after_signal = after_signal
+
+
@dataclass(frozen=True)
class _ResultSplit:
"""A proposed split, and whether every link in it survived.
@@ -467,6 +508,90 @@ class ConsolidatedMessageDispatcher:
else SETTLED_BY_TURN_ONLY_RESULT
)
+ @asynccontextmanager
+ async def _terminal_emit(
+ self,
+ context: MessageContext,
+ output_semantics: MessageOutput,
+ *,
+ mutates_turn_lifecycle: bool,
+ ) -> AsyncIterator[_TerminalCommit]:
+ """End a terminal emit: its IM surfaces first, then its Turn commit.
+
+ The emit runs its durable steps, arms the commit, then its IM surfaces.
+ On every exit, including a cancellation in a surface, the processing
+ indicator is finished and the armed commit then runs with no await
+ inside it: the signal, the emit's post-signal steps, the runtime release
+ and the Turn's evidence write. Neither the runner nor a cancellation can
+ get between the signal and that write (#2714). An emit that never armed
+ only releases the runtime, as before.
+ """
+
+ commit = _TerminalCommit(output_semantics)
+ failed = False
+ try:
+ yield commit
+ except BaseException:
+ failed = True
+ raise
+ finally:
+ if mutates_turn_lifecycle:
+ held: BaseException | None = None
+ try:
+ await self._finish_processing_indicator_turn(context)
+ finally:
+ held = self._commit_terminal_turn(context, commit)
+ if held is not None:
+ if not failed:
+ raise held
+ logger.warning(
+ "Terminal emit's post-signal step failed while the emit was already failing",
+ exc_info=held,
+ )
+
+ def _commit_terminal_turn(
+ self, context: MessageContext, commit: _TerminalCommit
+ ) -> BaseException | None:
+ """Run an armed commit synchronously; return the post-signal step's error."""
+
+ if commit.signal is None:
+ self._release_runtime_turn(context, commit.output_semantics)
+ return None
+ commit.signal()
+ held: BaseException | None = None
+ if commit.after_signal is not None:
+ try:
+ commit.after_signal()
+ except Exception as error:
+ held = error
+ try:
+ self._release_runtime_turn(context, commit.output_semantics)
+ finally:
+ self._record_terminal_commit(context)
+ manager = getattr(self.controller, "session_turns", None)
+ complete = getattr(manager, "on_terminal_delivery_complete", None)
+ if callable(complete):
+ complete(context)
+ return held
+
+ def _record_terminal_commit(self, context: MessageContext) -> None:
+ """Leave the evidence this commit hands to its Turn write on the Turn's sink.
+
+ ``dispatch_turn`` passes it to the runner, which commits it if the write
+ itself fails. Only a commit marks the sink, so a release that never
+ reached the write hands over nothing.
+ """
+
+ get_sink = getattr(self.controller, "get_turn_sink", None)
+ if not callable(get_sink):
+ return
+ sink = get_sink(resolve_turn_sink_key(self.controller, context))
+ if not isinstance(sink, dict) or not emit_matches_active_turn(sink, context):
+ return
+ evidence = sink.get("terminal_evidence")
+ if isinstance(evidence, dict):
+ sink[TERMINAL_COMMIT_KEY] = dict(evidence)
+
def _release_runtime_turn(
self, context: MessageContext, output_semantics: MessageOutput
) -> None:
@@ -2839,7 +2964,11 @@ class ConsolidatedMessageDispatcher:
# streaming — no user-facing bubble, regardless of body. An empty/stripped
# body (e.g. a ``<silent>`` directive reduced to nothing) is silent too.
if level == "silent" or not text or not text.strip():
- try:
+ async with self._terminal_emit(
+ context,
+ output_semantics,
+ mutates_turn_lifecycle=mutates_turn_lifecycle,
+ ) as commit:
if activity_local_settlement_only:
settlement_complete = self._settle_activity_output_claims(
output_semantics,
@@ -2893,34 +3022,39 @@ class ConsolidatedMessageDispatcher:
terminal_error=terminal_error,
output_semantics=output_semantics,
)
+
+ def settle_invisible_output() -> None:
+ if output_semantics.requires_delivery_for_run_settlement:
+ settlement_complete = self._settle_activity_output_claims(
+ output_semantics,
+ accepted_message_exists=False,
+ visible_output=False,
+ )
+ if settlement_complete is False:
+ raise ActivityOutputDeliveryError(
+ "Invisible Activity output local settlement is incomplete",
+ delivered=False,
+ )
+
if mutates_turn_lifecycle:
# A terminal result — even silent/empty — still means the turn
- # finished: release the streaming SSE waiter so it closes now
- # instead of hanging until the safety timeout, with no visible chunk.
- # Collapse any eagerly-posted footer-only bubble first so it
- # doesn't stay stuck (missing agent / exception / user stop).
+ # finished: the commit releases the streaming SSE waiter so it
+ # closes now instead of hanging until the safety timeout, with no
+ # visible chunk. Collapse any eagerly-posted footer-only bubble
+ # first so it doesn't stay stuck (missing agent / exception / user
+ # stop).
+ commit.arm(
+ lambda: self._signal_turn_complete(
+ context,
+ settled_by=self._turn_release_settlement(output_semantics),
+ ),
+ settle_invisible_output,
+ )
await self._collapse_status_bubble(context, im_client, reason=terminal_reason)
await self._clear_consolidated_state(context)
- self._signal_turn_complete(
- context,
- settled_by=self._turn_release_settlement(output_semantics),
- )
- if output_semantics.requires_delivery_for_run_settlement:
- settlement_complete = self._settle_activity_output_claims(
- output_semantics,
- accepted_message_exists=False,
- visible_output=False,
- )
- if settlement_complete is False:
- raise ActivityOutputDeliveryError(
- "Invisible Activity output local settlement is incomplete",
- delivered=False,
- )
+ else:
+ settle_invisible_output()
return None
- finally:
- if mutates_turn_lifecycle:
- await self._finish_processing_indicator_turn(context)
- self._release_runtime_turn(context, output_semantics)
# Resolve the delivery target once. Routed / post_to / thread replies
# land in a different channel than the source context, and the persisted
@@ -3013,7 +3147,11 @@ class ConsolidatedMessageDispatcher:
"native_message_id": native_output_id,
"evidence": "duplicate_short_circuit",
}
- try:
+ async with self._terminal_emit(
+ context,
+ accepted_output_semantics,
+ mutates_turn_lifecycle=mutates_turn_lifecycle,
+ ) as commit:
if canonical_type == "result" and accepted_output_semantics.settles_run:
accepted_is_error = self._accepted_message_is_error(
accepted_message,
@@ -3032,36 +3170,40 @@ class ConsolidatedMessageDispatcher:
terminal_error=(terminal_error if accepted_is_error else None),
output_semantics=accepted_output_semantics,
)
+
+ def settle_accepted_output() -> None:
+ if accepted_output_semantics.requires_delivery_for_run_settlement:
+ settlement_complete = self._settle_activity_output_claims(
+ accepted_output_semantics,
+ accepted_message_exists=True,
+ )
+ if settlement_complete is False:
+ raise ActivityOutputDeliveryError(
+ "Accepted Activity output has incomplete local settlement",
+ delivered=True,
+ durable=True,
+ message_id=native_output_id,
+ )
+
if mutates_turn_lifecycle:
- await self._collapse_status_bubble(context, im_client, reason=terminal_reason)
- await self._clear_consolidated_state(context)
- self._signal_turn_complete(
- context,
- settled_by=self._turn_release_settlement(
- accepted_output_semantics
+ commit.arm(
+ lambda: self._signal_turn_complete(
+ context,
+ settled_by=self._turn_release_settlement(
+ accepted_output_semantics
+ ),
),
+ settle_accepted_output,
)
- if accepted_output_semantics.requires_delivery_for_run_settlement:
- settlement_complete = self._settle_activity_output_claims(
- accepted_output_semantics,
- accepted_message_exists=True,
- )
- if settlement_complete is False:
- raise ActivityOutputDeliveryError(
- "Accepted Activity output has incomplete local settlement",
- delivered=True,
- durable=True,
- message_id=native_output_id,
- )
+ await self._collapse_status_bubble(context, im_client, reason=terminal_reason)
+ await self._clear_consolidated_state(context)
+ else:
+ settle_accepted_output()
# A previously persisted output is a successful idempotent
# delivery attempt. Return its stable identity so a recovered
# Activity can acknowledge its durable Outbox snapshot instead
# of retrying forever after a crash between emit and ack.
return native_output_id
- finally:
- if mutates_turn_lifecycle:
- await self._finish_processing_indicator_turn(context)
- self._release_runtime_turn(context, accepted_output_semantics)
if activity_batch_incomplete:
raise ActivityOutputDeliveryError(
@@ -3095,7 +3237,11 @@ class ConsolidatedMessageDispatcher:
persists_without_delivery = target_context.platform == "avibe"
if suppresses_outward_delivery:
- try:
+ async with self._terminal_emit(
+ context,
+ output_semantics,
+ mutates_turn_lifecycle=mutates_turn_lifecycle,
+ ) as commit:
recorded_text = self._fold_footer(persist_text, result_footer)
persisted_output = None
if visible_output_type:
@@ -3177,31 +3323,35 @@ class ConsolidatedMessageDispatcher:
None,
terminal_status=terminal_status,
)
+
+ def settle_suppressed_output() -> None:
+ if output_semantics.requires_delivery_for_run_settlement:
+ settlement_complete = self._settle_activity_output_claims(
+ output_semantics,
+ accepted_message_exists=persisted_output is not None,
+ visible_output=False,
+ )
+ if settlement_complete is False:
+ raise ActivityOutputDeliveryError(
+ "Suppressed Activity output local settlement is incomplete",
+ delivered=False,
+ )
+
if mutates_turn_lifecycle:
# A suppressed result still ends the turn; collapse any concise
# status bubble posted to a real channel so it doesn't stay stuck.
+ commit.arm(
+ lambda: self._signal_turn_complete(
+ context,
+ settled_by=self._turn_release_settlement(output_semantics),
+ ),
+ settle_suppressed_output,
+ )
await self._collapse_status_bubble(context, im_client, reason=terminal_reason)
await self._clear_consolidated_state(context)
- self._signal_turn_complete(
- context,
- settled_by=self._turn_release_settlement(output_semantics),
- )
- if output_semantics.requires_delivery_for_run_settlement:
- settlement_complete = self._settle_activity_output_claims(
- output_semantics,
- accepted_message_exists=persisted_output is not None,
- visible_output=False,
- )
- if settlement_complete is False:
- raise ActivityOutputDeliveryError(
- "Suppressed Activity output local settlement is incomplete",
- delivered=False,
- )
+ else:
+ settle_suppressed_output()
return local_message_id
- finally:
- if mutates_turn_lifecycle:
- await self._finish_processing_indicator_turn(context)
- self._release_runtime_turn(context, output_semantics)
if canonical_type == "notify":
# Three steps, three error scopes — deliberately NOT one blanket ``try``.
@@ -3263,7 +3413,11 @@ class ConsolidatedMessageDispatcher:
return message_id
if visible_output_type:
- try:
+ async with self._terminal_emit(
+ context,
+ output_semantics,
+ mutates_turn_lifecycle=mutates_turn_lifecycle,
+ ) as commit:
primary_message_id: Optional[str] = None
scheduled_anchor_message_id: Optional[str] = None
delivered_as_attachment = False
@@ -3451,56 +3605,12 @@ class ConsolidatedMessageDispatcher:
except Exception as split_err:
logger.error("Split message fallback also failed: %s", split_err)
- # Explain attachment-only delivery or total failure once all attempts settle.
- try:
- if delivered_as_attachment:
- notice = self._t("info.resultDeliveredAsAttachment")
- elif primary_message_id is None and display_text:
- notice = self._t("error.resultDeliveryFailed")
- else:
- notice = None
- if notice:
- await im_client.send_message(target_context, notice, parse_mode="markdown")
- except Exception:
- logger.error("Failed to send delivery status notification")
-
- # Upload extracted file attachments
- if enhanced and enhanced.files:
- await self._upload_file_links(
- im_client, target_context, _written_files(enhanced.files, citations)
- )
-
if scheduled_anchor_message_id and mutates_turn_lifecycle:
try:
self.controller.session_handler.finalize_scheduled_delivery(context, scheduled_anchor_message_id)
except Exception as err:
logger.warning("Failed to finalize scheduled delivery anchor: %s", err)
- # Retire the transient status bubble now the result is delivered as
- # its own (notifying) message: DELETE it so the turn ends as just the
- # fresh result. When the platform can't delete (or the delete fails,
- # or nothing was delivered) fall back to collapsing it to a terminal
- # marker (✅ done / ⏹ stopped|failed) so it never reads as running.
- if status_bubble_id:
- await self._retire_status_bubble(
- context,
- im_client,
- status_bubble_id,
- delivered=primary_message_id is not None,
- reason=terminal_reason,
- )
-
- # Final result closes the current turn: clear consolidated
- # assistant/tool/system message state so the next user turn starts
- # a fresh log message instead of appending to the previous one.
- # Use the key captured at the top of this branch when present so
- # teardown can't drift onto a recomputed key (C5).
- if mutates_turn_lifecycle:
- if status_consolidated_key:
- await self._drop_status_keys(status_consolidated_key)
- else:
- await self._clear_consolidated_state(context)
-
# Agent-run records keep the complete body+footer text for every
# platform. The Web message row separately persists a clean body
# plus structured footer below.
@@ -3665,49 +3775,112 @@ class ConsolidatedMessageDispatcher:
)
activity_settlement_error = err
- if primary_message_id and display_text and not output_semantics.detached:
- # Stream the delivered result to live consumers (avibe SSE).
+ # The result has reached the user (or, on avibe, its row is the
+ # delivery), and its durable steps have passed: arm the Turn commit
+ # before the IM surfaces below, so a failure or cancellation in them
+ # cannot skip it (#2714).
+ streams_result = bool(
+ primary_message_id and display_text and not output_semantics.detached
+ )
+
+ def settle_delivered_output() -> None:
+ if activity_delivery_error is not None:
+ raise activity_delivery_error
+ if output_semantics.requires_delivery_for_run_settlement:
+ settlement_complete = self._settle_activity_output_claims(
+ settlement_output_semantics,
+ accepted_message_exists=durable_output_exists,
+ settlement_error=activity_settlement_error,
+ )
+ if settlement_complete is False:
+ raise ActivityOutputDeliveryError(
+ "Activity output was delivered but local settlement is incomplete",
+ delivered=activity_was_delivered or durable_output_exists,
+ durable=durable_output_exists,
+ message_id=primary_message_id,
+ cause=activity_settlement_error,
+ )
+
+ if mutates_turn_lifecycle:
+ if streams_result:
+ commit.arm(
+ lambda: _complete_streamed_turn(self.controller, context),
+ settle_delivered_output,
+ )
+ else:
+ # A terminal result still completes the turn even if every IM
+ # delivery path failed and therefore produced no durable message
+ # id. Without this release, direct agent_run and avibe turn
+ # waiters keep waiting forever despite the backend having
+ # already finished.
+ commit.arm(
+ lambda: self._signal_turn_complete(
+ context,
+ settled_by=self._turn_release_settlement(output_semantics),
+ ),
+ settle_delivered_output,
+ )
+
+ # Explain attachment-only delivery or total failure once all attempts settle.
+ try:
+ if delivered_as_attachment:
+ notice = self._t("info.resultDeliveredAsAttachment")
+ elif primary_message_id is None and display_text:
+ notice = self._t("error.resultDeliveryFailed")
+ else:
+ notice = None
+ if notice:
+ await im_client.send_message(target_context, notice, parse_mode="markdown")
+ except Exception:
+ logger.error("Failed to send delivery status notification")
+
+ # Upload extracted file attachments
+ if enhanced and enhanced.files:
+ await self._upload_file_links(
+ im_client, target_context, _written_files(enhanced.files, citations)
+ )
+
+ # Retire the transient status bubble now the result is delivered as
+ # its own (notifying) message: DELETE it so the turn ends as just the
+ # fresh result. When the platform can't delete (or the delete fails,
+ # or nothing was delivered) fall back to collapsing it to a terminal
+ # marker (✅ done / ⏹ stopped|failed) so it never reads as running.
+ if status_bubble_id:
+ await self._retire_status_bubble(
+ context,
+ im_client,
+ status_bubble_id,
+ delivered=primary_message_id is not None,
+ reason=terminal_reason,
+ )
+
+ # Final result closes the current turn: clear consolidated
+ # assistant/tool/system message state so the next user turn starts
+ # a fresh log message instead of appending to the previous one.
+ # Use the key captured at the top of this branch when present so
+ # teardown can't drift onto a recomputed key (C5).
+ if mutates_turn_lifecycle:
+ if status_consolidated_key:
+ await self._drop_status_keys(status_consolidated_key)
+ else:
+ await self._clear_consolidated_state(context)
+
+ if streams_result:
+ # Stream the delivered result to live consumers (avibe SSE). Its
+ # completion is the Turn commit's signal.
await _stream_chunk(
self.controller,
context,
text=display_text,
message_id=primary_message_id,
kind=communication_type,
- completes_turn=mutates_turn_lifecycle,
- )
- elif mutates_turn_lifecycle:
- # A terminal result still completes the turn even if every IM
- # delivery path failed and therefore produced no durable message id.
- # Without this release, direct agent_run and avibe turn waiters keep
- # waiting forever despite the backend having already finished.
- self._signal_turn_complete(
- context,
- settled_by=self._turn_release_settlement(output_semantics),
+ completes_turn=False,
)
- if activity_delivery_error is not None:
- raise activity_delivery_error
-
- if output_semantics.requires_delivery_for_run_settlement:
- settlement_complete = self._settle_activity_output_claims(
- settlement_output_semantics,
- accepted_message_exists=durable_output_exists,
- settlement_error=activity_settlement_error,
- )
- if settlement_complete is False:
- raise ActivityOutputDeliveryError(
- "Activity output was delivered but local settlement is incomplete",
- delivered=activity_was_delivered or durable_output_exists,
- durable=durable_output_exists,
- message_id=primary_message_id,
- cause=activity_settlement_error,
- )
+ if not mutates_turn_lifecycle:
+ settle_delivered_output()
return primary_message_id
- finally:
- if mutates_turn_lifecycle:
- await self._finish_processing_indicator_turn(context)
- self._release_runtime_turn(context, output_semantics)
if canonical_type not in {"system", "assistant", "toolcall"}:
canonical_type = "assistant"
diff --git a/core/services/dispatch.py b/core/services/dispatch.py
index c1ecfe73d..2f0d21c69 100644
--- a/core/services/dispatch.py
+++ b/core/services/dispatch.py
@@ -36,7 +36,7 @@ from core.run_settlement import (
SETTLED_BY_NO_TERMINAL_RESULT,
SETTLED_BY_REFUSED_CONCURRENT_TURN,
)
-from core.message_context import resolve_turn_sink_key
+from core.message_context import TERMINAL_COMMIT_KEY, resolve_turn_sink_key
from core.native_dispatch_phase import backend_dispatch_attempted
from modules.im import MessageContext
@@ -236,6 +236,19 @@ async def dispatch_turn_with_outcome(
backend_dispatch_attempted=backend_dispatch_attempted(context),
)
finally:
+ # A terminal emit that reached its Turn write left that write's evidence
+ # on our sink. Hand it to the runner on every exit, cancellation included,
+ # since the sink goes away here.
+ sink = controller.get_turn_sink(session_key)
+ if isinstance(sink, dict) and sink.get("done_event") is done:
+ committed = sink.get(TERMINAL_COMMIT_KEY)
+ if isinstance(committed, dict):
+ if context.platform_specific is None:
+ context.platform_specific = {}
+ context.platform_specific[TERMINAL_COMMIT_KEY] = {
+ "settled_by": str(sink.get("settled_by") or ""),
+ "evidence": dict(committed),
+ }
# Pass our own done event so a turn that was superseded by a newer
# concurrent turn doesn't evict the newer turn's sink on cleanup.
controller.pop_turn_sink(session_key, done)
diff --git a/core/session_turns.py b/core/session_turns.py
index 3302bcc07..3a1227f40 100644
--- a/core/session_turns.py
+++ b/core/session_turns.py
@@ -34,6 +34,7 @@ from core.delivery_target import normalize_message_kind
from core.agent_input import AgentInputMetadata
from core.backend_failure import backend_failure_notification_output, harness_run_identity
from core.message_context import (
+ TERMINAL_COMMIT_KEY,
resolve_turn_sink_key,
)
from core.native_dispatch_phase import (
@@ -5332,10 +5333,41 @@ class SessionTurnManager:
terminal_is_error: bool,
cancel_defers_queue_resume: bool = False,
failure_evidence: dict[str, Any] | None = None,
+ terminal_commit: dict[str, Any] | None = None,
) -> dict[str, Any]:
- """Contain post-native ownership writes so runner cleanup always finishes."""
+ """Contain post-native ownership writes so runner cleanup always finishes.
+
+ ``terminal_commit`` is the evidence a terminal emit handed to its Turn write
+ (#2714). When that write already landed, the Turn is terminal and this is a
+ no-op; when it failed, the runner commits the same thing it would have.
+ """
try:
+ if terminal_commit is not None:
+ committed_settlement = (
+ str(terminal_commit.get("settled_by") or "") or settled_by
+ )
+ # The evidence is the emit's, but whether the queue moves on stays
+ # the runner's decision: a cancelled runner keeps the cancellation's
+ # resume rule below, so a shutdown never starts a successor.
+ resume = not cancelled or (
+ (settled_by or SETTLED_BY_NO_TERMINAL_RESULT) == SETTLED_BY_STOPPED
+ and not cancel_defers_queue_resume
+ )
+ result = self._terminalize_durable_turn(
+ turn_id,
+ self._durable_terminal_outcome(
+ is_error=terminal_is_error,
+ settled_by=committed_settlement,
+ ),
+ settled_by=committed_settlement or SETTLED_BY_TERMINAL_RESULT,
+ evidence_kind="terminal_result",
+ evidence=terminal_commit["evidence"],
+ resume_successors=resume,
+ )
+ if not resume:
+ result["defer_queue_resume"] = True
+ return result
if cancelled:
interruption = settled_by or SETTLED_BY_NO_TERMINAL_RESULT
resume_after_cancel = bool(
@@ -7788,6 +7820,9 @@ class SessionTurnManager:
terminal_is_error=terminal_is_error,
cancel_defers_queue_resume=cancel_defers_queue_resume,
failure_evidence=prewrite_failure_evidence(context),
+ terminal_commit=(context.platform_specific or {}).pop(
+ TERMINAL_COMMIT_KEY, None
+ ),
)
# Only definitive pre-write failure may synthesize an empty
# terminal result. Once native work may have produced output, a
diff --git a/docs/plans/terminal-emit-durable-first.md b/docs/plans/terminal-emit-durable-first.md
index 8c20ecbc4..58767ddd9 100644
--- a/docs/plans/terminal-emit-durable-first.md
+++ b/docs/plans/terminal-emit-durable-first.md
@@ -1,6 +1,6 @@
# Terminal emit: one synchronous Turn commit after the IM surfaces (#2714)
-Status: design note, proposed. No code yet.
+Status: implemented (#2714). The runner's fallback order is #2735.
## Background
@@ -70,12 +70,12 @@ The controller keeps its call after the dispatcher returns. That call is a no-op
`on_terminal_delivery_complete` first ends the Show checkpoint (`_end_show_checkpoint`). For a Session with a Show page, that runs `git` synchronously on the loop through the event bus. Master already makes this call synchronously on the loop, after the dispatcher returns, so the block keeps it where it is, before B's write, and adds no loop cost. Moving that work off the loop is #2734.
-**If B's own write fails.** `_finish_durable_terminal_result` logs and returns when `_terminalize_durable_turn` raises, for example a busy database. The runner's release would then commit without evidence. So `runner_release` carries the evidence `on_terminal_result` latched on the sink (`sink["terminal_evidence"]`), but only when the sink was released with a terminal-result settlement (`terminal_result` or `turn_only_result`). After this change, that settlement is set by the block, which attempts B in the same step. The one other setter is Claude's direct release after a failed agent-initiated settle, and it latches no evidence. `dispatch_turn` hands the sink's settlement and evidence to the runner's in-flight Turn in its `finally`, before it pops the sink. So they also survive a cancellation, where `dispatch_turn` returns no outcome. The runner's release uses them on every path, the cancelled one included, whenever the settlement is a terminal-result one. The runner then commits what B would have committed, even when a Stop is already propagating. In every other case, B wins and the runner's write is a no-op.
+**If B's own write fails.** `_finish_durable_terminal_result` logs and returns when `_terminalize_durable_turn` raises, for example a busy database. The runner's release would then commit without evidence. So the block marks the sink beside its B attempt: it copies the evidence `on_terminal_result` latched on the sink (`sink["terminal_evidence"]`) into the sink's `terminal_commit` field (`TERMINAL_COMMIT_KEY`). Only the block sets that field, so a release that never reached B, such as a backend's no-result release, hands over nothing. A bound Stop that won the sink's settlement is covered too (#2735 item 3): if the block ran before `dispatch_turn` pops the sink, the field says so. If the runner got there first, the Stop won outright: the runner commits it, and B then finds the Turn terminal. In its `finally`, before it pops the sink, `dispatch_turn` moves the field, with the sink's settlement, onto the dispatch context the runner holds. So it also survives a cancellation, where `dispatch_turn` returns no outcome. The runner's release commits it on every path, the cancelled one included, exactly as B would have: the same outcome, settlement and `terminal_result` evidence. When B landed, the Turn is already terminal and the runner's write is a no-op.
Not chosen:
- **Running the block before the IM surfaces** (this note's first version). After the release, a successor can start while the previous Turn's surfaces are still running. Status state is keyed by the thread's latest trigger message, and a successor without its own trigger, such as a quick-reply Turn (`message_id=None`), shares that key. The previous Turn's collapse and clear would then act on the successor's bubble and state. Avoiding that needs status state detached from its key at the release, which is a new mechanism. Leaving the surfaces before the release needs none.
- **Running the Activity settlement check before the signal**, and not signalling when it fails. Some failures are swallowed by their caller. For example, the Claude receiver logs a delivered Activity batch that is not durable and keeps it fail-closed; nothing else would then signal the Turn, and its runner would wait forever.
-- **Making `runner_release` carry the latched evidence unconditionally**, instead of the block. The latch is set before delivery, so on its own it does not prove the emit got that far. It also does not fix the cancellation case: a Stop that lands before the signal takes the runner's cancelled path. The fallback above therefore depends on the block's settlement.
+- **Making `runner_release` carry the latched evidence unconditionally**, instead of the block. The latch is set before delivery, so on its own it does not prove the emit got that far. It also does not fix the cancellation case: a Stop that lands before the signal takes the runner's cancelled path. The fallback above therefore depends on the block's own mark.
## What changes, and what does not
@@ -114,7 +114,7 @@ A cancellation caught during the Run terminal record (A, #2715, the `_store_step
- **The runner's evidence fallback.** Three cases:
- make `_terminalize_durable_turn` raise once for B: the runner's release commits the same evidence B would have;
- let B succeed: the runner's write is a no-op, and the snapshot keeps B's `terminal_result`;
- - release without a terminal-result settlement: the runner's write commits no evidence;
+ - a release that never reached the block: the runner's write commits no evidence;
- a Stop propagating from an IM surface while B's write fails: the runner's cancelled path commits B's evidence.
- **Existing tests that pin the current order, kept green:**
- `tests/test_controller_dispatch_loop.py`: the checkpoint ends after delivery, and a delivery failure keeps the Turn owner live;
@@ -126,9 +126,9 @@ A cancellation caught during the Run terminal record (A, #2715, the `_store_step
## Plan
-- [ ] Accept this note.
-- [ ] One change across the four terminal branches:
+- [x] Accept this note.
+- [x] One change across the four terminal branches:
- in the delivered branch, the durable steps, `finalize_scheduled_delivery` included, directly after the primary delivery;
- a synchronous terminal block that takes over master's signal-to-raise sequence and adds the release and B, run from the dispatcher's `finally` once the durable steps have passed;
- - the runner's release carrying the latched evidence after a terminal-result settlement, handed over in `dispatch_turn`'s `finally` so that the cancelled path has it too;
+ - the runner's release carrying the evidence the block marked beside its B attempt, handed over in `dispatch_turn`'s `finally` so that the cancelled path has it too;
- the tests above.
diff --git a/tests/test_terminal_emit_commit.py b/tests/test_terminal_emit_commit.py
new file mode 100644
index 000000000..c9a0469f9
--- /dev/null
+++ b/tests/test_terminal_emit_commit.py
@@ -0,0 +1,423 @@
+"""A terminal emit's Turn commit against the real runner (#2714).
+
+Each case runs the real ``SessionTurnManager._run``, the real ``dispatch_turn`` and
+the real dispatcher terminal emit, so the race between the emit's evidence-carrying
+Turn write and the runner's own release is the one production has.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+
+from core.controller import Controller
+from core.message_dispatcher import ActivityOutputDeliveryError, ConsolidatedMessageDispatcher
+from core.message_output import MessageOutput
+from core.native_dispatch_phase import DISPATCH_PHASE_ATTEMPTING, set_dispatch_phase
+from core.session_turns import SessionTurnManager
+from storage import message_deliveries as delivery_store
+from tests.test_message_dispatcher_result_fallback import (
+ _StubIMClient,
+ _StubSessionHandler,
+ _StubSettingsManager,
+)
+from core.session_turns import DeliveryRequest
+from tests.test_session_delivery_fsm import ( # noqa: F401 - pytest fixtures
+ _activate,
+ _context,
+ _fsm_schema_template,
+ managers,
+)
+
+RESULT = "the answer"
+# Bounds every wait on the runner or the emit: a Turn whose runner is never
+# released fails the test instead of hanging it.
+WAIT_S = 5
+
+
+class _Indicator:
+ """An IM processing indicator whose terminal cleanup awaits the network."""
+
+ def __init__(self) -> None:
+ self.finished = 0
+
+ async def finish_terminal_turn(self, _context) -> None:
+ await asyncio.sleep(0)
+ self.finished += 1
+
+
+class _IMClient(_StubIMClient):
+ """Accepts the done-footer subtext a status-bubble Turn delivers its result with."""
+
+ async def send_message(self, context, text, parse_mode=None, reply_to=None, **_kwargs):
+ return await super().send_message(context, text, parse_mode=parse_mode, reply_to=reply_to)
+
+
+class _Harness:
+ def __init__(self, manager: SessionTurnManager, engine, monkeypatch) -> None:
+ self.manager = manager
+ self.engine = engine
+ self.receivers: list[asyncio.Task] = []
+ self.emit: dict = {"text": RESULT}
+ controller = manager.controller
+ controller.session_turns = manager
+ controller.register_turn_sink = manager.register_turn_sink
+ controller.get_turn_sink = manager.get_turn_sink
+ controller.pop_turn_sink = manager.pop_turn_sink
+ controller.mark_turn_complete = Controller.mark_turn_complete.__get__(controller)
+ controller.emit_agent_message = Controller.emit_agent_message.__get__(controller)
+ controller.processing_indicator = _Indicator()
+ controller.im_client = _IMClient()
+ controller.get_im_client_for_context = lambda _context: controller.im_client
+ controller.get_settings_manager_for_context = lambda _context: _StubSettingsManager()
+ controller._get_settings_key = lambda context: context.channel_id
+ controller.config = SimpleNamespace(language="en", platform="avibe", reply_enhancements=False)
+ controller.session_handler = _StubSessionHandler()
+ controller.agent_service.release_runtime_turn = Mock()
+ controller.message_handler = SimpleNamespace(handle_user_message=self._handle_user_message)
+ self.controller = controller
+ self.dispatcher = ConsolidatedMessageDispatcher(controller)
+ controller.message_dispatcher = self.dispatcher
+ self.persisted: list[str] = []
+
+ def persist(_context, _type, text, **_kwargs):
+ self.persisted.append(text)
+ return {"id": f"row-{len(self.persisted)}", "native_message_id": None}
+
+ monkeypatch.setattr("core.message_dispatcher.persist_agent_message", persist)
+ monkeypatch.setattr("core.message_dispatcher.agent_message_exists", lambda *_args: None)
+
+ async def _handle_user_message(self, context, _text, **_kwargs):
+ set_dispatch_phase(context, DISPATCH_PHASE_ATTEMPTING)
+ self.receivers.append(asyncio.ensure_future(self.controller.emit_agent_message(context, "result", **self.emit)))
+ return None
+
+ async def start(self, turn_id: str, context) -> asyncio.Task:
+ # The fixture's admission records starts without running them; this Turn
+ # runs on the real runner from here.
+ self.manager._run = SessionTurnManager._run.__get__(self.manager, SessionTurnManager)
+ with self.engine.connect() as conn:
+ initial = delivery_store.initial_deliveries_for_turn(conn, turn_id)[0]
+ await self.manager._run(
+ "ses_fsm",
+ context,
+ "prompt",
+ logical_turn_id=turn_id,
+ delivery_id=str(initial["id"]),
+ durable_preallocated=True,
+ )
+ return self.manager.in_flight["ses_fsm"].task
+
+ async def receiver(self) -> asyncio.Task:
+ while not self.receivers:
+ await asyncio.sleep(0)
+ return self.receivers[0]
+
+ def turn(self, turn_id: str) -> dict:
+ with self.engine.connect() as conn:
+ turn = delivery_store.get_turn(conn, turn_id)
+ assert turn is not None
+ return turn
+
+
+def _evidence(turn: dict) -> dict:
+ return json.loads(str(turn.get("terminal_evidence_json") or "{}"))
+
+
+async def _blocked_at(dispatcher, monkeypatch, name: str) -> asyncio.Event:
+ """Make one of the emit's IM surfaces wait until the test cancels it."""
+
+ entered = asyncio.Event()
+
+ async def held(*_args, **_kwargs):
+ entered.set()
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(dispatcher, name, held)
+ return entered
+
+
+ "branch",
+ ["silent", "duplicate", "suppressed", "delivered"],
+)
+def test_a_terminal_result_reaches_the_runner_with_its_evidence(managers, monkeypatch, branch) -> None:
+ """#2714: the IM processing indicator's cleanup awaits after the Turn's signal.
+ On master the runner woke there and terminalized the Turn as ``runner_release``,
+ before the evidence-carrying write, so a late-accepted steer lost the result it
+ settles its Run from. The commit now runs signal, release and that write with no
+ await between them."""
+
+ manager, _other, engine, _engine_b, _starts = managers
+ harness = _Harness(manager, engine, monkeypatch)
+ if branch == "silent":
+ harness.emit = {"text": ""}
+ elif branch == "duplicate":
+ monkeypatch.setattr(
+ "core.message_dispatcher.agent_message_exists",
+ lambda *_args: {"id": "row-0", "text": RESULT, "metadata": {}},
+ )
+ harness.emit = {"text": RESULT, "output": MessageOutput(completes_turn=True, idempotency_key="native-1")}
+
+ async def run() -> str:
+ turn_id, context = await _activate(manager, text="prompt")
+ if branch == "suppressed":
+ context.platform_specific["suppress_delivery"] = True
+ runner = await harness.start(turn_id, context)
+ await asyncio.wait_for(runner, WAIT_S)
+ await asyncio.wait_for(asyncio.gather(*harness.receivers), WAIT_S)
+ return turn_id
+
+ turn_id = asyncio.run(run())
+
+ turn = harness.turn(turn_id)
+ assert turn["state"] == "terminal"
+ assert turn["terminal_evidence_kind"] == "terminal_result"
+ assert _evidence(turn)["result_text"] == ("" if branch == "silent" else RESULT)
+ assert harness.controller.processing_indicator.finished == 1
+ harness.controller.agent_service.release_runtime_turn.assert_called_once()
+
+
+ ("branch", "surface"),
+ [
+ ("silent", "_collapse_status_bubble"),
+ ("silent", "_clear_consolidated_state"),
+ ("delivered", "_upload_file_links"),
+ ("delivered", "_retire_status_bubble"),
+ ("delivered", "_drop_status_keys"),
+ ("delivered", "_finish_processing_indicator_turn"),
+ ],
+)
+def test_a_cancellation_in_an_im_surface_still_commits_the_turn(managers, monkeypatch, branch, surface) -> None:
+ """A Stop that lands in an IM surface after the durable steps interrupts that
+ surface, still runs the Turn commit, and then propagates. On master the collapse
+ and clear sat before the signal, and the delivered branch's uploads, bubble retire
+ and state drop sat before its durable writes, so such a Stop left the Turn to the
+ runner without evidence."""
+
+ manager, _other, engine, _engine_b, _starts = managers
+ harness = _Harness(manager, engine, monkeypatch)
+ if branch == "silent":
+ harness.emit = {"text": ""}
+ else:
+ harness.emit = {"text": f"{RESULT} [report](file:///etc/hosts)"}
+ monkeypatch.setattr(harness.dispatcher, "_concise_status_bubble_id", lambda _context: "bubble-1")
+ monkeypatch.setattr(harness.dispatcher, "_get_consolidated_message_key", lambda _context: "status-key")
+
+ async def run() -> tuple[str, BaseException | None]:
+ entered = await _blocked_at(harness.dispatcher, monkeypatch, surface)
+ turn_id, context = await _activate(manager, text="prompt")
+ runner = await harness.start(turn_id, context)
+ await asyncio.wait_for(entered.wait(), WAIT_S)
+ harness.receivers[0].cancel()
+ outcome = (await asyncio.wait_for(asyncio.gather(*harness.receivers, return_exceptions=True), WAIT_S))[0]
+ await asyncio.wait_for(runner, WAIT_S)
+ return turn_id, outcome
+
+ turn_id, outcome = asyncio.run(run())
+
+ assert isinstance(outcome, asyncio.CancelledError)
+ turn = harness.turn(turn_id)
+ assert turn["terminal_evidence_kind"] == "terminal_result"
+ assert _evidence(turn)["result_text"].startswith("" if branch == "silent" else RESULT)
+ harness.controller.agent_service.release_runtime_turn.assert_called_once()
+ if branch == "delivered":
+ assert harness.persisted, "the delivered result was not persisted before its IM surfaces"
+
+
+def test_a_held_activity_settlement_failure_still_commits_the_turn(managers, monkeypatch) -> None:
+ """The emit's post-signal step raises after the signal: its error is held until
+ the commit has run, then reaches the caller as on master. On master it skipped
+ the release and the Turn's write, and the runner won without evidence."""
+
+ manager, _other, engine, _engine_b, _starts = managers
+ harness = _Harness(manager, engine, monkeypatch)
+ harness.emit = {
+ "text": "",
+ "output": MessageOutput(completes_turn=True, requires_delivery_for_run_settlement=True),
+ }
+ monkeypatch.setattr(harness.dispatcher, "_settle_activity_output_claims", lambda *_args, **_kwargs: False)
+
+ async def run() -> tuple[str, BaseException | None]:
+ turn_id, context = await _activate(manager, text="prompt")
+ runner = await harness.start(turn_id, context)
+ await asyncio.wait_for(runner, WAIT_S)
+ outcome = (await asyncio.wait_for(asyncio.gather(*harness.receivers, return_exceptions=True), WAIT_S))[0]
+ return turn_id, outcome
+
+ turn_id, outcome = asyncio.run(run())
+
+ assert isinstance(outcome, ActivityOutputDeliveryError)
+ turn = harness.turn(turn_id)
+ assert turn["terminal_evidence_kind"] == "terminal_result"
+ harness.controller.agent_service.release_runtime_turn.assert_called_once()
+
+
+def test_a_durable_step_failure_keeps_the_turn_owner_live(managers, monkeypatch) -> None:
+ """A Workbench result whose row was not persisted fails before the signal, as on
+ master: the emit raises, the runtime is released, and neither the signal nor the
+ Turn's write runs, so the Turn stays active for exact reconciliation."""
+
+ manager, _other, engine, _engine_b, _starts = managers
+ harness = _Harness(manager, engine, monkeypatch)
+ monkeypatch.setattr("core.message_dispatcher.persist_agent_message", lambda *_args, **_kwargs: None)
+
+ async def run() -> tuple[str, BaseException | None, dict]:
+ turn_id, context = await _activate(manager, text="prompt")
+ runner = await harness.start(turn_id, context)
+ outcome = (await asyncio.wait_for(asyncio.gather(await harness.receiver(), return_exceptions=True), WAIT_S))[0]
+ turn = harness.turn(turn_id)
+ runner.cancel()
+ await asyncio.gather(runner, return_exceptions=True)
+ return turn_id, outcome, turn
+
+ _turn_id, outcome, turn = asyncio.run(run())
+
+ assert isinstance(outcome, RuntimeError)
+ assert turn["state"] == "active"
+ harness.controller.agent_service.release_runtime_turn.assert_called_once()
+
+
+def _fail_first_turn_write(manager, monkeypatch) -> list[str]:
+ """Make the first evidence-carrying Turn write raise, as a busy database would."""
+
+ original = manager._terminalize_durable_turn
+ calls: list[str] = []
+
+ def terminalize(turn_id, outcome, **kwargs):
+ calls.append(str(kwargs.get("evidence_kind")))
+ if calls == ["terminal_result"]:
+ raise RuntimeError("database is locked")
+ return original(turn_id, outcome, **kwargs)
+
+ monkeypatch.setattr(manager, "_terminalize_durable_turn", terminalize)
+ return calls
+
+
+def test_the_runner_commits_the_evidence_when_the_turn_write_fails(managers, monkeypatch) -> None:
+ """``_finish_durable_terminal_result`` logs and returns when its write raises. The
+ runner's release then commits the evidence the commit handed to that write,
+ instead of a ``runner_release`` without it."""
+
+ manager, _other, engine, _engine_b, _starts = managers
+ harness = _Harness(manager, engine, monkeypatch)
+ calls = _fail_first_turn_write(manager, monkeypatch)
+
+ async def run() -> str:
+ turn_id, context = await _activate(manager, text="prompt")
+ runner = await harness.start(turn_id, context)
+ await asyncio.wait_for(runner, WAIT_S)
+ await asyncio.wait_for(asyncio.gather(*harness.receivers), WAIT_S)
+ return turn_id
+
+ turn_id = asyncio.run(run())
+
+ assert calls == ["terminal_result", "terminal_result"]
+ turn = harness.turn(turn_id)
+ assert turn["terminal_evidence_kind"] == "terminal_result"
+ assert _evidence(turn)["result_text"] == RESULT
+
+
+def test_the_runner_write_is_a_no_op_when_the_turn_write_lands(managers, monkeypatch) -> None:
+ """With the evidence write landed, the runner's fallback finds the Turn terminal
+ and changes nothing."""
+
+ manager, _other, engine, _engine_b, _starts = managers
+ harness = _Harness(manager, engine, monkeypatch)
+ original = manager._reconcile_durable_runner_release
+ releases: list[dict] = []
+
+ def reconcile(*args, **kwargs):
+ result = original(*args, **kwargs)
+ releases.append({"terminal_commit": kwargs.get("terminal_commit"), "result": result})
+ return result
+
+ monkeypatch.setattr(manager, "_reconcile_durable_runner_release", reconcile)
+
+ async def run() -> str:
+ turn_id, context = await _activate(manager, text="prompt")
+ runner = await harness.start(turn_id, context)
+ await asyncio.wait_for(runner, WAIT_S)
+ await asyncio.wait_for(asyncio.gather(*harness.receivers), WAIT_S)
+ return turn_id
+
+ turn_id = asyncio.run(run())
+
+ (release,) = releases
+ assert release["terminal_commit"]["evidence"]["result_text"] == RESULT
+ assert release["result"]["changed"] is False
+ assert harness.turn(turn_id)["terminal_evidence_kind"] == "terminal_result"
+
+
+def test_a_release_without_a_commit_hands_the_runner_no_evidence(managers, monkeypatch) -> None:
+ """A Turn released without a terminal emit (a backend's no-result release) gives the
+ runner nothing to commit: it terminalizes as ``runner_release``, as before."""
+
+ manager, _other, engine, _engine_b, _starts = managers
+ harness = _Harness(manager, engine, monkeypatch)
+
+ async def release_without_result(context, _text, **_kwargs):
+ set_dispatch_phase(context, DISPATCH_PHASE_ATTEMPTING)
+ harness.controller.mark_turn_complete(context)
+ return None
+
+ harness.controller.message_handler = SimpleNamespace(handle_user_message=release_without_result)
+
+ async def run() -> str:
+ turn_id, context = await _activate(manager, text="prompt")
+ runner = await harness.start(turn_id, context)
+ await asyncio.wait_for(runner, WAIT_S)
+ return turn_id
+
+ turn_id = asyncio.run(run())
+
+ turn = harness.turn(turn_id)
+ assert turn["terminal_evidence_kind"] == "runner_release"
+ assert _evidence(turn) == {}
+
+
+def test_a_stop_while_the_turn_write_fails_still_commits_the_evidence(managers, monkeypatch) -> None:
+ """A cancellation reaches both the emit, in an IM surface, and the runner. The
+ commit runs and its write fails; the runner is then cancelled before it resumes,
+ so ``dispatch_turn`` returns no outcome. Its ``finally`` still hands the commit's
+ evidence over, and the runner's cancelled path commits it. Whether the queue moves
+ on stays the cancellation's call: this one names no Stop, so the backlog waits."""
+
+ manager, _other, engine, _engine_b, _starts = managers
+ harness = _Harness(manager, engine, monkeypatch)
+ harness.emit = {"text": ""}
+ harness.controller.processing_indicator.finish_terminal_turn = lambda _context: asyncio.sleep(0, None)
+ monkeypatch.setattr(harness.dispatcher, "_finish_processing_indicator_turn", _no_wait)
+ calls = _fail_first_turn_write(manager, monkeypatch)
+
+ async def run() -> tuple[str, str]:
+ entered = await _blocked_at(harness.dispatcher, monkeypatch, "_collapse_status_bubble")
+ turn_id, context = await _activate(manager, text="prompt")
+ runner = await harness.start(turn_id, context)
+ await asyncio.wait_for(entered.wait(), WAIT_S)
+ backlog = await manager.deliver(
+ DeliveryRequest(session_id="ses_fsm", priority="p3", content="backlog"),
+ context=_context(),
+ )
+ harness.receivers[0].cancel()
+ runner.cancel()
+ await asyncio.wait_for(asyncio.gather(runner, *harness.receivers, return_exceptions=True), WAIT_S)
+ return turn_id, str(backlog.delivery_id)
+
+ turn_id, backlog_id = asyncio.run(run())
+
+ assert calls == ["terminal_result", "terminal_result"]
+ turn = harness.turn(turn_id)
+ assert turn["terminal_evidence_kind"] == "terminal_result"
+ assert "result_text" in _evidence(turn)
+ with engine.connect() as conn:
+ assert delivery_store.get_delivery(conn, backlog_id)["state"] == "queued"
+
+
+async def _no_wait(_context) -> None:
+ return None