Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(runtime): record a Turn's Run terminal off the controller loop (#2715)

Call Whitefea12e745c36
diff --git a/core/message_dispatcher.py b/core/message_dispatcher.py
index d52591eae..fde3dc878 100644
--- a/core/message_dispatcher.py
+++ b/core/message_dispatcher.py
@@ -60,6 +60,13 @@ from core.run_settlement import (
)
from core.session_activities import SessionActivity
from core.session_turns import emit_matches_active_turn
+from core.runtime_work import (
+ RuntimeWorkSupervisor,
+ answer_caught_cancellation,
+ land_step,
+ rearm_caught_cancellation,
+ step_failed,
+)
from storage.background import (
SQLiteBackgroundTaskStore,
TERMINAL_RUN_STATUSES,
@@ -1296,7 +1303,19 @@ class ConsolidatedMessageDispatcher:
terminal_error: str | None,
output_semantics: MessageOutput,
provenance: dict[str, Any],
+ blocked_run_ids: frozenset[str],
+ session_id: str | None = None,
) -> list[str]:
+ """Record a Turn's output on its Runs; may run off the loop.
+
+ ``blocked_run_ids`` is the in-memory blocking-Activity check, made on the loop
+ before this runs: the Activity registry is loop-owned, and Activity writes
+ take its lock and then SQLite's writer, so reading it under the writer would
+ take the two in the opposite order. The SQLite store adds the Runs its durable
+ Activity rows still block when the write commits, and returns the Runs it
+ deferred.
+ """
+
normalized_run_ids = list(
dict.fromkeys(
run_id
@@ -1318,11 +1337,25 @@ class ConsolidatedMessageDispatcher:
[
run_id
for run_id in normalized_run_ids
- if terminal_status and self._run_has_blocking_activity(run_id)
+ if terminal_status and run_id in blocked_run_ids
]
if terminal_status
else []
)
+ # Looked up on the class: a stand-in store answers any attribute.
+ if callable(getattr(type(store), "record_turn_terminal_outputs", None)):
+ return store.record_turn_terminal_outputs(
+ normalized_run_ids,
+ activity_session_ids=[session_id] if session_id else [],
+ output_id=output_id,
+ text=text,
+ message_id=message_id,
+ sequence=output_semantics.sequence,
+ provenance=provenance,
+ terminal_status=terminal_status,
+ error=terminal_error,
+ deferred_run_ids=deferred_run_ids,
+ )
record_turn_outputs(
normalized_run_ids,
output_id=output_id,
@@ -1364,7 +1397,7 @@ class ConsolidatedMessageDispatcher:
if callable(get_run) and _run_is_cancelled(get_run(run_id)):
continue
run_terminal_status = terminal_status
- if run_terminal_status and self._run_has_blocking_activity(run_id):
+ if run_terminal_status and run_id in blocked_run_ids:
deferred_run_ids.append(run_id)
defer_terminal = getattr(store, "defer_run_terminal", None)
if callable(defer_terminal):
@@ -1661,7 +1694,7 @@ class ConsolidatedMessageDispatcher:
return False
return fallback
- def _record_agent_run_terminal_result(
+ async def _record_agent_run_terminal_result(
self,
context: MessageContext,
text: str,
@@ -1672,6 +1705,21 @@ class ConsolidatedMessageDispatcher:
output_semantics: MessageOutput | None = None,
log_label: str = "agent run terminal result",
) -> None:
+ """Record a Turn's terminal result on its Runs, off the controller loop.
+
+ The write waits for SQLite's writer, which another process can hold for the
+ whole busy timeout, so it runs on the runtime-work door (#2706). It keeps the
+ cancellation point it had on the loop: a cancellation that arrives while the
+ write runs waits for it to land, the synchronous tail (close-after) runs, and
+ the cancellation is delivered at the caller's next await -- where it landed
+ before.
+
+ Which Runs an open Activity still blocks is decided inside the write's own
+ transaction, from the durable Activity rows (``defer_for_live_activities``),
+ so the record follows the order the loop committed the Activity events in,
+ whatever happened while it waited for the writer.
+ """
+
semantics = output_semantics or MessageOutput(completes_turn=True)
if not semantics.records_run_output:
return
@@ -1682,27 +1730,46 @@ class ConsolidatedMessageDispatcher:
terminal_status = None
if semantics.settles_run:
terminal_status = "failed" if is_error else "succeeded"
- store = None
try:
- store = SQLiteBackgroundTaskStore()
- deferred_run_ids = self._record_agent_run_terminal_for_ids(
- store=store,
- run_ids=run_ids,
- text=text,
- message_id=message_id,
- terminal_status=terminal_status,
- terminal_error=terminal_error,
- output_semantics=semantics,
- provenance=semantics.provenance(context),
- )
+ provenance = semantics.provenance(context)
except Exception as err:
logger.warning("Failed to record %s for %s: %s", log_label, ",".join(run_ids), err)
if require_confirmation:
raise
return
- finally:
- if store is not None:
+ blocked_run_ids = frozenset(
+ run_id for run_id in run_ids if terminal_status and self._run_has_blocking_activity(run_id)
+ )
+
+ def record() -> list[str]:
+ store = SQLiteBackgroundTaskStore()
+ try:
+ return self._record_agent_run_terminal_for_ids(
+ store=store,
+ run_ids=run_ids,
+ text=text,
+ message_id=message_id,
+ terminal_status=terminal_status,
+ terminal_error=terminal_error,
+ output_semantics=semantics,
+ provenance=provenance,
+ blocked_run_ids=blocked_run_ids,
+ session_id=str((getattr(context, "platform_specific", None) or {}).get("agent_session_id") or "")
+ or None,
+ )
+ finally:
store.close()
+
+ step, cancelled = await land_step(self._run_off_loop(record))
+ if step_failed(step):
+ if cancelled:
+ answer_caught_cancellation(step)
+ err = step.exception()
+ logger.warning("Failed to record %s for %s: %s", log_label, ",".join(run_ids), err)
+ if require_confirmation:
+ raise err
+ return
+ deferred_run_ids = step.result()
if semantics.settles_run:
payload = getattr(context, "platform_specific", None) or {}
defer_close_after = bool(
@@ -1722,6 +1789,14 @@ class ConsolidatedMessageDispatcher:
payload["_close_after_runtime_pending"] = True
else:
self._schedule_close_after_runtime(context)
+ if cancelled:
+ rearm_caught_cancellation()
+
+ async def _run_off_loop(self, operation: Callable[[], Any]) -> Any:
+ supervisor = getattr(self.controller, "runtime_work_supervisor", None)
+ if isinstance(supervisor, RuntimeWorkSupervisor):
+ return await supervisor.run_sync(operation)
+ return await asyncio.to_thread(operation)
async def _wait_for_close_after_runs(self, run_ids: tuple[str, ...]) -> bool:
"""Wait for the Run writer, not a Turn output, to finish disposal ownership."""
@@ -2036,7 +2111,7 @@ class ConsolidatedMessageDispatcher:
)
return []
- def _record_suppressed_agent_run_terminal_result(
+ async def _record_suppressed_agent_run_terminal_result(
self,
context: MessageContext,
text: str,
@@ -2046,7 +2121,7 @@ class ConsolidatedMessageDispatcher:
terminal_error: str | None = None,
output_semantics: MessageOutput | None = None,
) -> None:
- self._record_agent_run_terminal_result(
+ await self._record_agent_run_terminal_result(
context,
text,
message_id,
@@ -2810,7 +2885,7 @@ class ConsolidatedMessageDispatcher:
# Run completion is independent from visible Message and Turn
# completion cardinality. A detached/empty final output may
# settle its origin Run without touching the current Turn.
- self._record_agent_run_terminal_result(
+ await self._record_agent_run_terminal_result(
context,
materialize_citations(text, citations),
None,
@@ -2949,7 +3024,7 @@ class ConsolidatedMessageDispatcher:
materialize_citations(persist_text, citations),
result_footer if mutates_turn_lifecycle else None,
)
- self._record_agent_run_terminal_result(
+ await self._record_agent_run_terminal_result(
context,
duplicate_result_text,
None,
@@ -3087,7 +3162,7 @@ class ConsolidatedMessageDispatcher:
# a sidecar bound to the body the reader is actually shown.
written_recorded_text = materialize_citations(recorded_text, citations)
if canonical_type == "result" and suppressed_trigger_kind in HARNESS_TRIGGER_KINDS:
- self._record_suppressed_agent_run_terminal_result(
+ await self._record_suppressed_agent_run_terminal_result(
context,
written_recorded_text,
None,
@@ -3457,7 +3532,7 @@ class ConsolidatedMessageDispatcher:
settlement_result_text = written_result_text
if not settlement_waits_for_persistence:
- self._record_agent_run_terminal_result(
+ await self._record_agent_run_terminal_result(
context,
written_result_text,
primary_message_id,
@@ -3558,7 +3633,7 @@ class ConsolidatedMessageDispatcher:
if workbench_run_waits_for_persistence and not durable_output_exists:
raise RuntimeError("Workbench run output was not durably persisted")
try:
- self._record_agent_run_terminal_result(
+ await self._record_agent_run_terminal_result(
context,
settlement_result_text,
primary_message_id,
diff --git a/core/runtime_work.py b/core/runtime_work.py
index 7a2539d2c..525171124 100644
--- a/core/runtime_work.py
+++ b/core/runtime_work.py
@@ -68,6 +68,24 @@ def answer_caught_cancellation(step: asyncio.Future[Any]) -> NoReturn:
)
raise asyncio.CancelledError from error
+
+def rearm_caught_cancellation() -> None:
+ """Deliver a cancellation ``land_step`` caught, but the caller did not answer, at
+ the caller's next await -- where it would have landed had the step run on the loop.
+
+ ``uncancel`` first (Python 3.11+, absent on 3.10, which counts no requests), so the
+ request is counted once, as an ``asyncio.timeout`` around the caller expects.
+ """
+
+ current = asyncio.current_task()
+ if current is None:
+ return
+ uncancel = getattr(current, "uncancel", None)
+ if uncancel is not None:
+ uncancel()
+ current.cancel()
+
+
_DEFAULT_RECONCILE_INTERVAL_SECONDS = 30.0
_DEFAULT_LANE_CAPACITY = 4
_DEFAULT_SCAN_PAGE_SIZE = 32
diff --git a/core/scheduled_tasks.py b/core/scheduled_tasks.py
index 41e4c89a1..3b54a2c03 100644
--- a/core/scheduled_tasks.py
+++ b/core/scheduled_tasks.py
@@ -54,6 +54,7 @@ from core.runtime_work import (
RuntimeWorkSupervisor,
answer_caught_cancellation,
land_step,
+ rearm_caught_cancellation,
step_failed,
)
from core.run_settlement import (
@@ -4034,22 +4035,6 @@ class _ScheduledRuntimeWorkHandler(RuntimeWorkHandler):
return True
-def _rearm_cancellation() -> None:
- """Deliver a cancellation the current task caught, but did not answer, at its next await.
-
- ``uncancel`` first (Python 3.11+, absent on 3.10, which counts no requests), so the
- request is counted once, as an ``asyncio.timeout`` around the caller expects.
- """
-
- current = asyncio.current_task()
- if current is None:
- return
- uncancel = getattr(current, "uncancel", None)
- if uncancel is not None:
- uncancel()
- current.cancel()
-
-
class ScheduledTaskService:
"""Controller-owned runtime that executes persisted scheduled tasks."""
@@ -5083,7 +5068,7 @@ class ScheduledTaskService:
if cancelled:
if step_failed(step):
answer_caught_cancellation(step)
- _rearm_cancellation()
+ rearm_caught_cancellation()
return step.result()
async def _land_runtime_step(self, operation, /, *args, **kwargs): # noqa: ANN001, ANN202
diff --git a/docs/plans/controller-loop-sqlite-writers.md b/docs/plans/controller-loop-sqlite-writers.md
index 2199d5e80..c9b85b6f8 100644
--- a/docs/plans/controller-loop-sqlite-writers.md
+++ b/docs/plans/controller-loop-sqlite-writers.md
@@ -28,12 +28,18 @@ The rows are bounded by the Turn's Run ids. The store is also constructed on eve
**Orderings a move must keep.**
1. **The Run write lands before the Turn's sink is released.** Awaiting an off-loop step at the same point keeps that order. While the step is in flight, a Stop on the loop can land first and settle the Run `canceled`, which is first-writer-wins in the store. That is a legitimate outcome of a Stop racing a result, and the note accepts it.
-2. **A cancellation caught during the write is held until the whole terminal-delivery tail has completed**, not just the close-after tail. After `_record_agent_run_terminal_result`, the dispatcher still has to run `_signal_turn_complete` and release the sink. After the dispatcher returns, `Controller.emit_agent_message` calls `on_terminal_delivery_complete`, which is path B's evidence-carrying terminal write. If the cancellation were answered any earlier, the runner's `runner_release` could terminalize the Turn first, without evidence, which breaks B's ordering 1. So the terminal emit runs as one unit that a cancellation cannot split: the Run write (through `land_step`), the close-after tail, `_signal_turn_complete`, the sink release and `on_terminal_delivery_complete`. The caught cancellation is answered only after all of them have run.
+2. **A cancellation caught during the write keeps the cancellation point it had on the loop** (the `_store_step` rule from #2654). It waits for the write to land, the synchronous tail runs (held Activity settlements, close-after arming), and it is delivered at the emitter's next await -- where it landed when the write ran on the loop. The uncancellable unit is the write and that tail, with no network inside.
+
+ The rule first proposed here held the cancellation through the whole terminal emit, up to `on_terminal_delivery_complete`. In every terminal branch, though, IM network awaits (`_collapse_status_bubble`, `_clear_consolidated_state`, `_stream_chunk`, `_finish_processing_indicator_turn`) sit between the Run write and `_signal_turn_complete`, the sink release and that evidence-carrying write. That rule would hold a cancellation across network calls, so the orchestrator chose this one instead. A cancellation that lands at those network awaits can skip the evidence write and let `runner_release` terminalize first. That is the case on master too; it is unchanged here and tracked in #2714.
3. **The blocking-Activity check is made without the Activity registry's lock while SQLite's writer is held.** Activity start and completion hold `SessionActivityRegistry._lock` while their store write waits for SQLite's writer. A worker that reserved the writer and then called `has_blocking_run_activity` would take the two locks in the opposite order, which is a lock inversion. Two placements look possible, and the choice belongs to the implementation:
- the check reads the durable Activity rows for the Turn's Runs (session Activity records), inside the same transaction, after the writer is reserved;
- the in-memory check stays on the loop before the step, and the write re-validates it against the durable rows.
- Either way, the lock order stays single, and an Activity that settles in between cannot make the Turn's status get dropped.
+ As implemented: the in-memory check stays on the loop, and the transaction adds the Runs that its durable Activity rows still block (`record_turn_terminal_outputs`). The registry persists an Activity before it changes its own state, so its `active` rows are exactly the Activities it holds open. The record therefore follows the order in which the loop committed the Activity events:
+ - an Activity that started while the record waited for the writer still defers the Run, and close-after waits for it;
+ - an Activity that ended and settled the Run first stays first, with its own status, exactly as when it ended before the result on the loop.
+
+ No settlement is held or replayed. A first version held Activity terminals until the record landed; every round of review then found a new interleaving, which this rule removes at its root.
**Recommendation.** Move it to the runtime-work door (`run_sync` through `land_step`), with the three conditions above. Construct the store once rather than per call. The same dispatcher already records Run activity through the door (`_flush_agent_run_activity` → `run_sync`).
@@ -102,7 +108,7 @@ Each move needs the check this note applies to A. Does a loop section rely on th
## Plan
-- [ ] C: bound the recovery reads (start from live Runs and pending legacy notices, with a pinned plan; read-only transactions; mismatch-only status projection).
-- [ ] A: record the Turn's Run terminal through the runtime-work door with `land_step`; make the blocking-Activity check without taking the registry lock under the writer; hold a caught cancellation until the whole terminal-delivery tail, including `on_terminal_delivery_complete`, has run; construct the store once.
+- [x] C: bound the recovery reads (start from live Runs and pending legacy notices, with a pinned plan; read-only transactions; mismatch-only status projection).
+- [x] A: record the Turn's Run terminal through the runtime-work door with `land_step`, keeping the cancellation point it had on the loop; make the blocking-Activity check on the loop and re-read the durable Activity rows inside the write's transaction; build the store off the loop. The IM network tail is #2714.
- [ ] B, step 1: skip the writer for redundant terminalization, the post-terminal read and an empty drain.
- [ ] B, step 2: a design note for B's remaining write that keeps terminalization and successor selection atomic with respect to admission (orderings 1–5).
diff --git a/storage/background.py b/storage/background.py
index efd4656ac..12e461547 100644
--- a/storage/background.py
+++ b/storage/background.py
@@ -51,6 +51,7 @@ from storage.models import (
message_deliveries,
messages,
run_definitions,
+ runtime_records,
scopes,
)
from storage.pagination import PageRequest, PageResult, page_result_from_limit_plus_one
@@ -3342,6 +3343,49 @@ def enqueue_run_in_connection(conn: Any, values: dict[str, Any]) -> None:
_defer_run_ids_updated_from_connection(conn, [values["id"]])
+
+def _runs_blocked_by_live_activities(
+ conn: Any, run_ids: Sequence[str], session_ids: Iterable[str]
+) -> set[str]:
+ """The Runs among ``run_ids`` that a durable, still-active, attached Activity owns.
+
+ The durable twin of ``SessionActivityRegistry.has_blocking_run_activity``: the
+ registry persists an Activity before it changes its own state, so its ``active``
+ rows are exactly the Activities it holds open. Only the Turn's Sessions are read,
+ by ``ix_runtime_records_scope_anchor`` (an Activity row has no scope). Read under
+ the caller's writer.
+ """
+
+ from storage.session_activities import ACTIVITY_RECORD_TYPE
+
+ sessions = sorted({session_id for session_id in session_ids if session_id})
+ if not sessions:
+ return set()
+ wanted = set(run_ids)
+ blocked: set[str] = set()
+ # ``record_type`` is checked here, not in SQL: as a term it would let the planner
+ # prefer ``ix_runtime_records_type_scope_expiry``, which matches every Activity row.
+ for batch in _id_batches(sessions):
+ for record_type, payload_json in conn.execute(
+ select(runtime_records.c.record_type, runtime_records.c.payload_json)
+ .where(runtime_records.c.scope_id.is_(None))
+ .where(runtime_records.c.session_anchor.in_(batch))
+ ):
+ if record_type != ACTIVITY_RECORD_TYPE:
+ continue
+ payload = _json_loads(payload_json, {})
+ if not isinstance(payload, dict):
+ continue
+ activity = payload.get("activity")
+ if payload.get("phase") != "active" or not isinstance(activity, dict) or activity.get("detached_from_run"):
+ continue
+ owners = {str(activity.get("run_id") or "").strip()}
+ metadata = activity.get("metadata")
+ if isinstance(metadata, dict) and isinstance(metadata.get("run_ids"), list):
+ owners.update(str(value or "").strip() for value in metadata["run_ids"])
+ blocked |= owners & wanted
+ return blocked
+
class SQLiteBackgroundTaskStore:
def __init__(self, db_path: Optional[Path] = None, *, include_private_metadata: bool = True):
self._include_private_metadata = include_private_metadata
@@ -6285,6 +6329,29 @@ class SQLiteBackgroundTaskStore:
"run": run_payload,
}
+ def record_turn_terminal_outputs(
+ self,
+ run_ids: Sequence[str],
+ *,
+ activity_session_ids: Sequence[str] = (),
+ **kwargs: Any,
+ ) -> list[str]:
+ """``record_turn_run_outputs`` for a Turn's terminal result recorded off the
+ controller loop: the Runs an open Activity blocks are re-read from the durable
+ Activity rows inside the write's transaction, so an Activity that started or
+ ended while the write waited for SQLite's writer is seen as committed (#2706).
+ Returns the Runs whose terminal intent was deferred. ``activity_session_ids``
+ names the Turn's Session, read with the Runs' own."""
+
+ deferred: set[str] = set()
+ self.record_turn_run_outputs(
+ run_ids,
+ _durable_activity_blockers=deferred,
+ _activity_session_ids=activity_session_ids,
+ **kwargs,
+ )
+ return sorted(deferred)
+
def record_turn_run_outputs(
self,
run_ids: Sequence[str],
@@ -6298,6 +6365,8 @@ class SQLiteBackgroundTaskStore:
error: Optional[str] = None,
deferred_run_ids: Sequence[str] = (),
updated_at: Optional[str] = None,
+ _durable_activity_blockers: Optional[set[str]] = None,
+ _activity_session_ids: Sequence[str] = (),
) -> dict[str, dict[str, Any]]:
"""Record one Turn output across all participant Runs atomically.
@@ -6327,6 +6396,18 @@ class SQLiteBackgroundTaskStore:
results: dict[str, dict[str, Any]] = {}
with run_update_event_transaction(self.engine) as conn:
reserve_write_lock(conn)
+ if _durable_activity_blockers is not None and terminal_status:
+ run_sessions = [
+ session_id
+ for batch in _id_batches(normalized_run_ids)
+ for session_id in conn.execute(
+ select(agent_runs.c.session_id).where(agent_runs.c.id.in_(batch))
+ ).scalars()
+ ]
+ deferred_ids |= _runs_blocked_by_live_activities(
+ conn, normalized_run_ids, {*_activity_session_ids, *run_sessions}
+ )
+ _durable_activity_blockers.update(deferred_ids)
terminal_provenance = dict(provenance or {})
notification = terminal_provenance.get("turn_failure_notification")
current_owner = (
diff --git a/tests/test_agent_run_close_after.py b/tests/test_agent_run_close_after.py
index a6707b5de..8da71c3a1 100644
--- a/tests/test_agent_run_close_after.py
+++ b/tests/test_agent_run_close_after.py
@@ -1,14 +1,20 @@
from __future__ import annotations
import asyncio
+import inspect
+import threading
+import time
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, patch
+import pytest
+
from core.inbox_events import RUNS_UPDATED_EVENT, bus
from modules.im import MessageContext
from core.message_dispatcher import ConsolidatedMessageDispatcher
from core.message_output import MessageOutput, stop_output_for
from modules.agents.service import AgentService
+from storage.background import normalize_run_status
def test_close_after_releases_the_owned_runtime_after_terminal_result() -> None:
@@ -449,7 +455,7 @@ def test_detached_close_after_reserves_idle_gate_before_teardown() -> None:
):
dispatcher._terminal_agent_run_ids = lambda *_args: ["run-1"]
dispatcher._record_agent_run_terminal_for_ids = lambda **_kwargs: None
- dispatcher._record_agent_run_terminal_result(
+ await dispatcher._record_agent_run_terminal_result(
context,
"done",
None,
@@ -512,7 +518,7 @@ def test_failed_run_write_does_not_arm_close_after() -> None:
dispatcher._record_agent_run_terminal_for_ids = Mock(
side_effect=RuntimeError("SQLite write failed")
)
- dispatcher._record_agent_run_terminal_result(
+ await dispatcher._record_agent_run_terminal_result(
context,
"done",
None,
@@ -583,7 +589,7 @@ def test_blocking_activity_delays_close_after_until_run_settles() -> None:
patch("core.message_dispatcher.SQLiteBackgroundTaskStore", return_value=store),
patch("core.services.running_agents.end_running_agent", new=end_running_agent),
):
- dispatcher._record_agent_run_terminal_result(
+ await dispatcher._record_agent_run_terminal_result(
context, "done", None, is_error=False, output_semantics=output
)
assert recorded[0]["deferred_run_ids"] == ["run-1"]
@@ -1002,3 +1008,368 @@ def test_turn_only_result_keeps_close_after_runtime_for_activity_retry() -> None
end_running_agent.assert_not_awaited()
asyncio.run(exercise())
+
+
+def _agent_run_context(run_id: str) -> MessageContext:
+ return MessageContext(
+ user_id="user",
+ channel_id="session-1",
+ platform="avibe",
+ platform_specific={
+ "agent_session_id": "session-1",
+ "agent_backend": "claude",
+ "close_after": True,
+ "task_execution_id": run_id,
+ "task_trigger_kind": "agent_run",
+ "agent_session_target": {
+ "agent_backend": "claude",
+ "session_anchor": "base-session-1",
+ },
+ "agent_runtime_turn_key": "runtime-1",
+ "agent_runtime_turn_token": "turn-1",
+ },
+ )
+
+
+async def _record_terminal(dispatcher: ConsolidatedMessageDispatcher, context: MessageContext, **kwargs) -> None:
+ # The recorder became a coroutine when its write left the loop; called the same way on either side of that.
+ result = dispatcher._record_agent_run_terminal_result(context, "done", None, **kwargs)
+ if inspect.isawaitable(result):
+ await result
+
+
+def _blocking_store(writing: threading.Event, release: threading.Event, record=None) -> SimpleNamespace:
+ def record_turn_run_outputs(run_ids, **kwargs):
+ writing.set()
+ assert release.wait(5)
+ if record is not None:
+ return record(run_ids, **kwargs)
+ return None
+
+ return SimpleNamespace(record_turn_run_outputs=record_turn_run_outputs, close=lambda: None)
+
+
+def test_a_runs_terminal_record_never_holds_the_controller_loop() -> None:
+ """#2706: a Turn's terminal result is recorded on its Runs in one SQLite write transaction, which waits while
+ another process holds SQLite's writer. The emit runs on the controller loop, so the write runs off it: the loop
+ keeps turning while the write waits, and the recorder returns once it lands."""
+
+ async def exercise() -> None:
+ controller = SimpleNamespace()
+ controller.agent_service = AgentService(controller)
+ dispatcher = ConsolidatedMessageDispatcher(controller)
+ writing, release = threading.Event(), threading.Event()
+ threading.Timer(1.0, release.set).start() # the other writer lets go, eventually
+ with patch("core.message_dispatcher.SQLiteBackgroundTaskStore", return_value=_blocking_store(writing, release)):
+ record = asyncio.create_task(
+ _record_terminal(
+ dispatcher,
+ _agent_run_context("run-1"),
+ is_error=False,
+ output_semantics=MessageOutput(completes_turn=True, completes_run=True),
+ )
+ )
+ started = time.monotonic()
+ await asyncio.sleep(0.05) # the recorder starts here, and must not hold the loop
+ loop_lag = time.monotonic() - started - 0.05
+ release.set()
+ await record
+ assert writing.is_set()
+ assert loop_lag < 0.1, f"the controller loop was held {loop_lag:.2f}s by the Run terminal write"
+
+ asyncio.run(exercise())
+
+
+def test_a_cancellation_during_the_runs_terminal_record_is_answered_at_the_next_await() -> None:
+ """#2706: the Run terminal record keeps the cancellation point it had on the loop. A cancellation (a Stop, a
+ shutdown) that reaches the emitting task while the write runs off the loop waits for it: the Run row lands, the
+ close-after tail runs, and the cancellation is delivered at the emitter's next await -- never between the write and
+ the bookkeeping that depends on it."""
+
+ async def exercise() -> list[str]:
+ controller = SimpleNamespace()
+ service = AgentService(controller)
+ controller.agent_service = service
+ dispatcher = ConsolidatedMessageDispatcher(controller)
+ context = _agent_run_context("run-1")
+ gate = service._get_turn_gate("runtime-1")
+ await gate.lock.acquire()
+ gate.token = "turn-1"
+ gate.backend = "claude"
+ writing, release = threading.Event(), threading.Event()
+ recorded: list[dict] = []
+ store = _blocking_store(writing, release, record=lambda _run_ids, **kwargs: recorded.append(kwargs))
+ outcomes: list[str] = []
+
+ async def emit() -> None:
+ await _record_terminal(
+ dispatcher,
+ context,
+ is_error=False,
+ output_semantics=MessageOutput(completes_turn=True, completes_run=True),
+ )
+ outcomes.append("recorded")
+ try:
+ await asyncio.sleep(5)
+ except asyncio.CancelledError:
+ outcomes.append("cancelled at the next await")
+ raise
+
+ with patch("core.message_dispatcher.SQLiteBackgroundTaskStore", return_value=store):
+ emitter = asyncio.create_task(emit())
+ assert await asyncio.to_thread(writing.wait, 5)
+ emitter.cancel()
+ await asyncio.sleep(0.05)
+ release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await emitter
+ assert recorded, "the Run write did not land"
+ assert context.platform_specific.get("_close_after_runtime_pending") is True, "close-after was not armed"
+ gate.lock.release()
+ return outcomes
+
+ assert asyncio.run(exercise()) == ["recorded", "cancelled at the next await"]
+
+
+def test_an_activity_started_while_the_runs_terminal_record_waits_still_defers_the_run(tmp_path, monkeypatch) -> None:
+ """#2706: the Run terminal record runs off the loop, so an owned Activity can start -- durably -- while it waits
+ for SQLite's writer, after the loop's in-memory blocking check. The record re-reads the durable Activity rows inside
+ its own transaction: the Run's terminal intent is deferred while that Activity is open, and close-after waits for
+ the Run, exactly as when the Activity had started before the record."""
+
+ from core.scheduled_tasks import TaskExecutionStore
+ from storage.background import SQLiteBackgroundTaskStore
+ from storage.db import create_sqlite_engine
+ from storage.session_activities import SQLiteSessionActivityStore
+
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ request_store = TaskExecutionStore()
+ queued = request_store.enqueue_agent_run(message="run", agent_backend="claude", session_id="session-1")
+ assert request_store.claim(queued.id) is not None
+ db_path = request_store.sqlite_backend.db_path
+ writing, release = threading.Event(), threading.Event()
+
+ class _WaitsForTheWriter(SQLiteBackgroundTaskStore):
+ def record_turn_run_outputs(self, *args, **kwargs):
+ writing.set()
+ assert release.wait(5)
+ return super().record_turn_run_outputs(*args, **kwargs)
+
+ async def exercise() -> MessageContext:
+ controller = SimpleNamespace()
+ service = AgentService(controller)
+ controller.agent_service = service
+ dispatcher = ConsolidatedMessageDispatcher(controller)
+ context = _agent_run_context(queued.id)
+ gate = service._get_turn_gate("runtime-1")
+ await gate.lock.acquire()
+ gate.token = "turn-1"
+ gate.backend = "claude"
+ with patch("core.message_dispatcher.SQLiteBackgroundTaskStore", new=_WaitsForTheWriter):
+ record = asyncio.create_task(
+ _record_terminal(
+ dispatcher,
+ context,
+ is_error=False,
+ output_semantics=MessageOutput(completes_turn=True, completes_run=True),
+ )
+ )
+ assert await asyncio.to_thread(writing.wait, 5)
+ # The receiver starts an Activity owned by the same Run; it persists before the record commits.
+ SQLiteSessionActivityStore(create_sqlite_engine(db_path)).upsert_activity(
+ {
+ "id": "activity-1",
+ "backend": "claude",
+ "runtime_key": "runtime-1",
+ "session_id": "session-1",
+ "run_id": queued.id,
+ "detached_from_run": False,
+ "metadata": {},
+ },
+ phase="active",
+ )
+ release.set()
+ await record
+ gate.lock.release()
+ return context
+
+ context = asyncio.run(exercise())
+ assert normalize_run_status(request_store.get_run(queued.id)["status"]) == "running", (
+ "the record settled a Run that an open Activity still owns"
+ )
+ assert context.platform_specific.get("_close_after_wait_for_run_ids") == (queued.id,), (
+ "close-after does not wait for the Run the open Activity holds"
+ )
+
+
+def test_an_activity_that_settles_while_the_runs_terminal_record_waits_stays_first(tmp_path, monkeypatch) -> None:
+ """#2706: the other order. An owned Activity that the loop saw open ends -- durably -- and settles the Run while
+ the record waits for SQLite's writer. The Activity's settlement committed first, so it stays first: the Run keeps
+ the Activity's status and the record adds only its output, exactly as when the Activity had ended before the result
+ on the loop."""
+
+ from core.scheduled_tasks import ScheduledTaskService, ScheduledTaskStore, TaskExecutionStore
+ from storage.background import SQLiteBackgroundTaskStore
+ from storage.db import create_sqlite_engine
+ from storage.session_activities import SQLiteSessionActivityStore
+
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ request_store = TaskExecutionStore()
+ queued = request_store.enqueue_agent_run(message="run", agent_backend="claude", session_id="session-1")
+ assert request_store.claim(queued.id) is not None
+ activity_store = SQLiteSessionActivityStore(create_sqlite_engine(request_store.sqlite_backend.db_path))
+ activity_row = {
+ "id": "activity-1",
+ "backend": "claude",
+ "runtime_key": "runtime-1",
+ "session_id": "session-1",
+ "run_id": queued.id,
+ "detached_from_run": False,
+ "metadata": {},
+ }
+ activity_store.upsert_activity(activity_row, phase="active")
+ writing, release = threading.Event(), threading.Event()
+
+ class _WaitsForTheWriter(SQLiteBackgroundTaskStore):
+ def record_turn_run_outputs(self, *args, **kwargs):
+ writing.set()
+ assert release.wait(5)
+ return super().record_turn_run_outputs(*args, **kwargs)
+
+ async def exercise() -> None:
+ controller = SimpleNamespace()
+ service = AgentService(controller)
+ controller.agent_service = service
+ dispatcher = ConsolidatedMessageDispatcher(controller)
+ scheduled = ScheduledTaskService(
+ controller=controller,
+ store=ScheduledTaskStore(tmp_path / "scheduled_tasks.json"),
+ request_store=request_store,
+ )
+ blocked = [True]
+ service.activities.has_blocking_run_activity = lambda _run_id: blocked[0]
+ with patch("core.message_dispatcher.SQLiteBackgroundTaskStore", new=_WaitsForTheWriter):
+ record = asyncio.create_task(
+ _record_terminal(
+ dispatcher,
+ _agent_run_context(queued.id),
+ is_error=True,
+ terminal_error="the turn failed",
+ output_semantics=MessageOutput(completes_turn=True, completes_run=True),
+ )
+ )
+ assert await asyncio.to_thread(writing.wait, 5)
+ # The Activity ends on the loop and its Run owner settles it, before the record commits.
+ blocked[0] = False
+ activity_store.upsert_activity(activity_row, phase="awaiting_output")
+ activity = SimpleNamespace(id="activity-1", status="stopped", run_id=queued.id, metadata={})
+ assert scheduled.settle_activity_runs(activity) == [queued.id]
+ release.set()
+ await record
+
+ asyncio.run(exercise())
+ settled = request_store.get_run(queued.id)
+ assert normalize_run_status(settled["status"]) == "canceled", "the later record replaced the Activity's settlement"
+
+
+def test_the_runs_terminal_record_reads_only_its_sessions_activities(tmp_path, monkeypatch) -> None:
+ """#2706: the record re-reads open Activities under SQLite's writer, so that read must stay bounded by the Turn's
+ work: it seeks the Turn's Session through ``ix_runtime_records_scope_anchor`` instead of walking every Activity row,
+ and a malformed Activity row is skipped rather than failing every Run's terminal record."""
+
+ import sqlite3
+
+ from sqlalchemy import event
+
+ from core.scheduled_tasks import TaskExecutionStore
+ from storage.db import create_sqlite_engine
+ from storage.session_activities import SQLiteSessionActivityStore
+
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ request_store = TaskExecutionStore()
+ queued = request_store.enqueue_agent_run(message="run", agent_backend="claude", session_id="session-1")
+ assert request_store.claim(queued.id) is not None
+ store = request_store.sqlite_backend
+ engine = create_sqlite_engine(store.db_path)
+ with engine.begin() as conn:
+ conn.exec_driver_sql(
+ "insert into runtime_records (id, record_type, record_key, scope_id, session_anchor, workdir, payload_json,"
+ " expires_at, created_at, updated_at) values ('runtime::session_activity::bad', 'session_activity', 'bad',"
+ " null, 'session-1', null, '[]', null, '2026-10-16T00:00:00Z', '2026-10-16T00:00:00Z')"
+ )
+ SQLiteSessionActivityStore(engine).upsert_activity(
+ {
+ "id": "activity-elsewhere",
+ "backend": "claude",
+ "runtime_key": "runtime-2",
+ "session_id": "session-2",
+ "run_id": queued.id,
+ "detached_from_run": False,
+ "metadata": {},
+ },
+ phase="active",
+ )
+ statements: list[tuple[str, object]] = []
+
+ def capture(_conn, _cursor, statement, parameters, _context, _executemany):
+ if "runtime_records" in statement:
+ statements.append((statement, parameters))
+
+ event.listen(store.engine, "before_cursor_execute", capture)
+ try:
+ deferred = store.record_turn_terminal_outputs(
+ [queued.id],
+ activity_session_ids=["session-1"],
+ output_id="terminal",
+ text="done",
+ terminal_status="succeeded",
+ )
+ finally:
+ event.remove(store.engine, "before_cursor_execute", capture)
+
+ assert deferred == [], "an Activity in another Session blocked this Turn's Run"
+ assert normalize_run_status(request_store.get_run(queued.id)["status"]) == "succeeded"
+ (statement, parameters), = statements
+ with sqlite3.connect(store.db_path) as raw:
+ plan = " ".join(row[-1] for row in raw.execute("EXPLAIN QUERY PLAN " + statement, parameters))
+ assert "ix_runtime_records_scope_anchor" in plan and "SCAN runtime_records" not in plan, plan
+
+
+def test_the_runs_terminal_record_binds_within_an_old_sqlite_variable_limit(tmp_path, monkeypatch) -> None:
+ """#2706: a terminal output's Run ids have no size cap, and the record's Activity re-read looks up their Sessions.
+ That lookup binds in batches like the rest of the record, so an output with more Run ids than an older SQLite
+ build's 999 host parameters still records its Runs instead of failing with ``too many SQL variables``."""
+
+ import sqlite3
+
+ from sqlalchemy import event
+
+ from core.scheduled_tasks import TaskExecutionStore
+
+ if not hasattr(sqlite3.Connection, "setlimit"):
+ pytest.skip("sqlite3.Connection.setlimit needs Python 3.11")
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ request_store = TaskExecutionStore()
+ queued = request_store.enqueue_agent_run(message="run", agent_backend="claude", session_id="session-1")
+ assert request_store.claim(queued.id) is not None
+ store = request_store.sqlite_backend
+
+ def old_sqlite_limit(dbapi_connection, _record):
+ dbapi_connection.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 999)
+
+ event.listen(store.engine, "connect", old_sqlite_limit)
+ store.engine.dispose()
+ try:
+ deferred = store.record_turn_terminal_outputs(
+ [queued.id, *(f"run-gone-{index}" for index in range(1000))],
+ activity_session_ids=["session-1"],
+ output_id="terminal",
+ text="done",
+ terminal_status="succeeded",
+ )
+ finally:
+ event.remove(store.engine, "connect", old_sqlite_limit)
+ store.engine.dispose()
+
+ assert deferred == []
+ assert normalize_run_status(request_store.get_run(queued.id)["status"]) == "succeeded"
diff --git a/tests/test_message_dispatcher_result_fallback.py b/tests/test_message_dispatcher_result_fallback.py
index 68588ccf4..d600a88d5 100644
--- a/tests/test_message_dispatcher_result_fallback.py
+++ b/tests/test_message_dispatcher_result_fallback.py
@@ -247,6 +247,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):
terminal_error="stream disconnected",
output_semantics=output,
provenance=dict(output.metadata),
+ blocked_run_ids=frozenset(),
)
store.record_turn_run_outputs.assert_called_once_with(
@@ -535,7 +536,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):
dispatcher = ConsolidatedMessageDispatcher(controller)
dispatcher._collapse_status_bubble = mock.AsyncMock()
dispatcher._clear_consolidated_state = mock.AsyncMock()
- dispatcher._record_agent_run_terminal_result = mock.Mock()
+ dispatcher._record_agent_run_terminal_result = mock.AsyncMock()
context = MessageContext(
user_id="U1",
channel_id="C1",
@@ -623,7 +624,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):
dispatcher = ConsolidatedMessageDispatcher(controller)
dispatcher._collapse_status_bubble = mock.AsyncMock()
dispatcher._clear_consolidated_state = mock.AsyncMock()
- dispatcher._record_agent_run_terminal_result = mock.Mock()
+ dispatcher._record_agent_run_terminal_result = mock.AsyncMock()
platform_specific = {
"agent_runtime_turn_key": "runtime-1",
"agent_runtime_turn_token": "runtime-turn-1",
@@ -1120,7 +1121,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):
controller = _StubController(platform="avibe")
dispatcher = ConsolidatedMessageDispatcher(controller)
- dispatcher._record_agent_run_terminal_result = mock.Mock()
+ dispatcher._record_agent_run_terminal_result = mock.AsyncMock()
context = MessageContext(
user_id="U1",
channel_id="ses-1",
@@ -1153,7 +1154,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):
async def test_workbench_persistence_failure_does_not_settle_the_run(self):
controller = _StubController(platform="avibe")
dispatcher = ConsolidatedMessageDispatcher(controller)
- dispatcher._record_agent_run_terminal_result = mock.Mock()
+ dispatcher._record_agent_run_terminal_result = mock.AsyncMock()
context = MessageContext(
user_id="U1",
channel_id="ses-1",
@@ -1186,7 +1187,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):
async def test_workbench_duplicate_persistence_receipt_can_settle_the_run(self):
controller = _StubController(platform="avibe")
dispatcher = ConsolidatedMessageDispatcher(controller)
- dispatcher._record_agent_run_terminal_result = mock.Mock()
+ dispatcher._record_agent_run_terminal_result = mock.AsyncMock()
context = MessageContext(
user_id="U1",
channel_id="ses-1",
@@ -1245,7 +1246,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):
async def test_suppressed_history_id_is_not_recorded_as_run_delivery(self):
controller = _StubController(platform="slack")
dispatcher = ConsolidatedMessageDispatcher(controller)
- dispatcher._record_suppressed_agent_run_terminal_result = mock.Mock()
+ dispatcher._record_suppressed_agent_run_terminal_result = mock.AsyncMock()
context = MessageContext(
user_id="U1",
channel_id="C1",