Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(runtime): record a Turn's Run terminal off the controller loop (#2715)
Call Whitefea12e745c36
diff --git a/core/message_dispatcher.py b/core/message_dispatcher.pyindex d52591eae..fde3dc878 100644--- a/core/message_dispatcher.py+++ b/core/message_dispatcher.py@@ -60,6 +60,13 @@ from core.run_settlement import ()from core.session_activities import SessionActivityfrom core.session_turns import emit_matches_active_turn+from core.runtime_work import (+ RuntimeWorkSupervisor,+ answer_caught_cancellation,+ land_step,+ rearm_caught_cancellation,+ step_failed,+)from storage.background import (SQLiteBackgroundTaskStore,TERMINAL_RUN_STATUSES,@@ -1296,7 +1303,19 @@ class ConsolidatedMessageDispatcher:terminal_error: str | None,output_semantics: MessageOutput,provenance: dict[str, Any],+ blocked_run_ids: frozenset[str],+ session_id: str | None = None,) -> list[str]:+ """Record a Turn's output on its Runs; may run off the loop.++ ``blocked_run_ids`` is the in-memory blocking-Activity check, made on the loop+ before this runs: the Activity registry is loop-owned, and Activity writes+ take its lock and then SQLite's writer, so reading it under the writer would+ take the two in the opposite order. The SQLite store adds the Runs its durable+ Activity rows still block when the write commits, and returns the Runs it+ deferred.+ """+normalized_run_ids = list(dict.fromkeys(run_id@@ -1318,11 +1337,25 @@ class ConsolidatedMessageDispatcher:[run_idfor run_id in normalized_run_ids- if terminal_status and self._run_has_blocking_activity(run_id)+ if terminal_status and run_id in blocked_run_ids]if terminal_statuselse [])+ # Looked up on the class: a stand-in store answers any attribute.+ if callable(getattr(type(store), "record_turn_terminal_outputs", None)):+ return store.record_turn_terminal_outputs(+ normalized_run_ids,+ activity_session_ids=[session_id] if session_id else [],+ output_id=output_id,+ text=text,+ message_id=message_id,+ sequence=output_semantics.sequence,+ provenance=provenance,+ terminal_status=terminal_status,+ error=terminal_error,+ deferred_run_ids=deferred_run_ids,+ )record_turn_outputs(normalized_run_ids,output_id=output_id,@@ -1364,7 +1397,7 @@ class ConsolidatedMessageDispatcher:if callable(get_run) and _run_is_cancelled(get_run(run_id)):continuerun_terminal_status = terminal_status- if run_terminal_status and self._run_has_blocking_activity(run_id):+ if run_terminal_status and run_id in blocked_run_ids:deferred_run_ids.append(run_id)defer_terminal = getattr(store, "defer_run_terminal", None)if callable(defer_terminal):@@ -1661,7 +1694,7 @@ class ConsolidatedMessageDispatcher:return Falsereturn fallback- def _record_agent_run_terminal_result(+ async def _record_agent_run_terminal_result(self,context: MessageContext,text: str,@@ -1672,6 +1705,21 @@ class ConsolidatedMessageDispatcher:output_semantics: MessageOutput | None = None,log_label: str = "agent run terminal result",) -> None:+ """Record a Turn's terminal result on its Runs, off the controller loop.++ The write waits for SQLite's writer, which another process can hold for the+ whole busy timeout, so it runs on the runtime-work door (#2706). It keeps the+ cancellation point it had on the loop: a cancellation that arrives while the+ write runs waits for it to land, the synchronous tail (close-after) runs, and+ the cancellation is delivered at the caller's next await -- where it landed+ before.++ Which Runs an open Activity still blocks is decided inside the write's own+ transaction, from the durable Activity rows (``defer_for_live_activities``),+ so the record follows the order the loop committed the Activity events in,+ whatever happened while it waited for the writer.+ """+semantics = output_semantics or MessageOutput(completes_turn=True)if not semantics.records_run_output:return@@ -1682,27 +1730,46 @@ class ConsolidatedMessageDispatcher:terminal_status = Noneif semantics.settles_run:terminal_status = "failed" if is_error else "succeeded"- store = Nonetry:- store = SQLiteBackgroundTaskStore()- deferred_run_ids = self._record_agent_run_terminal_for_ids(- store=store,- run_ids=run_ids,- text=text,- message_id=message_id,- terminal_status=terminal_status,- terminal_error=terminal_error,- output_semantics=semantics,- provenance=semantics.provenance(context),- )+ provenance = semantics.provenance(context)except Exception as err:logger.warning("Failed to record %s for %s: %s", log_label, ",".join(run_ids), err)if require_confirmation:raisereturn- finally:- if store is not None:+ blocked_run_ids = frozenset(+ run_id for run_id in run_ids if terminal_status and self._run_has_blocking_activity(run_id)+ )++ def record() -> list[str]:+ store = SQLiteBackgroundTaskStore()+ try:+ return self._record_agent_run_terminal_for_ids(+ store=store,+ run_ids=run_ids,+ text=text,+ message_id=message_id,+ terminal_status=terminal_status,+ terminal_error=terminal_error,+ output_semantics=semantics,+ provenance=provenance,+ blocked_run_ids=blocked_run_ids,+ session_id=str((getattr(context, "platform_specific", None) or {}).get("agent_session_id") or "")+ or None,+ )+ finally:store.close()++ step, cancelled = await land_step(self._run_off_loop(record))+ if step_failed(step):+ if cancelled:+ answer_caught_cancellation(step)+ err = step.exception()+ logger.warning("Failed to record %s for %s: %s", log_label, ",".join(run_ids), err)+ if require_confirmation:+ raise err+ return+ deferred_run_ids = step.result()if semantics.settles_run:payload = getattr(context, "platform_specific", None) or {}defer_close_after = bool(@@ -1722,6 +1789,14 @@ class ConsolidatedMessageDispatcher:payload["_close_after_runtime_pending"] = Trueelse:self._schedule_close_after_runtime(context)+ if cancelled:+ rearm_caught_cancellation()++ async def _run_off_loop(self, operation: Callable[[], Any]) -> Any:+ supervisor = getattr(self.controller, "runtime_work_supervisor", None)+ if isinstance(supervisor, RuntimeWorkSupervisor):+ return await supervisor.run_sync(operation)+ return await asyncio.to_thread(operation)async def _wait_for_close_after_runs(self, run_ids: tuple[str, ...]) -> bool:"""Wait for the Run writer, not a Turn output, to finish disposal ownership."""@@ -2036,7 +2111,7 @@ class ConsolidatedMessageDispatcher:)return []- def _record_suppressed_agent_run_terminal_result(+ async def _record_suppressed_agent_run_terminal_result(self,context: MessageContext,text: str,@@ -2046,7 +2121,7 @@ class ConsolidatedMessageDispatcher:terminal_error: str | None = None,output_semantics: MessageOutput | None = None,) -> None:- self._record_agent_run_terminal_result(+ await self._record_agent_run_terminal_result(context,text,message_id,@@ -2810,7 +2885,7 @@ class ConsolidatedMessageDispatcher:# Run completion is independent from visible Message and Turn# completion cardinality. A detached/empty final output may# settle its origin Run without touching the current Turn.- self._record_agent_run_terminal_result(+ await self._record_agent_run_terminal_result(context,materialize_citations(text, citations),None,@@ -2949,7 +3024,7 @@ class ConsolidatedMessageDispatcher:materialize_citations(persist_text, citations),result_footer if mutates_turn_lifecycle else None,)- self._record_agent_run_terminal_result(+ await self._record_agent_run_terminal_result(context,duplicate_result_text,None,@@ -3087,7 +3162,7 @@ class ConsolidatedMessageDispatcher:# a sidecar bound to the body the reader is actually shown.written_recorded_text = materialize_citations(recorded_text, citations)if canonical_type == "result" and suppressed_trigger_kind in HARNESS_TRIGGER_KINDS:- self._record_suppressed_agent_run_terminal_result(+ await self._record_suppressed_agent_run_terminal_result(context,written_recorded_text,None,@@ -3457,7 +3532,7 @@ class ConsolidatedMessageDispatcher:settlement_result_text = written_result_textif not settlement_waits_for_persistence:- self._record_agent_run_terminal_result(+ await self._record_agent_run_terminal_result(context,written_result_text,primary_message_id,@@ -3558,7 +3633,7 @@ class ConsolidatedMessageDispatcher:if workbench_run_waits_for_persistence and not durable_output_exists:raise RuntimeError("Workbench run output was not durably persisted")try:- self._record_agent_run_terminal_result(+ await self._record_agent_run_terminal_result(context,settlement_result_text,primary_message_id,diff --git a/core/runtime_work.py b/core/runtime_work.pyindex 7a2539d2c..525171124 100644--- a/core/runtime_work.py+++ b/core/runtime_work.py@@ -68,6 +68,24 @@ def answer_caught_cancellation(step: asyncio.Future[Any]) -> NoReturn:)raise asyncio.CancelledError from error++def rearm_caught_cancellation() -> None:+ """Deliver a cancellation ``land_step`` caught, but the caller did not answer, at+ the caller's next await -- where it would have landed had the step run on the loop.++ ``uncancel`` first (Python 3.11+, absent on 3.10, which counts no requests), so the+ request is counted once, as an ``asyncio.timeout`` around the caller expects.+ """++ current = asyncio.current_task()+ if current is None:+ return+ uncancel = getattr(current, "uncancel", None)+ if uncancel is not None:+ uncancel()+ current.cancel()++_DEFAULT_RECONCILE_INTERVAL_SECONDS = 30.0_DEFAULT_LANE_CAPACITY = 4_DEFAULT_SCAN_PAGE_SIZE = 32diff --git a/core/scheduled_tasks.py b/core/scheduled_tasks.pyindex 41e4c89a1..3b54a2c03 100644--- a/core/scheduled_tasks.py+++ b/core/scheduled_tasks.py@@ -54,6 +54,7 @@ from core.runtime_work import (RuntimeWorkSupervisor,answer_caught_cancellation,land_step,+ rearm_caught_cancellation,step_failed,)from core.run_settlement import (@@ -4034,22 +4035,6 @@ class _ScheduledRuntimeWorkHandler(RuntimeWorkHandler):return True-def _rearm_cancellation() -> None:- """Deliver a cancellation the current task caught, but did not answer, at its next await.-- ``uncancel`` first (Python 3.11+, absent on 3.10, which counts no requests), so the- request is counted once, as an ``asyncio.timeout`` around the caller expects.- """-- current = asyncio.current_task()- if current is None:- return- uncancel = getattr(current, "uncancel", None)- if uncancel is not None:- uncancel()- current.cancel()--class ScheduledTaskService:"""Controller-owned runtime that executes persisted scheduled tasks."""@@ -5083,7 +5068,7 @@ class ScheduledTaskService:if cancelled:if step_failed(step):answer_caught_cancellation(step)- _rearm_cancellation()+ rearm_caught_cancellation()return step.result()async def _land_runtime_step(self, operation, /, *args, **kwargs): # noqa: ANN001, ANN202diff --git a/docs/plans/controller-loop-sqlite-writers.md b/docs/plans/controller-loop-sqlite-writers.mdindex 2199d5e80..c9b85b6f8 100644--- a/docs/plans/controller-loop-sqlite-writers.md+++ b/docs/plans/controller-loop-sqlite-writers.md@@ -28,12 +28,18 @@ The rows are bounded by the Turn's Run ids. The store is also constructed on eve**Orderings a move must keep.**1. **The Run write lands before the Turn's sink is released.** Awaiting an off-loop step at the same point keeps that order. While the step is in flight, a Stop on the loop can land first and settle the Run `canceled`, which is first-writer-wins in the store. That is a legitimate outcome of a Stop racing a result, and the note accepts it.-2. **A cancellation caught during the write is held until the whole terminal-delivery tail has completed**, not just the close-after tail. After `_record_agent_run_terminal_result`, the dispatcher still has to run `_signal_turn_complete` and release the sink. After the dispatcher returns, `Controller.emit_agent_message` calls `on_terminal_delivery_complete`, which is path B's evidence-carrying terminal write. If the cancellation were answered any earlier, the runner's `runner_release` could terminalize the Turn first, without evidence, which breaks B's ordering 1. So the terminal emit runs as one unit that a cancellation cannot split: the Run write (through `land_step`), the close-after tail, `_signal_turn_complete`, the sink release and `on_terminal_delivery_complete`. The caught cancellation is answered only after all of them have run.+2. **A cancellation caught during the write keeps the cancellation point it had on the loop** (the `_store_step` rule from #2654). It waits for the write to land, the synchronous tail runs (held Activity settlements, close-after arming), and it is delivered at the emitter's next await -- where it landed when the write ran on the loop. The uncancellable unit is the write and that tail, with no network inside.++ The rule first proposed here held the cancellation through the whole terminal emit, up to `on_terminal_delivery_complete`. In every terminal branch, though, IM network awaits (`_collapse_status_bubble`, `_clear_consolidated_state`, `_stream_chunk`, `_finish_processing_indicator_turn`) sit between the Run write and `_signal_turn_complete`, the sink release and that evidence-carrying write. That rule would hold a cancellation across network calls, so the orchestrator chose this one instead. A cancellation that lands at those network awaits can skip the evidence write and let `runner_release` terminalize first. That is the case on master too; it is unchanged here and tracked in #2714.3. **The blocking-Activity check is made without the Activity registry's lock while SQLite's writer is held.** Activity start and completion hold `SessionActivityRegistry._lock` while their store write waits for SQLite's writer. A worker that reserved the writer and then called `has_blocking_run_activity` would take the two locks in the opposite order, which is a lock inversion. Two placements look possible, and the choice belongs to the implementation:- the check reads the durable Activity rows for the Turn's Runs (session Activity records), inside the same transaction, after the writer is reserved;- the in-memory check stays on the loop before the step, and the write re-validates it against the durable rows.- Either way, the lock order stays single, and an Activity that settles in between cannot make the Turn's status get dropped.+ As implemented: the in-memory check stays on the loop, and the transaction adds the Runs that its durable Activity rows still block (`record_turn_terminal_outputs`). The registry persists an Activity before it changes its own state, so its `active` rows are exactly the Activities it holds open. The record therefore follows the order in which the loop committed the Activity events:+ - an Activity that started while the record waited for the writer still defers the Run, and close-after waits for it;+ - an Activity that ended and settled the Run first stays first, with its own status, exactly as when it ended before the result on the loop.++ No settlement is held or replayed. A first version held Activity terminals until the record landed; every round of review then found a new interleaving, which this rule removes at its root.**Recommendation.** Move it to the runtime-work door (`run_sync` through `land_step`), with the three conditions above. Construct the store once rather than per call. The same dispatcher already records Run activity through the door (`_flush_agent_run_activity` → `run_sync`).@@ -102,7 +108,7 @@ Each move needs the check this note applies to A. Does a loop section rely on th## Plan-- [ ] C: bound the recovery reads (start from live Runs and pending legacy notices, with a pinned plan; read-only transactions; mismatch-only status projection).-- [ ] A: record the Turn's Run terminal through the runtime-work door with `land_step`; make the blocking-Activity check without taking the registry lock under the writer; hold a caught cancellation until the whole terminal-delivery tail, including `on_terminal_delivery_complete`, has run; construct the store once.+- [x] C: bound the recovery reads (start from live Runs and pending legacy notices, with a pinned plan; read-only transactions; mismatch-only status projection).+- [x] A: record the Turn's Run terminal through the runtime-work door with `land_step`, keeping the cancellation point it had on the loop; make the blocking-Activity check on the loop and re-read the durable Activity rows inside the write's transaction; build the store off the loop. The IM network tail is #2714.- [ ] B, step 1: skip the writer for redundant terminalization, the post-terminal read and an empty drain.- [ ] B, step 2: a design note for B's remaining write that keeps terminalization and successor selection atomic with respect to admission (orderings 1–5).diff --git a/storage/background.py b/storage/background.pyindex efd4656ac..12e461547 100644--- a/storage/background.py+++ b/storage/background.py@@ -51,6 +51,7 @@ from storage.models import (message_deliveries,messages,run_definitions,+ runtime_records,scopes,)from storage.pagination import PageRequest, PageResult, page_result_from_limit_plus_one@@ -3342,6 +3343,49 @@ def enqueue_run_in_connection(conn: Any, values: dict[str, Any]) -> None:_defer_run_ids_updated_from_connection(conn, [values["id"]])++def _runs_blocked_by_live_activities(+ conn: Any, run_ids: Sequence[str], session_ids: Iterable[str]+) -> set[str]:+ """The Runs among ``run_ids`` that a durable, still-active, attached Activity owns.++ The durable twin of ``SessionActivityRegistry.has_blocking_run_activity``: the+ registry persists an Activity before it changes its own state, so its ``active``+ rows are exactly the Activities it holds open. Only the Turn's Sessions are read,+ by ``ix_runtime_records_scope_anchor`` (an Activity row has no scope). Read under+ the caller's writer.+ """++ from storage.session_activities import ACTIVITY_RECORD_TYPE++ sessions = sorted({session_id for session_id in session_ids if session_id})+ if not sessions:+ return set()+ wanted = set(run_ids)+ blocked: set[str] = set()+ # ``record_type`` is checked here, not in SQL: as a term it would let the planner+ # prefer ``ix_runtime_records_type_scope_expiry``, which matches every Activity row.+ for batch in _id_batches(sessions):+ for record_type, payload_json in conn.execute(+ select(runtime_records.c.record_type, runtime_records.c.payload_json)+ .where(runtime_records.c.scope_id.is_(None))+ .where(runtime_records.c.session_anchor.in_(batch))+ ):+ if record_type != ACTIVITY_RECORD_TYPE:+ continue+ payload = _json_loads(payload_json, {})+ if not isinstance(payload, dict):+ continue+ activity = payload.get("activity")+ if payload.get("phase") != "active" or not isinstance(activity, dict) or activity.get("detached_from_run"):+ continue+ owners = {str(activity.get("run_id") or "").strip()}+ metadata = activity.get("metadata")+ if isinstance(metadata, dict) and isinstance(metadata.get("run_ids"), list):+ owners.update(str(value or "").strip() for value in metadata["run_ids"])+ blocked |= owners & wanted+ return blocked+class SQLiteBackgroundTaskStore:def __init__(self, db_path: Optional[Path] = None, *, include_private_metadata: bool = True):self._include_private_metadata = include_private_metadata@@ -6285,6 +6329,29 @@ class SQLiteBackgroundTaskStore:"run": run_payload,}+ def record_turn_terminal_outputs(+ self,+ run_ids: Sequence[str],+ *,+ activity_session_ids: Sequence[str] = (),+ **kwargs: Any,+ ) -> list[str]:+ """``record_turn_run_outputs`` for a Turn's terminal result recorded off the+ controller loop: the Runs an open Activity blocks are re-read from the durable+ Activity rows inside the write's transaction, so an Activity that started or+ ended while the write waited for SQLite's writer is seen as committed (#2706).+ Returns the Runs whose terminal intent was deferred. ``activity_session_ids``+ names the Turn's Session, read with the Runs' own."""++ deferred: set[str] = set()+ self.record_turn_run_outputs(+ run_ids,+ _durable_activity_blockers=deferred,+ _activity_session_ids=activity_session_ids,+ **kwargs,+ )+ return sorted(deferred)+def record_turn_run_outputs(self,run_ids: Sequence[str],@@ -6298,6 +6365,8 @@ class SQLiteBackgroundTaskStore:error: Optional[str] = None,deferred_run_ids: Sequence[str] = (),updated_at: Optional[str] = None,+ _durable_activity_blockers: Optional[set[str]] = None,+ _activity_session_ids: Sequence[str] = (),) -> dict[str, dict[str, Any]]:"""Record one Turn output across all participant Runs atomically.@@ -6327,6 +6396,18 @@ class SQLiteBackgroundTaskStore:results: dict[str, dict[str, Any]] = {}with run_update_event_transaction(self.engine) as conn:reserve_write_lock(conn)+ if _durable_activity_blockers is not None and terminal_status:+ run_sessions = [+ session_id+ for batch in _id_batches(normalized_run_ids)+ for session_id in conn.execute(+ select(agent_runs.c.session_id).where(agent_runs.c.id.in_(batch))+ ).scalars()+ ]+ deferred_ids |= _runs_blocked_by_live_activities(+ conn, normalized_run_ids, {*_activity_session_ids, *run_sessions}+ )+ _durable_activity_blockers.update(deferred_ids)terminal_provenance = dict(provenance or {})notification = terminal_provenance.get("turn_failure_notification")current_owner = (diff --git a/tests/test_agent_run_close_after.py b/tests/test_agent_run_close_after.pyindex a6707b5de..8da71c3a1 100644--- a/tests/test_agent_run_close_after.py+++ b/tests/test_agent_run_close_after.py@@ -1,14 +1,20 @@from __future__ import annotationsimport asyncio+import inspect+import threading+import timefrom types import SimpleNamespacefrom unittest.mock import AsyncMock, Mock, patch+import pytest+from core.inbox_events import RUNS_UPDATED_EVENT, busfrom modules.im import MessageContextfrom core.message_dispatcher import ConsolidatedMessageDispatcherfrom core.message_output import MessageOutput, stop_output_forfrom modules.agents.service import AgentService+from storage.background import normalize_run_statusdef test_close_after_releases_the_owned_runtime_after_terminal_result() -> None:@@ -449,7 +455,7 @@ def test_detached_close_after_reserves_idle_gate_before_teardown() -> None:):dispatcher._terminal_agent_run_ids = lambda *_args: ["run-1"]dispatcher._record_agent_run_terminal_for_ids = lambda **_kwargs: None- dispatcher._record_agent_run_terminal_result(+ await dispatcher._record_agent_run_terminal_result(context,"done",None,@@ -512,7 +518,7 @@ def test_failed_run_write_does_not_arm_close_after() -> None:dispatcher._record_agent_run_terminal_for_ids = Mock(side_effect=RuntimeError("SQLite write failed"))- dispatcher._record_agent_run_terminal_result(+ await dispatcher._record_agent_run_terminal_result(context,"done",None,@@ -583,7 +589,7 @@ def test_blocking_activity_delays_close_after_until_run_settles() -> None:patch("core.message_dispatcher.SQLiteBackgroundTaskStore", return_value=store),patch("core.services.running_agents.end_running_agent", new=end_running_agent),):- dispatcher._record_agent_run_terminal_result(+ await dispatcher._record_agent_run_terminal_result(context, "done", None, is_error=False, output_semantics=output)assert recorded[0]["deferred_run_ids"] == ["run-1"]@@ -1002,3 +1008,368 @@ def test_turn_only_result_keeps_close_after_runtime_for_activity_retry() -> Noneend_running_agent.assert_not_awaited()asyncio.run(exercise())+++def _agent_run_context(run_id: str) -> MessageContext:+ return MessageContext(+ user_id="user",+ channel_id="session-1",+ platform="avibe",+ platform_specific={+ "agent_session_id": "session-1",+ "agent_backend": "claude",+ "close_after": True,+ "task_execution_id": run_id,+ "task_trigger_kind": "agent_run",+ "agent_session_target": {+ "agent_backend": "claude",+ "session_anchor": "base-session-1",+ },+ "agent_runtime_turn_key": "runtime-1",+ "agent_runtime_turn_token": "turn-1",+ },+ )+++async def _record_terminal(dispatcher: ConsolidatedMessageDispatcher, context: MessageContext, **kwargs) -> None:+ # The recorder became a coroutine when its write left the loop; called the same way on either side of that.+ result = dispatcher._record_agent_run_terminal_result(context, "done", None, **kwargs)+ if inspect.isawaitable(result):+ await result+++def _blocking_store(writing: threading.Event, release: threading.Event, record=None) -> SimpleNamespace:+ def record_turn_run_outputs(run_ids, **kwargs):+ writing.set()+ assert release.wait(5)+ if record is not None:+ return record(run_ids, **kwargs)+ return None++ return SimpleNamespace(record_turn_run_outputs=record_turn_run_outputs, close=lambda: None)+++def test_a_runs_terminal_record_never_holds_the_controller_loop() -> None:+ """#2706: a Turn's terminal result is recorded on its Runs in one SQLite write transaction, which waits while+ another process holds SQLite's writer. The emit runs on the controller loop, so the write runs off it: the loop+ keeps turning while the write waits, and the recorder returns once it lands."""++ async def exercise() -> None:+ controller = SimpleNamespace()+ controller.agent_service = AgentService(controller)+ dispatcher = ConsolidatedMessageDispatcher(controller)+ writing, release = threading.Event(), threading.Event()+ threading.Timer(1.0, release.set).start() # the other writer lets go, eventually+ with patch("core.message_dispatcher.SQLiteBackgroundTaskStore", return_value=_blocking_store(writing, release)):+ record = asyncio.create_task(+ _record_terminal(+ dispatcher,+ _agent_run_context("run-1"),+ is_error=False,+ output_semantics=MessageOutput(completes_turn=True, completes_run=True),+ )+ )+ started = time.monotonic()+ await asyncio.sleep(0.05) # the recorder starts here, and must not hold the loop+ loop_lag = time.monotonic() - started - 0.05+ release.set()+ await record+ assert writing.is_set()+ assert loop_lag < 0.1, f"the controller loop was held {loop_lag:.2f}s by the Run terminal write"++ asyncio.run(exercise())+++def test_a_cancellation_during_the_runs_terminal_record_is_answered_at_the_next_await() -> None:+ """#2706: the Run terminal record keeps the cancellation point it had on the loop. A cancellation (a Stop, a+ shutdown) that reaches the emitting task while the write runs off the loop waits for it: the Run row lands, the+ close-after tail runs, and the cancellation is delivered at the emitter's next await -- never between the write and+ the bookkeeping that depends on it."""++ async def exercise() -> list[str]:+ controller = SimpleNamespace()+ service = AgentService(controller)+ controller.agent_service = service+ dispatcher = ConsolidatedMessageDispatcher(controller)+ context = _agent_run_context("run-1")+ gate = service._get_turn_gate("runtime-1")+ await gate.lock.acquire()+ gate.token = "turn-1"+ gate.backend = "claude"+ writing, release = threading.Event(), threading.Event()+ recorded: list[dict] = []+ store = _blocking_store(writing, release, record=lambda _run_ids, **kwargs: recorded.append(kwargs))+ outcomes: list[str] = []++ async def emit() -> None:+ await _record_terminal(+ dispatcher,+ context,+ is_error=False,+ output_semantics=MessageOutput(completes_turn=True, completes_run=True),+ )+ outcomes.append("recorded")+ try:+ await asyncio.sleep(5)+ except asyncio.CancelledError:+ outcomes.append("cancelled at the next await")+ raise++ with patch("core.message_dispatcher.SQLiteBackgroundTaskStore", return_value=store):+ emitter = asyncio.create_task(emit())+ assert await asyncio.to_thread(writing.wait, 5)+ emitter.cancel()+ await asyncio.sleep(0.05)+ release.set()+ with pytest.raises(asyncio.CancelledError):+ await emitter+ assert recorded, "the Run write did not land"+ assert context.platform_specific.get("_close_after_runtime_pending") is True, "close-after was not armed"+ gate.lock.release()+ return outcomes++ assert asyncio.run(exercise()) == ["recorded", "cancelled at the next await"]+++def test_an_activity_started_while_the_runs_terminal_record_waits_still_defers_the_run(tmp_path, monkeypatch) -> None:+ """#2706: the Run terminal record runs off the loop, so an owned Activity can start -- durably -- while it waits+ for SQLite's writer, after the loop's in-memory blocking check. The record re-reads the durable Activity rows inside+ its own transaction: the Run's terminal intent is deferred while that Activity is open, and close-after waits for+ the Run, exactly as when the Activity had started before the record."""++ from core.scheduled_tasks import TaskExecutionStore+ from storage.background import SQLiteBackgroundTaskStore+ from storage.db import create_sqlite_engine+ from storage.session_activities import SQLiteSessionActivityStore++ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))+ request_store = TaskExecutionStore()+ queued = request_store.enqueue_agent_run(message="run", agent_backend="claude", session_id="session-1")+ assert request_store.claim(queued.id) is not None+ db_path = request_store.sqlite_backend.db_path+ writing, release = threading.Event(), threading.Event()++ class _WaitsForTheWriter(SQLiteBackgroundTaskStore):+ def record_turn_run_outputs(self, *args, **kwargs):+ writing.set()+ assert release.wait(5)+ return super().record_turn_run_outputs(*args, **kwargs)++ async def exercise() -> MessageContext:+ controller = SimpleNamespace()+ service = AgentService(controller)+ controller.agent_service = service+ dispatcher = ConsolidatedMessageDispatcher(controller)+ context = _agent_run_context(queued.id)+ gate = service._get_turn_gate("runtime-1")+ await gate.lock.acquire()+ gate.token = "turn-1"+ gate.backend = "claude"+ with patch("core.message_dispatcher.SQLiteBackgroundTaskStore", new=_WaitsForTheWriter):+ record = asyncio.create_task(+ _record_terminal(+ dispatcher,+ context,+ is_error=False,+ output_semantics=MessageOutput(completes_turn=True, completes_run=True),+ )+ )+ assert await asyncio.to_thread(writing.wait, 5)+ # The receiver starts an Activity owned by the same Run; it persists before the record commits.+ SQLiteSessionActivityStore(create_sqlite_engine(db_path)).upsert_activity(+ {+ "id": "activity-1",+ "backend": "claude",+ "runtime_key": "runtime-1",+ "session_id": "session-1",+ "run_id": queued.id,+ "detached_from_run": False,+ "metadata": {},+ },+ phase="active",+ )+ release.set()+ await record+ gate.lock.release()+ return context++ context = asyncio.run(exercise())+ assert normalize_run_status(request_store.get_run(queued.id)["status"]) == "running", (+ "the record settled a Run that an open Activity still owns"+ )+ assert context.platform_specific.get("_close_after_wait_for_run_ids") == (queued.id,), (+ "close-after does not wait for the Run the open Activity holds"+ )+++def test_an_activity_that_settles_while_the_runs_terminal_record_waits_stays_first(tmp_path, monkeypatch) -> None:+ """#2706: the other order. An owned Activity that the loop saw open ends -- durably -- and settles the Run while+ the record waits for SQLite's writer. The Activity's settlement committed first, so it stays first: the Run keeps+ the Activity's status and the record adds only its output, exactly as when the Activity had ended before the result+ on the loop."""++ from core.scheduled_tasks import ScheduledTaskService, ScheduledTaskStore, TaskExecutionStore+ from storage.background import SQLiteBackgroundTaskStore+ from storage.db import create_sqlite_engine+ from storage.session_activities import SQLiteSessionActivityStore++ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))+ request_store = TaskExecutionStore()+ queued = request_store.enqueue_agent_run(message="run", agent_backend="claude", session_id="session-1")+ assert request_store.claim(queued.id) is not None+ activity_store = SQLiteSessionActivityStore(create_sqlite_engine(request_store.sqlite_backend.db_path))+ activity_row = {+ "id": "activity-1",+ "backend": "claude",+ "runtime_key": "runtime-1",+ "session_id": "session-1",+ "run_id": queued.id,+ "detached_from_run": False,+ "metadata": {},+ }+ activity_store.upsert_activity(activity_row, phase="active")+ writing, release = threading.Event(), threading.Event()++ class _WaitsForTheWriter(SQLiteBackgroundTaskStore):+ def record_turn_run_outputs(self, *args, **kwargs):+ writing.set()+ assert release.wait(5)+ return super().record_turn_run_outputs(*args, **kwargs)++ async def exercise() -> None:+ controller = SimpleNamespace()+ service = AgentService(controller)+ controller.agent_service = service+ dispatcher = ConsolidatedMessageDispatcher(controller)+ scheduled = ScheduledTaskService(+ controller=controller,+ store=ScheduledTaskStore(tmp_path / "scheduled_tasks.json"),+ request_store=request_store,+ )+ blocked = [True]+ service.activities.has_blocking_run_activity = lambda _run_id: blocked[0]+ with patch("core.message_dispatcher.SQLiteBackgroundTaskStore", new=_WaitsForTheWriter):+ record = asyncio.create_task(+ _record_terminal(+ dispatcher,+ _agent_run_context(queued.id),+ is_error=True,+ terminal_error="the turn failed",+ output_semantics=MessageOutput(completes_turn=True, completes_run=True),+ )+ )+ assert await asyncio.to_thread(writing.wait, 5)+ # The Activity ends on the loop and its Run owner settles it, before the record commits.+ blocked[0] = False+ activity_store.upsert_activity(activity_row, phase="awaiting_output")+ activity = SimpleNamespace(id="activity-1", status="stopped", run_id=queued.id, metadata={})+ assert scheduled.settle_activity_runs(activity) == [queued.id]+ release.set()+ await record++ asyncio.run(exercise())+ settled = request_store.get_run(queued.id)+ assert normalize_run_status(settled["status"]) == "canceled", "the later record replaced the Activity's settlement"+++def test_the_runs_terminal_record_reads_only_its_sessions_activities(tmp_path, monkeypatch) -> None:+ """#2706: the record re-reads open Activities under SQLite's writer, so that read must stay bounded by the Turn's+ work: it seeks the Turn's Session through ``ix_runtime_records_scope_anchor`` instead of walking every Activity row,+ and a malformed Activity row is skipped rather than failing every Run's terminal record."""++ import sqlite3++ from sqlalchemy import event++ from core.scheduled_tasks import TaskExecutionStore+ from storage.db import create_sqlite_engine+ from storage.session_activities import SQLiteSessionActivityStore++ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))+ request_store = TaskExecutionStore()+ queued = request_store.enqueue_agent_run(message="run", agent_backend="claude", session_id="session-1")+ assert request_store.claim(queued.id) is not None+ store = request_store.sqlite_backend+ engine = create_sqlite_engine(store.db_path)+ with engine.begin() as conn:+ conn.exec_driver_sql(+ "insert into runtime_records (id, record_type, record_key, scope_id, session_anchor, workdir, payload_json,"+ " expires_at, created_at, updated_at) values ('runtime::session_activity::bad', 'session_activity', 'bad',"+ " null, 'session-1', null, '[]', null, '2026-10-16T00:00:00Z', '2026-10-16T00:00:00Z')"+ )+ SQLiteSessionActivityStore(engine).upsert_activity(+ {+ "id": "activity-elsewhere",+ "backend": "claude",+ "runtime_key": "runtime-2",+ "session_id": "session-2",+ "run_id": queued.id,+ "detached_from_run": False,+ "metadata": {},+ },+ phase="active",+ )+ statements: list[tuple[str, object]] = []++ def capture(_conn, _cursor, statement, parameters, _context, _executemany):+ if "runtime_records" in statement:+ statements.append((statement, parameters))++ event.listen(store.engine, "before_cursor_execute", capture)+ try:+ deferred = store.record_turn_terminal_outputs(+ [queued.id],+ activity_session_ids=["session-1"],+ output_id="terminal",+ text="done",+ terminal_status="succeeded",+ )+ finally:+ event.remove(store.engine, "before_cursor_execute", capture)++ assert deferred == [], "an Activity in another Session blocked this Turn's Run"+ assert normalize_run_status(request_store.get_run(queued.id)["status"]) == "succeeded"+ (statement, parameters), = statements+ with sqlite3.connect(store.db_path) as raw:+ plan = " ".join(row[-1] for row in raw.execute("EXPLAIN QUERY PLAN " + statement, parameters))+ assert "ix_runtime_records_scope_anchor" in plan and "SCAN runtime_records" not in plan, plan+++def test_the_runs_terminal_record_binds_within_an_old_sqlite_variable_limit(tmp_path, monkeypatch) -> None:+ """#2706: a terminal output's Run ids have no size cap, and the record's Activity re-read looks up their Sessions.+ That lookup binds in batches like the rest of the record, so an output with more Run ids than an older SQLite+ build's 999 host parameters still records its Runs instead of failing with ``too many SQL variables``."""++ import sqlite3++ from sqlalchemy import event++ from core.scheduled_tasks import TaskExecutionStore++ if not hasattr(sqlite3.Connection, "setlimit"):+ pytest.skip("sqlite3.Connection.setlimit needs Python 3.11")+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))+ request_store = TaskExecutionStore()+ queued = request_store.enqueue_agent_run(message="run", agent_backend="claude", session_id="session-1")+ assert request_store.claim(queued.id) is not None+ store = request_store.sqlite_backend++ def old_sqlite_limit(dbapi_connection, _record):+ dbapi_connection.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 999)++ event.listen(store.engine, "connect", old_sqlite_limit)+ store.engine.dispose()+ try:+ deferred = store.record_turn_terminal_outputs(+ [queued.id, *(f"run-gone-{index}" for index in range(1000))],+ activity_session_ids=["session-1"],+ output_id="terminal",+ text="done",+ terminal_status="succeeded",+ )+ finally:+ event.remove(store.engine, "connect", old_sqlite_limit)+ store.engine.dispose()++ assert deferred == []+ assert normalize_run_status(request_store.get_run(queued.id)["status"]) == "succeeded"diff --git a/tests/test_message_dispatcher_result_fallback.py b/tests/test_message_dispatcher_result_fallback.pyindex 68588ccf4..d600a88d5 100644--- a/tests/test_message_dispatcher_result_fallback.py+++ b/tests/test_message_dispatcher_result_fallback.py@@ -247,6 +247,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):terminal_error="stream disconnected",output_semantics=output,provenance=dict(output.metadata),+ blocked_run_ids=frozenset(),)store.record_turn_run_outputs.assert_called_once_with(@@ -535,7 +536,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):dispatcher = ConsolidatedMessageDispatcher(controller)dispatcher._collapse_status_bubble = mock.AsyncMock()dispatcher._clear_consolidated_state = mock.AsyncMock()- dispatcher._record_agent_run_terminal_result = mock.Mock()+ dispatcher._record_agent_run_terminal_result = mock.AsyncMock()context = MessageContext(user_id="U1",channel_id="C1",@@ -623,7 +624,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):dispatcher = ConsolidatedMessageDispatcher(controller)dispatcher._collapse_status_bubble = mock.AsyncMock()dispatcher._clear_consolidated_state = mock.AsyncMock()- dispatcher._record_agent_run_terminal_result = mock.Mock()+ dispatcher._record_agent_run_terminal_result = mock.AsyncMock()platform_specific = {"agent_runtime_turn_key": "runtime-1","agent_runtime_turn_token": "runtime-turn-1",@@ -1120,7 +1121,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):controller = _StubController(platform="avibe")dispatcher = ConsolidatedMessageDispatcher(controller)- dispatcher._record_agent_run_terminal_result = mock.Mock()+ dispatcher._record_agent_run_terminal_result = mock.AsyncMock()context = MessageContext(user_id="U1",channel_id="ses-1",@@ -1153,7 +1154,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):async def test_workbench_persistence_failure_does_not_settle_the_run(self):controller = _StubController(platform="avibe")dispatcher = ConsolidatedMessageDispatcher(controller)- dispatcher._record_agent_run_terminal_result = mock.Mock()+ dispatcher._record_agent_run_terminal_result = mock.AsyncMock()context = MessageContext(user_id="U1",channel_id="ses-1",@@ -1186,7 +1187,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):async def test_workbench_duplicate_persistence_receipt_can_settle_the_run(self):controller = _StubController(platform="avibe")dispatcher = ConsolidatedMessageDispatcher(controller)- dispatcher._record_agent_run_terminal_result = mock.Mock()+ dispatcher._record_agent_run_terminal_result = mock.AsyncMock()context = MessageContext(user_id="U1",channel_id="ses-1",@@ -1245,7 +1246,7 @@ class MessageDispatcherResultFallbackTests(unittest.IsolatedAsyncioTestCase):async def test_suppressed_history_id_is_not_recorded_as_run_delivery(self):controller = _StubController(platform="slack")dispatcher = ConsolidatedMessageDispatcher(controller)- dispatcher._record_suppressed_agent_run_terminal_result = mock.Mock()+ dispatcher._record_suppressed_agent_run_terminal_result = mock.AsyncMock()context = MessageContext(user_id="U1",channel_id="C1",