Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(harness): admit scheduled turns and drain callbacks off the controller loop (#2650)

Call Whitef8bb72d773b1
diff --git a/core/internal_server.py b/core/internal_server.py
index 37015cb3c..064aeb91e 100644
--- a/core/internal_server.py
+++ b/core/internal_server.py
@@ -34,7 +34,7 @@ import socket
import time
from pathlib import Path
from types import SimpleNamespace
-from typing import Any, Optional, TYPE_CHECKING
+from typing import Any, Callable, Optional, TYPE_CHECKING
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse
@@ -225,6 +225,40 @@ def create_app(
session_id,
)
+ # Falling through admission's transaction, as opposed to answering from inside it.
+ _ADMITTED = object()
+
+ async def _run_store_step(operation: Callable[[], Any]) -> Any:
+ from core.runtime_work import RuntimeWorkSupervisor
+
+ supervisor = getattr(controller, "runtime_work_supervisor", None)
+ if isinstance(supervisor, RuntimeWorkSupervisor):
+ return await supervisor.run_sync(operation)
+ return await asyncio.to_thread(operation)
+
+ async def _off_loop(operation: Callable[[], Any]) -> tuple[Any, bool]:
+ """Run one scheduled-turn store step on the runtime-work executor, the door the
+ scheduler's own store work uses, so its write-lock wait and commit never hold
+ the loop (#2630).
+
+ Cancellation cannot split the step, as it could not while the step ran
+ synchronously on the loop: one that arrives meanwhile waits for the step to
+ land. Returns ``(result, cancelled)``, and the caller answers the cancellation
+ from what the step committed. Once a Run's ownership has moved to its Delivery,
+ the answer is recovery, never an escaping cancellation that would let the
+ executor settle the Run and drop the turn.
+ """
+
+ step = asyncio.ensure_future(_run_store_step(operation))
+ cancelled = False
+ while True:
+ try:
+ return await asyncio.shield(step), cancelled
+ except asyncio.CancelledError:
+ if step.cancelled():
+ raise
+ cancelled = True
+
async def _submit_scheduled_turn(
session_id: str,
context: MessageContext,
@@ -279,212 +313,228 @@ def create_app(
execution_id = str(
(context.platform_specific or {}).get("task_execution_id") or ""
).strip()
- with run_update_event_transaction(get_cached_sqlite_engine()) as conn:
- from sqlalchemy import select
- from storage.agent_session_rows import reserve_write_lock
- from storage.models import agent_runs, agent_sessions
-
- reserve_write_lock(conn)
- scope_id = _scope_id_for_session(conn, session_id)
- dedupe_key = message_deliveries.native_dedupe_key(
- submission_platform,
- native_message_id,
- scope_id=scope_id,
- )
- existing = message_deliveries.get_delivery_by_native_identity(
- conn,
- platform=submission_platform,
- native_message_id=native_message_id,
- scope_id=scope_id,
- session_id=session_id,
- normalize_legacy=True,
- )
- legacy_accepted = bool(
- native_message_id
- and messages_service.native_message_exists(
+ # Admission's write transaction: a fixed set of keyed statements, run on the
+ # runtime-work executor rather than the loop, where its write-lock wait and
+ # commit would stall every session's turns (#2630). An early return inside
+ # it is the submission's answer; falling through means admitted.
+ def _admit_delivery() -> Any:
+ nonlocal delivery_id, delivery_owner_transferred, delivery_request
+ nonlocal effective_delivery_intent, execution_id, scope_id, target_was_busy
+ with run_update_event_transaction(get_cached_sqlite_engine()) as conn:
+ from sqlalchemy import select
+ from storage.agent_session_rows import reserve_write_lock
+ from storage.models import agent_runs, agent_sessions
+
+ reserve_write_lock(conn)
+ scope_id = _scope_id_for_session(conn, session_id)
+ dedupe_key = message_deliveries.native_dedupe_key(
+ submission_platform,
+ native_message_id,
+ scope_id=scope_id,
+ )
+ existing = message_deliveries.get_delivery_by_native_identity(
conn,
platform=submission_platform,
- scope_id=scope_id,
native_message_id=native_message_id,
+ scope_id=scope_id,
+ session_id=session_id,
+ normalize_legacy=True,
)
- )
- target_was_busy = message_deliveries.active_turn(conn, session_id) is not None
- if existing is not None and existing["session_id"] != session_id:
- return "duplicate"
- if existing is not None and existing["state"] != "reserved":
- if execution_id:
- run_owner = conn.execute(
- select(
- agent_runs.c.status,
- agent_runs.c.session_id,
- agent_runs.c.delivery_id,
- )
- .where(agent_runs.c.id == execution_id)
- .limit(1)
- ).mappings().first()
- if (
- run_owner is not None
- and normalize_run_status(run_owner["status"])
- in {"queued", "running"}
- and run_owner["session_id"] == session_id
- and run_owner["delivery_id"] == existing["id"]
- ):
- existing_state = str(existing["state"])
- if existing_state == "retired":
- cancel_queued_agent_run_delivery_in_connection(
- conn,
- execution_id,
- session_id=session_id,
- delivery_id=str(existing["id"]),
- )
- return TurnSubmissionResult(
- route="enqueued",
- queue_persisted=False,
- target_was_busy=target_was_busy,
- delivery_status="canceled",
+ legacy_accepted = bool(
+ native_message_id
+ and messages_service.native_message_exists(
+ conn,
+ platform=submission_platform,
+ scope_id=scope_id,
+ native_message_id=native_message_id,
+ )
+ )
+ target_was_busy = message_deliveries.active_turn(conn, session_id) is not None
+ if existing is not None and existing["session_id"] != session_id:
+ return "duplicate"
+ if existing is not None and existing["state"] != "reserved":
+ if execution_id:
+ run_owner = conn.execute(
+ select(
+ agent_runs.c.status,
+ agent_runs.c.session_id,
+ agent_runs.c.delivery_id,
)
+ .where(agent_runs.c.id == execution_id)
+ .limit(1)
+ ).mappings().first()
if (
- legacy_send_now
- and existing_state == "queued"
- and existing.get("priority") == "p3"
- and message_deliveries.delivery_has_history_event(
- existing,
- kind="admission",
- )
- and not message_deliveries.delivery_has_history_event(
- existing,
- kind="steer",
- )
+ run_owner is not None
+ and normalize_run_status(run_owner["status"])
+ in {"queued", "running"}
+ and run_owner["session_id"] == session_id
+ and run_owner["delivery_id"] == existing["id"]
):
- existing = message_deliveries.cas_delivery(
- conn,
- str(existing["id"]),
- expected_version=int(existing["version"]),
- expected_states=("queued",),
- values={"priority": "p1", "state": "reserved"},
- history_event={
- "kind": "legacy_send_now_re_admission",
- "from_priority": "p3",
- },
- )
- if existing is None:
- raise RuntimeError(
- "legacy send-now Delivery re-admission lost"
+ existing_state = str(existing["state"])
+ if existing_state == "retired":
+ cancel_queued_agent_run_delivery_in_connection(
+ conn,
+ execution_id,
+ session_id=session_id,
+ delivery_id=str(existing["id"]),
)
- delivery_owner_transferred = True
- else:
- return TurnSubmissionResult(
- route=(
- "enqueued"
- if existing_state
- in {
- "queued",
- "interrupt_waiting",
- "waiting_terminal",
- "reconciling_steer",
- }
- else "ran"
- ),
- queue_persisted=True,
- target_was_busy=target_was_busy,
- delivery_status=existing_state,
- delivery_owner_transferred=True,
- )
- if not delivery_owner_transferred:
+ return TurnSubmissionResult(
+ route="enqueued",
+ queue_persisted=False,
+ target_was_busy=target_was_busy,
+ delivery_status="canceled",
+ )
+ if (
+ legacy_send_now
+ and existing_state == "queued"
+ and existing.get("priority") == "p3"
+ and message_deliveries.delivery_has_history_event(
+ existing,
+ kind="admission",
+ )
+ and not message_deliveries.delivery_has_history_event(
+ existing,
+ kind="steer",
+ )
+ ):
+ existing = message_deliveries.cas_delivery(
+ conn,
+ str(existing["id"]),
+ expected_version=int(existing["version"]),
+ expected_states=("queued",),
+ values={"priority": "p1", "state": "reserved"},
+ history_event={
+ "kind": "legacy_send_now_re_admission",
+ "from_priority": "p3",
+ },
+ )
+ if existing is None:
+ raise RuntimeError(
+ "legacy send-now Delivery re-admission lost"
+ )
+ delivery_owner_transferred = True
+ else:
+ return TurnSubmissionResult(
+ route=(
+ "enqueued"
+ if existing_state
+ in {
+ "queued",
+ "interrupt_waiting",
+ "waiting_terminal",
+ "reconciling_steer",
+ }
+ else "ran"
+ ),
+ queue_persisted=True,
+ target_was_busy=target_was_busy,
+ delivery_status=existing_state,
+ delivery_owner_transferred=True,
+ )
+ if not delivery_owner_transferred:
+ return "duplicate"
+ if legacy_accepted:
return "duplicate"
- if legacy_accepted:
- return "duplicate"
- status = conn.execute(
- select(agent_sessions.c.status).where(agent_sessions.c.id == session_id)
- ).scalar_one_or_none()
- if status != "active":
- raise ValueError("Session is archived")
- if existing is not None:
- delivery_id = str(existing["id"])
- delivery_request = manager._request_from_delivery(existing)
- scope_id = delivery_request.scope_id
- from core.message_priority import delivery_intent_for_priority
-
- persisted_intent = delivery_intent_for_priority(
- delivery_request.priority
- )
- effective_delivery_intent = persisted_intent
- provenance = (delivery_request.metadata or {}).get(
- SCHEDULED_PROVENANCE_KEY
- )
- persisted_spec = (
- provenance.get("platform_specific")
- if isinstance(provenance, dict)
- else None
- )
- if isinstance(persisted_spec, dict):
- execution_id = str(
- persisted_spec.get("task_execution_id") or execution_id
- ).strip()
- else:
- from core.message_priority import priority_for_delivery_intent
+ status = conn.execute(
+ select(agent_sessions.c.status).where(agent_sessions.c.id == session_id)
+ ).scalar_one_or_none()
+ if status != "active":
+ raise ValueError("Session is archived")
+ if existing is not None:
+ delivery_id = str(existing["id"])
+ delivery_request = manager._request_from_delivery(existing)
+ scope_id = delivery_request.scope_id
+ from core.message_priority import delivery_intent_for_priority
+
+ persisted_intent = delivery_intent_for_priority(
+ delivery_request.priority
+ )
+ effective_delivery_intent = persisted_intent
+ provenance = (delivery_request.metadata or {}).get(
+ SCHEDULED_PROVENANCE_KEY
+ )
+ persisted_spec = (
+ provenance.get("platform_specific")
+ if isinstance(provenance, dict)
+ else None
+ )
+ if isinstance(persisted_spec, dict):
+ execution_id = str(
+ persisted_spec.get("task_execution_id") or execution_id
+ ).strip()
+ else:
+ from core.message_priority import priority_for_delivery_intent
- delivery_id = message_deliveries.new_delivery_id()
- admitted_state = "reserved"
- priority = priority_for_delivery_intent(delivery_intent)
- message_deliveries.insert_delivery(
- conn,
- delivery_id=delivery_id,
- session_id=session_id,
- priority=priority,
- state=admitted_state,
- snapshot=message_deliveries.message_snapshot(
- scope_id=scope_id,
+ delivery_id = message_deliveries.new_delivery_id()
+ admitted_state = "reserved"
+ priority = priority_for_delivery_intent(delivery_intent)
+ message_deliveries.insert_delivery(
+ conn,
+ delivery_id=delivery_id,
session_id=session_id,
- platform=submission_platform,
- author="harness",
- author_id=author_id,
- author_name=author_name,
- source="harness",
- message_type="harness",
- text=text,
- metadata={
- **dict(submission_spec.get("message_metadata") or {}),
- SCHEDULED_PROVENANCE_KEY: capture_scheduled_provenance(
- context
- )
+ priority=priority,
+ state=admitted_state,
+ snapshot=message_deliveries.message_snapshot(
+ scope_id=scope_id,
+ session_id=session_id,
+ platform=submission_platform,
+ author="harness",
+ author_id=author_id,
+ author_name=author_name,
+ source="harness",
+ message_type="harness",
+ text=text,
+ metadata={
+ **dict(submission_spec.get("message_metadata") or {}),
+ SCHEDULED_PROVENANCE_KEY: capture_scheduled_provenance(
+ context
+ )
+ },
+ native_message_id=native_message_id or None,
+ ),
+ dispatch_text=text,
+ dedupe_key=dedupe_key,
+ history_event={
+ "kind": "admission",
+ "priority": priority,
+ "state": admitted_state,
},
- native_message_id=native_message_id or None,
- ),
- dispatch_text=text,
- dedupe_key=dedupe_key,
- history_event={
- "kind": "admission",
- "priority": priority,
- "state": admitted_state,
- },
- )
- if execution_id and not delivery_owner_transferred:
- if not attach_agent_run_delivery_in_connection(
- conn,
- execution_id,
- session_id=session_id,
- delivery_id=delivery_id,
- delivery_outcome={
- "intent": effective_delivery_intent,
- "status": "admitted",
- "target_was_busy": target_was_busy,
- },
- ):
- if agent_run_cancellation_won_in_connection(conn, execution_id):
- message_deliveries.retire_not_written(
- conn,
- session_id,
- delivery_id,
- reason="agent_run_canceled",
- )
- return TurnSubmissionResult(
- route="enqueued",
- queue_persisted=False,
- delivery_status="canceled",
- )
- raise RuntimeError("Agent Run Delivery binding was refused")
- delivery_owner_transferred = True
+ )
+ if execution_id and not delivery_owner_transferred:
+ if not attach_agent_run_delivery_in_connection(
+ conn,
+ execution_id,
+ session_id=session_id,
+ delivery_id=delivery_id,
+ delivery_outcome={
+ "intent": effective_delivery_intent,
+ "status": "admitted",
+ "target_was_busy": target_was_busy,
+ },
+ ):
+ if agent_run_cancellation_won_in_connection(conn, execution_id):
+ message_deliveries.retire_not_written(
+ conn,
+ session_id,
+ delivery_id,
+ reason="agent_run_canceled",
+ )
+ return TurnSubmissionResult(
+ route="enqueued",
+ queue_persisted=False,
+ delivery_status="canceled",
+ )
+ raise RuntimeError("Agent Run Delivery binding was refused")
+ delivery_owner_transferred = True
+ return _ADMITTED
+
+ admission, admission_cancelled = await _off_loop(_admit_delivery)
+ if admission is not _ADMITTED:
+ if admission_cancelled and not getattr(admission, "delivery_owner_transferred", False):
+ raise asyncio.CancelledError
+ return admission
+ if admission_cancelled and not delivery_owner_transferred:
+ raise asyncio.CancelledError
if delivery_request is None:
from core.message_priority import priority_for_delivery_intent
@@ -522,23 +572,26 @@ def create_app(
}
)
- def _defer_transferred_delivery() -> TurnSubmissionResult:
+ def _record_deferred_outcome() -> tuple[str, bool]:
+ with run_update_event_transaction(get_cached_sqlite_engine()) as conn:
+ persisted = message_deliveries.get_delivery(conn, delivery_id)
+ status = str((persisted or {}).get("state") or "reserved")
+ recorded = record_agent_run_delivery_outcome_in_connection(
+ conn,
+ execution_id,
+ {
+ "intent": effective_delivery_intent,
+ "status": status,
+ "target_was_busy": target_was_busy,
+ },
+ )
+ return status, recorded
+
+ async def _defer_transferred_delivery() -> TurnSubmissionResult:
delivery_status = "reserved"
try:
- with run_update_event_transaction(get_cached_sqlite_engine()) as conn:
- persisted = message_deliveries.get_delivery(conn, delivery_id)
- delivery_status = str(
- (persisted or {}).get("state") or delivery_status
- )
- recorded = record_agent_run_delivery_outcome_in_connection(
- conn,
- execution_id,
- {
- "intent": effective_delivery_intent,
- "status": delivery_status,
- "target_was_busy": target_was_busy,
- },
- )
+ # Already answering a cancellation: a further one cannot change the answer.
+ (delivery_status, recorded), _cancelled = await _off_loop(_record_deferred_outcome)
if not recorded:
logger.warning(
"Agent Run deferred Delivery outcome lost its exact CAS: "
@@ -562,6 +615,15 @@ def create_app(
delivery_owner_transferred=True,
)
+ if admission_cancelled:
+ logger.warning(
+ "Delivery admission canceled after Agent Run ownership transfer; "
+ "deferring to recovery: run=%s delivery=%s",
+ execution_id,
+ delivery_id,
+ )
+ return await _defer_transferred_delivery()
+
try:
result = await manager.deliver(
delivery_request,
@@ -582,7 +644,7 @@ def create_app(
execution_id,
delivery_id,
)
- return _defer_transferred_delivery()
+ return await _defer_transferred_delivery()
except Exception:
if not delivery_owner_transferred:
raise
@@ -592,7 +654,7 @@ def create_app(
execution_id,
delivery_id,
)
- return _defer_transferred_delivery()
+ return await _defer_transferred_delivery()
delivery_state = str(result.state)
route = "enqueued" if delivery_state in {
"queued",
@@ -611,9 +673,10 @@ def create_app(
)
_publish_scheduled_queue_growth(session_id, delivery_state)
if delivery_owner_transferred and execution_id:
- try:
+
+ def _record_outcome() -> bool:
with run_update_event_transaction(get_cached_sqlite_engine()) as conn:
- recorded = record_agent_run_delivery_outcome_in_connection(
+ return record_agent_run_delivery_outcome_in_connection(
conn,
execution_id,
{
@@ -622,6 +685,11 @@ def create_app(
"target_was_busy": submission.target_was_busy,
},
)
+
+ try:
+ # Ownership has moved to the Delivery: the submission's answer must reach
+ # the executor, so a cancellation here does not escape it.
+ recorded, _cancelled = await _off_loop(_record_outcome)
if not recorded:
logger.warning(
"Agent Run Delivery outcome lost its exact CAS after ownership "
diff --git a/core/scheduled_tasks.py b/core/scheduled_tasks.py
index 03eb4b74b..ac66fa0d4 100644
--- a/core/scheduled_tasks.py
+++ b/core/scheduled_tasks.py
@@ -51,6 +51,7 @@ from core.runtime_work import (
RuntimeWorkItem,
RuntimeWorkLane,
RuntimeWorkRegistrationToken,
+ RuntimeWorkSupervisor,
)
from core.run_settlement import (
INTERRUPT_REASON_DELIVERY_TARGET_MISSING,
@@ -4957,9 +4958,9 @@ class ScheduledTaskService:
async def _run_runtime_sync(self, operation, /, *args, **kwargs): # noqa: ANN001, ANN202
call = partial(operation, *args, **kwargs)
supervisor = getattr(self.controller, "runtime_work_supervisor", None)
- run_sync = getattr(supervisor, "run_sync", None)
- if callable(run_sync):
- return await run_sync(call)
+ # Gate on the type: a stand-in controller's attribute can be callable without being awaitable.
+ if isinstance(supervisor, RuntimeWorkSupervisor):
+ return await supervisor.run_sync(call)
return await asyncio.to_thread(call)
def _start_legacy_runtime_work_probes(self) -> None:
@@ -6019,28 +6020,19 @@ class ScheduledTaskService:
self._spawn_execution(request, lock_key)
async def _drain_callbacks(self) -> None:
+ """Send every owed callback now, each through the RUN_CALLBACKS lane's own step.
+
+ Its store work (the listing, the callback enqueue under the write lock, the
+ status update) runs on the runtime-work executor like the lane's, never on the
+ loop: a settled run calls this from its execution task (#2630).
+ """
if not self._owns_service_instance():
return
- for run in self.request_store.list_pending_callbacks():
- run_id = str(run.get("id") or "")
- if not run_id:
- continue
- try:
- callback_run = self._enqueue_callback_run(run)
- except Exception as exc:
- logger.error("Agent run callback failed for %s: %s", run_id, exc, exc_info=True)
- self.request_store.update_callback_status(run_id, status="failed", error=str(exc))
- self._wake_runtime_work(RuntimeWorkLane.RUN_CALLBACKS)
- continue
- if callback_run is None:
- self.request_store.update_callback_status(run_id, status="skipped")
- self._wake_runtime_work(RuntimeWorkLane.RUN_CALLBACKS)
- continue
- self.request_store.update_callback_status(run_id, status="sent", callback_run_id=callback_run.id)
- self._wake_runtime_work(
- RuntimeWorkLane.REQUESTS,
- RuntimeWorkLane.RUN_CALLBACKS,
- )
+ pending = await self._run_runtime_sync(self.request_store.list_pending_callbacks)
+ for run in pending:
+ await self._process_run_callback(run)
+ if pending:
+ self._wake_runtime_work(RuntimeWorkLane.RUN_CALLBACKS)
# --- the owed-failure-notice drain ---------------------------------------
#
diff --git a/tests/test_internal_server.py b/tests/test_internal_server.py
index 3126da91d..0413a5ae9 100644
--- a/tests/test_internal_server.py
+++ b/tests/test_internal_server.py
@@ -5565,7 +5565,10 @@ def test_scheduled_send_now_recovers_transferred_reservation(monkeypatch, tmp_pa
resumed_owner = asyncio.run(_go())
assert isinstance(resumed_owner, session_turns.TurnSubmissionResult)
assert resumed_owner.route == "ran"
- assert resumed_owner.delivery_status == "claimed"
+ # The duplicate answers with the Delivery's state when its admission reads it. Admission
+ # now yields the loop while it waits for SQLite (#2630), so the recovered turn may have
+ # moved its Delivery from claimed to accepted by then; either way the turn owns it.
+ assert resumed_owner.delivery_status in {"claimed", "accepted"}
assert resumed_owner.delivery_owner_transferred is True
assert dispatched == ["original send-now prompt"]
@@ -8534,3 +8537,139 @@ def test_show_access_apply_serializes_controller_writes(monkeypatch):
assert [response.status_code for response in responses] == [200, 200]
assert calls == ["ses-first", "ses-second"]
+
+
+def test_a_scheduled_turns_admission_never_holds_the_loop(monkeypatch, tmp_path):
+ """#2630: admitting a scheduled turn writes its Delivery and Run rows in one transaction that waits for SQLite's
+ write lock and commits. The loop runs every session's turns, so it stays free while another writer holds the
+ lock."""
+ import sqlite3
+ import threading
+
+ from core import session_turns
+
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ engine, session = _create_test_session(tmp_path, native_id="proj_admission_off_loop")
+ session_id = session["id"]
+
+ async def _fake_dispatch_turn(ctrl, ctx, text, *, source=SOURCE_HUMAN, on_chunk=None, **_kwargs):
+ return TurnDispatchOutcome(error=None, settled_by=SETTLED_BY_TERMINAL_RESULT)
+
+ monkeypatch.setattr(session_turns, "dispatch_turn_with_outcome", _fake_dispatch_turn)
+ controller = _build_controller_double()
+ internal_server.create_app(controller)
+ ctx = MessageContext(
+ user_id="workbench",
+ channel_id=session_id,
+ platform="avibe",
+ message_id="scheduled:admission-off-loop:1",
+ platform_specific={"agent_session_id": session_id, "agent_session_target": {"agent_backend": "claude"}},
+ )
+
+ held, release = threading.Event(), threading.Event()
+ released_in_time: list[bool] = []
+
+ def _hold_write_lock() -> None:
+ # Another writer, as a second process or a long transaction would be.
+ holder = sqlite3.connect(engine.url.database, timeout=10)
+ try:
+ holder.execute("BEGIN IMMEDIATE")
+ held.set()
+ released_in_time.append(release.wait(2))
+ holder.rollback()
+ finally:
+ holder.close()
+
+ async def _go():
+ holder = threading.Thread(target=_hold_write_lock)
+ holder.start()
+ assert await asyncio.to_thread(held.wait, 5)
+ submitting = asyncio.create_task(controller.session_turn_gate.submit_scheduled(session_id, ctx, "digest please"))
+ await asyncio.sleep(0.1)
+ # The loop lets the other writer go while admission waits for it.
+ release.set()
+ await submitting
+ await asyncio.to_thread(holder.join, 5)
+
+ asyncio.run(_go())
+ assert released_in_time == [True]
+
+
+def test_a_cancellation_during_admission_keeps_the_runs_delivery_ownership(monkeypatch, tmp_path):
+ """#2630: admission runs on a thread now, so a cancellation can arrive while it waits for SQLite's write lock.
+ Once its transaction moves the Run's ownership to the Delivery, the submission must hand the turn to recovery,
+ as a cancellation during delivery does, instead of letting the Run be terminalized and the turn dropped."""
+ import sqlite3
+ import threading
+
+ from core import session_turns
+ from core.scheduled_tasks import TaskExecutionStore
+
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ engine, session = _create_test_session(tmp_path, native_id="proj_admission_cancel")
+ session_id = session["id"]
+ request_store = TaskExecutionStore()
+ run = request_store.enqueue_definition_run(
+ definition_id="scheduled-admission-cancel",
+ run_type="scheduled",
+ source_kind="scheduler",
+ session_key="",
+ session_id=session_id,
+ post_to=None,
+ deliver_key=None,
+ prompt="digest please",
+ agent_name="worker",
+ session_policy="existing",
+ )
+ assert request_store.claim(run.id) is not None
+
+ async def _fake_dispatch_turn(ctrl, ctx, text, *, source=SOURCE_HUMAN, on_chunk=None, **_kwargs):
+ return TurnDispatchOutcome(error=None, settled_by=SETTLED_BY_TERMINAL_RESULT)
+
+ monkeypatch.setattr(session_turns, "dispatch_turn_with_outcome", _fake_dispatch_turn)
+ controller = _build_controller_double()
+ internal_server.create_app(controller)
+ ctx = MessageContext(
+ user_id="workbench",
+ channel_id=session_id,
+ platform="avibe",
+ message_id=f"scheduled:scheduled-admission-cancel:{run.id}",
+ platform_specific={
+ "agent_session_id": session_id,
+ "agent_session_target": {"agent_backend": "claude"},
+ "task_execution_id": run.id,
+ "task_trigger_kind": "scheduled",
+ "task_definition_id": "scheduled-admission-cancel",
+ },
+ )
+ held, release = threading.Event(), threading.Event()
+
+ def _hold_write_lock() -> None:
+ holder = sqlite3.connect(engine.url.database, timeout=10)
+ try:
+ holder.execute("BEGIN IMMEDIATE")
+ held.set()
+ release.wait(5)
+ holder.rollback()
+ finally:
+ holder.close()
+
+ async def _go():
+ holder = threading.Thread(target=_hold_write_lock)
+ holder.start()
+ assert await asyncio.to_thread(held.wait, 5)
+ submitting = asyncio.create_task(
+ controller.session_turn_gate.submit_scheduled(session_id, ctx, "digest please", delivery_intent="queue")
+ )
+ await asyncio.sleep(0.1)
+ submitting.cancel()
+ release.set()
+ try:
+ return await submitting
+ finally:
+ await asyncio.to_thread(holder.join, 5)
+
+ result = asyncio.run(_go())
+ assert result.delivery_owner_transferred is True
+ stored = request_store.get_run(run.id)
+ assert stored["delivery_id"] and stored["status"] in {"running", "queued"}
diff --git a/tests/test_scheduled_tasks.py b/tests/test_scheduled_tasks.py
index 9e5988dcd..2a6de1662 100644
--- a/tests/test_scheduled_tasks.py
+++ b/tests/test_scheduled_tasks.py
@@ -6742,6 +6742,61 @@ def _callback_service(
)
+def test_a_settled_runs_callback_drain_never_runs_store_work_on_the_loop(tmp_path: Path, monkeypatch) -> None:
+ """#2630: a settled run drains its callbacks from the execution task on the loop. The listing, the callback enqueue
+ (under SQLite's write lock) and the status update run on a thread, as the RUN_CALLBACKS lane's do."""
+ import threading
+
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ caller_session_id = _make_avibe_session(monkeypatch, tmp_path)
+ request_store = TaskExecutionStore()
+ request = request_store.enqueue_agent_run(
+ session_id="target-session",
+ message="delegated work",
+ agent_name="codex",
+ callback_session_id=caller_session_id,
+ )
+ request_store.complete(request, ok=True, session_id="target-session")
+ store = SQLiteBackgroundTaskStore()
+ try:
+ store.record_run_message(
+ request.id, text="delegated result", message_id=f"suppressed:{request.id}", terminal_status="succeeded",
+ )
+ finally:
+ store.close()
+
+ async def _handle_scheduled_message(context, message, parsed_session_key=None):
+ return None
+
+ controller = _avibe_controller_double(
+ gate=SimpleNamespace(submit_scheduled=lambda *_args, **_kwargs: None, in_flight={}),
+ handle_scheduled_message=_handle_scheduled_message,
+ )
+ service = ScheduledTaskService(
+ controller=controller,
+ store=ScheduledTaskStore(tmp_path / "scheduled_tasks.json"),
+ request_store=request_store,
+ )
+ threads: dict[str, int] = {}
+ for name in ("list_pending_callbacks", "update_callback_status"):
+ original = getattr(request_store, name)
+
+ def recording(*args, _original=original, _name=name, **kwargs):
+ threads[_name] = threading.get_ident()
+ return _original(*args, **kwargs)
+
+ monkeypatch.setattr(request_store, name, recording)
+
+ async def _drain() -> int:
+ await service._drain_callbacks()
+ return threading.get_ident()
+
+ loop_thread = asyncio.run(_drain())
+ assert set(threads) == {"list_pending_callbacks", "update_callback_status"}
+ assert loop_thread not in threads.values()
+ assert request_store.get_run(request.id)["callback_status"] == "sent"
+
+
def test_agent_run_callback_enqueues_only_result_to_caller_session(tmp_path: Path, monkeypatch) -> None:
monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
caller_session_id = _make_avibe_session(monkeypatch, tmp_path)