Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(harness): run a claimed execution's store work off the controller loop (#2654)

Call White33f6f4ca4234
diff --git a/core/command_runner.py b/core/command_runner.py
index 045ed43fe..6f966d5fa 100644
--- a/core/command_runner.py
+++ b/core/command_runner.py
@@ -15,6 +15,7 @@ differ, under a running runner.
from __future__ import annotations
import asyncio
+import inspect
import logging
import os
import shlex
@@ -22,7 +23,7 @@ import signal
import sys
from collections import deque
from dataclasses import dataclass
-from typing import Callable, Collection, Mapping, Optional
+from typing import Awaitable, Callable, Collection, Mapping, Optional
from config import paths as config_paths
from core import watch_worker
@@ -289,7 +290,9 @@ async def run_supervised_command(
cwd: str,
timeout_seconds: float,
label: str,
- on_spawn: Optional[Callable[[int, Optional[PersistedProcessIdentity]], None]] = None,
+ on_spawn: Optional[
+ Callable[[int, Optional[PersistedProcessIdentity]], Optional[Awaitable[None]]]
+ ] = None,
max_output_bytes: Optional[int] = None,
env: Optional[Mapping[str, str]] = None,
extra_env: Optional[Mapping[str, str]] = None,
@@ -338,7 +341,11 @@ async def run_supervised_command(
# other failure here rather than leaking the very orphan it is protecting
# against. Still before the spec is written, so the supervisor is blocked
# on stdin and nothing the user asked for has started yet.
- on_spawn(process.pid, identity)
+ # A callback that persists the worker may do so off the loop and hand back
+ # the awaitable; the spec is not written until it has landed.
+ recorded = on_spawn(process.pid, identity)
+ if inspect.isawaitable(recorded):
+ await recorded
if process.stdin is None:
await terminate_and_communicate(process, logger, label)
raise RuntimeError("watch worker supervisor stdin is unavailable")
diff --git a/core/scheduled_tasks.py b/core/scheduled_tasks.py
index ac66fa0d4..43cd2dcb4 100644
--- a/core/scheduled_tasks.py
+++ b/core/scheduled_tasks.py
@@ -1279,12 +1279,17 @@ def _serialize_task_mirror(
return _locked
+# A cancelled claim's settlement that projected no definition result.
+_NOT_PROJECTED = object()
+
+
class ScheduledTaskStore:
def __init__(self, path: Optional[Path] = None):
self.path = path or (paths.get_state_dir() / "scheduled_tasks.json")
self._sqlite = SQLiteBackgroundTaskStore() if path is None else None
self._signature: Optional[_PathSignature] = None
- self._tasks: Dict[str, ScheduledTask] = {}
+ # The published mirror and its generation, one tuple so a reader takes both at once.
+ self._published: tuple[int, Dict[str, ScheduledTask]] = (0, {})
#: Set when a failed write left this mirror INCOMPLETE, cleared by the reload
#: that repairs it. See ``maybe_reload`` and ``_reload_after_lost_write``.
self._reload_required = False
@@ -1382,24 +1387,68 @@ class ScheduledTaskStore:
self.load()
return True
- def _save(self) -> None:
+ def _save(self, tasks: Optional[Dict[str, ScheduledTask]] = None) -> None:
+ """Write the file backend's mirror: ``tasks`` when a writer is about to publish
+ them, else the published one."""
+
if self._sqlite is not None:
return
self.path.parent.mkdir(parents=True, exist_ok=True)
- payload = {"tasks": [task.to_dict() for task in self.list_tasks()]}
+ ordered = sorted(
+ (tasks if tasks is not None else self._tasks).values(),
+ key=lambda item: (item.created_at, item.id),
+ )
+ payload = {"tasks": [task.to_dict() for task in ordered]}
write_atomic(self.path, json.dumps(payload, indent=2))
self._signature = _path_signature(self.path)
+ # The mirror is PUBLISHED, never edited in place: a writer changes a private copy
+ # (``_working_copy``), writes it, and only then swaps in a new dict
+ # (``_publish``/``_unpublish``); a reload builds a new dict too. A reader therefore
+ # takes no lock -- one dict reference is one consistent mirror -- so the controller
+ # loop never waits behind a writer that holds ``_reload_lock`` across SQLite I/O
+ # on a worker thread, and never sees an edit the database has not accepted.
+
+ @property
+ def _tasks(self) -> Dict[str, ScheduledTask]:
+ return self._published[1]
+
+ @_tasks.setter
+ def _tasks(self, tasks: Dict[str, ScheduledTask]) -> None:
+ # Every publication is a new generation, so a reader can tell which of two
+ # mirrors it captured is newer, whatever order the captures complete in.
+ self._published = (self._published[0] + 1, tasks)
+
def list_tasks(self) -> list[ScheduledTask]:
- with self._reload_lock:
- return sorted(
- self._tasks.values(),
- key=lambda item: (item.created_at, item.id),
- )
+ return self.list_tasks_with_generation()[1]
+
+ def list_tasks_with_generation(self) -> tuple[int, list[ScheduledTask]]:
+ """The published mirror, sorted, with the generation it was published as."""
+
+ generation, tasks = self._published
+ return generation, sorted(tasks.values(), key=lambda item: (item.created_at, item.id))
def get_task(self, task_id: str) -> Optional[ScheduledTask]:
- with self._reload_lock:
- return self._tasks.get(task_id)
+ return self._tasks.get(task_id)
+
+ def _working_copy(self, task_id: str) -> Optional[ScheduledTask]:
+ """A private copy of a published definition, for a writer to change."""
+
+ task = self._tasks.get(task_id)
+ return ScheduledTask.from_dict(task.to_dict()) if task is not None else None
+
+ def _required_working_copy(self, task_id: str) -> ScheduledTask:
+ task = self._working_copy(task_id)
+ if task is None:
+ raise KeyError(task_id)
+ return task
+
+ def _publish(self, task: ScheduledTask) -> None:
+ self._tasks = {**self._tasks, task.id: task}
+
+ def _unpublish(self, task_id: str) -> None:
+ if task_id in self._tasks:
+ self._tasks = {key: value for key, value in self._tasks.items() if key != task_id}
def refresh_task(self, task_id: str) -> Optional[ScheduledTask]:
"""Reload and read one scheduler definition under the same mirror lock."""
@@ -1495,7 +1544,8 @@ class ScheduledTaskStore:
"a file-backed scheduled task store cannot commit a queued run "
"with the task row"
)
- self._save()
+ self._save({**self._tasks, task.id: task})
+ self._publish(task)
_publish_task_definitions_updated()
return True
if queued_run is None:
@@ -1522,6 +1572,7 @@ class ScheduledTaskStore:
self._reload_after_lost_write(task.id)
raise
if landed:
+ self._publish(task)
_publish_task_definitions_updated()
return True
self.load()
@@ -1547,7 +1598,7 @@ class ScheduledTaskStore:
"stale mirror entry for %s",
task_id,
)
- self._tasks.pop(task_id, None)
+ self._unpublish(task_id)
self._signature = None
self._reload_required = True
@@ -1569,7 +1620,6 @@ class ScheduledTaskStore:
"""
task.updated_at = _utc_now_iso()
- self._tasks[task.id] = task
try:
if self._sqlite is not None:
# No ``expect``: this is the create/adopt entry point (``add_task``),
@@ -1583,12 +1633,14 @@ class ScheduledTaskStore:
self.load()
_publish_task_definitions_updated()
return self._tasks[task.id]
+ self._publish(task)
_publish_task_definitions_updated()
return task
- self._save()
+ self._save({**self._tasks, task.id: task})
except Exception:
self._reload_after_lost_write(task.id)
raise
+ self._publish(task)
_publish_task_definitions_updated()
return task
@@ -1663,22 +1715,23 @@ class ScheduledTaskStore:
if task_id not in self._tasks:
return False
- del self._tasks[task_id]
try:
if self._sqlite is not None:
self._sqlite.remove_task(task_id)
+ self._unpublish(task_id)
_publish_task_definitions_updated()
return True
- self._save()
+ self._save({key: value for key, value in self._tasks.items() if key != task_id})
except Exception:
self._reload_after_lost_write(task_id)
raise
+ self._unpublish(task_id)
_publish_task_definitions_updated()
return True
@_serialize_task_mirror
def set_enabled(self, task_id: str, enabled: bool) -> ScheduledTask:
- task = self._tasks[task_id]
+ task = self._required_working_copy(task_id)
if enabled and not task.enabled:
require_task_resumable(
task_id,
@@ -1698,11 +1751,24 @@ class ScheduledTaskStore:
return task
@_serialize_task_mirror
- def rebind_session(self, task_id: str, session_id: str) -> None:
- """Host recovery changes only the binding, retaining authenticated metadata."""
- task = self._tasks[task_id]
+ def rebind_session(self, task_id: str, session_id: str, *, follows_session: bool = False) -> None:
+ """Host recovery changes the binding, retaining authenticated metadata.
+
+ ``follows_session`` is a reset rebind's recorded choice: the definition follows
+ its rebound session's Agent from now on, so its own Agent pin is dropped and the
+ marker that keeps an unrelated edit from re-pinning it is set. Applied to the
+ store's copy in the same write as the binding, so neither lands without the other.
+ """
+
+ task = self._required_working_copy(task_id)
expect = self._read_state(task)
task.session_id = session_id
+ if follows_session:
+ task.metadata = {
+ **(task.metadata if isinstance(task.metadata, dict) else {}),
+ BINDING_FOLLOWS_SESSION_METADATA_KEY: True,
+ }
+ task.agent_name = None
task.updated_at = _utc_now_iso()
if not self._write_task(task, expect):
raise DefinitionWriteConflict(task_id, definition_type="scheduled task")
@@ -1743,7 +1809,7 @@ class ScheduledTaskStore:
ensure_harness_definition_write(user_context)
ensure_agent_name_access(agent_name, user_context=user_context)
- task = self._tasks[task_id]
+ task = self._required_working_copy(task_id)
# Captured before the first mutation: this is the state the CALLER read
# (``vibe task update`` resolved Agents and Sessions from this very object),
# and it is what the write below re-asserts.
@@ -1809,7 +1875,7 @@ class ScheduledTaskStore:
) -> bool:
"""Retire the exact one-shot schedule APScheduler reported missed."""
- task = self._tasks.get(task_id)
+ task = self._working_copy(task_id)
if (
task is None
or not task.enabled
@@ -1840,7 +1906,8 @@ class ScheduledTaskStore:
task.metadata.pop(COMMAND_TIMED_OUT_METADATA_KEY, None)
task.metadata.pop(TASK_LAST_RESULT_STATUS_METADATA_KEY, None)
task.updated_at = task.retired_at
- self._save()
+ self._save({**self._tasks, task.id: task})
+ self._publish(task)
return True
@_serialize_task_mirror
@@ -1861,7 +1928,7 @@ class ScheduledTaskStore:
"""
self.maybe_reload()
- task = self._tasks.get(task_id)
+ task = self._working_copy(task_id)
if task is None:
return False
expect = self._read_state(task)
@@ -1883,6 +1950,7 @@ class ScheduledTaskStore:
self._reload_after_lost_write(task.id)
raise
if landed:
+ self._publish(task)
return True
self.load()
return False
@@ -1918,7 +1986,7 @@ class ScheduledTaskStore:
"""
self.maybe_reload()
- task = self._tasks.get(task_id)
+ task = self._working_copy(task_id)
if task is None:
return False
expect = self._read_state(task)
@@ -1979,7 +2047,7 @@ class ScheduledTaskStore:
"""
self.maybe_reload()
- task = self._tasks.get(task_id)
+ task = self._working_copy(task_id)
if task is None:
return False
non_owner_expectation = expected_non_owner_retired_one_shot
@@ -2097,11 +2165,12 @@ class ScheduledTaskStore:
expected_terminal_run_id=expected_terminal_run_id,
)
+ @_serialize_task_mirror
def suspend_task(self, task_id: str, *, error: str) -> bool:
"""Atomically disable a definition before an unsafe execution boundary."""
self.maybe_reload()
- task = self._tasks.get(task_id)
+ task = self._working_copy(task_id)
if task is None:
return False
expect = self._read_state(task)
@@ -3962,6 +4031,22 @@ class _ScheduledRuntimeWorkHandler(RuntimeWorkHandler):
return True
+def _rearm_cancellation() -> None:
+ """Deliver a cancellation the current task caught, but did not answer, at its next await.
+
+ ``uncancel`` first (Python 3.11+, absent on 3.10, which counts no requests), so the
+ request is counted once, as an ``asyncio.timeout`` around the caller expects.
+ """
+
+ current = asyncio.current_task()
+ if current is None:
+ return
+ uncancel = getattr(current, "uncancel", None)
+ if uncancel is not None:
+ uncancel()
+ current.cancel()
+
+
class ScheduledTaskService:
"""Controller-owned runtime that executes persisted scheduled tasks."""
@@ -4010,6 +4095,13 @@ class ScheduledTaskService:
# job ids are unique per registration so a queued event from a replaced
# DateTrigger cannot be mistaken for its successor.
self._job_ids: Dict[str, str] = {}
+ # Settlements of claims cancelled before they ran, started by their done-callback,
+ # and the claims they settle: a claim stays live, so recovery cannot reclaim it,
+ # until its settlement has landed.
+ self._cancel_settlements: set["asyncio.Task[None]"] = set()
+ self._settling_claim_ids: set[str] = set()
+ # The store and mirror generation ``reconcile_jobs`` last applied.
+ self._job_reconcile_applied: tuple[Any, int] = (None, 0)
self._one_shot_job_identities: Dict[str, tuple[str, str, str, str]] = {}
self._running = False
self._watch_store_restart_count = 0
@@ -4956,12 +5048,62 @@ class ScheduledTaskService:
notify_after(token, delay)
async def _run_runtime_sync(self, operation, /, *args, **kwargs): # noqa: ANN001, ANN202
+ """The door for the service's blocking store work: the runtime-work executor.
+
+ A store step cannot be split by cancellation, as it could not while it ran on
+ the loop: a cancellation that arrives meanwhile waits for the step to land and
+ is raised after it, so the caller sees exactly what committed. A step that
+ raised raises its own error, with the cancellation re-armed behind it.
+ """
+
+ step, cancelled = await self._land_runtime_step(operation, *args, **kwargs)
+ if cancelled and not step.cancelled():
+ if step.exception() is not None:
+ _rearm_cancellation()
+ return step.result()
+ raise asyncio.CancelledError
+ return step.result()
+
+ async def _store_step(self, operation, /, *args, **kwargs): # noqa: ANN001, ANN202
+ """One store step of a claimed execution: on the runtime-work door, with the
+ cancellation point it had on the loop.
+
+ These steps ran synchronously on the loop before #2649, so a cancellation could
+ land only at the execution's next real await -- the dispatch, the command, a
+ delivery -- with every step before it complete and its outcome in hand. It still
+ does: a cancellation that arrives while a step runs waits for the step to land,
+ the step's result is returned (or its error raised), and the cancellation is
+ re-armed for the next await that is not itself a store step. So a result a step
+ committed (a reservation, a binding recovery, a settlement) is never dropped
+ between steps, and each cancellation is answered where the execution always
+ answered it.
+ """
+
+ step, cancelled = await self._land_runtime_step(operation, *args, **kwargs)
+ if cancelled and not step.cancelled():
+ _rearm_cancellation()
+ return step.result()
+
+ async def _land_runtime_step(self, operation, /, *args, **kwargs): # noqa: ANN001, ANN202
+ """Run one store step on the runtime-work door and wait for it to land, whatever
+ cancels the caller meanwhile. Returns ``(step, cancelled)``: the finished step,
+ whose ``result()`` gives its outcome, and whether a cancellation was caught."""
+
call = partial(operation, *args, **kwargs)
supervisor = getattr(self.controller, "runtime_work_supervisor", None)
# Gate on the type: a stand-in controller's attribute can be callable without being awaitable.
if isinstance(supervisor, RuntimeWorkSupervisor):
- return await supervisor.run_sync(call)
- return await asyncio.to_thread(call)
+ step = asyncio.ensure_future(supervisor.run_sync(call))
+ else:
+ step = asyncio.ensure_future(asyncio.to_thread(call))
+ cancelled = False
+ while not step.done():
+ try:
+ # ``wait`` never cancels the step, and raises only the caller's cancellation.
+ await asyncio.wait((step,))
+ except asyncio.CancelledError:
+ cancelled = True
+ return step, cancelled
def _start_legacy_runtime_work_probes(self) -> None:
if self.store.sqlite_backend is None:
@@ -5260,6 +5402,10 @@ class ScheduledTaskService:
await task
except (asyncio.CancelledError, Exception):
pass
+ # A claim cancelled before it ran is settled by an owned task its done-callback
+ # started; it lands before shutdown returns, like an execution's own settlement.
+ if self._cancel_settlements:
+ await asyncio.gather(*tuple(self._cancel_settlements), return_exceptions=True)
teardown = self._ensure_service_teardown_task()
if teardown is not None:
await teardown
@@ -5271,6 +5417,7 @@ class ScheduledTaskService:
self._request_recovery_unregistration_task = None
self._inflight_executions.clear()
self._request_capacity_reservations.clear()
+ self._settling_claim_ids.clear()
self._live_claimed_run_ids = frozenset()
self._inflight_requests.clear()
self._inflight_cancellation_causes.clear()
@@ -5566,9 +5713,29 @@ class ScheduledTaskService:
def reconcile_jobs(self) -> None:
if not self._owns_service_instance():
return
+ self._apply_job_reconcile(self.store, *self.store.list_tasks_with_generation())
+
+ async def _reconcile_jobs_off_loop(self) -> None:
+ """``reconcile_jobs`` for a claimed execution: the mirror read (which waits on the
+ store's reload lock while a scan holds it) runs on the runtime-work executor, the
+ scheduler jobs are changed on the loop, which owns them."""
+
+ if not self._owns_service_instance():
+ return
+ store = self.store
+ self._apply_job_reconcile(store, *await self._store_step(store.list_tasks_with_generation))
+
+ def _apply_job_reconcile(self, store: Any, generation: int, tasks: list[ScheduledTask]) -> None:
+ # Captures may complete out of order once one waits on a thread: only a mirror at
+ # least as new as the last one applied may change the jobs, so an older mirror
+ # never undoes a newer one. Ordered by the generation each capture actually read.
+ applied_store, applied_generation = self._job_reconcile_applied
+ if applied_store is store and generation < applied_generation:
+ return
+ self._job_reconcile_applied = (store, generation)
desired_ids = set()
desired_job_ids = set()
- for task in self.store.list_tasks():
+ for task in tasks:
if not task.enabled or task.retired_at is not None:
continue
desired_ids.add(task.id)
@@ -7951,18 +8118,23 @@ class ScheduledTaskService:
self._session_lock_owners[lock_key] = request.id
task = asyncio.create_task(self._execute_claimed_request(request))
self._inflight_executions[request.id] = task
- self._live_claimed_run_ids = frozenset(self._inflight_executions)
+ self._refresh_live_claims()
self._inflight_requests[request.id] = request
task.add_done_callback(
lambda finished, rid=request.id, key=lock_key: self._on_execution_done(rid, key, finished)
)
+ def _refresh_live_claims(self) -> None:
+ self._live_claimed_run_ids = frozenset(self._inflight_executions) | frozenset(self._settling_claim_ids)
+
def _on_execution_done(
self, request_id: str, lock_key: Optional[str], task: "asyncio.Task[Any]"
) -> None:
self._inflight_executions.pop(request_id, None)
- self._live_claimed_run_ids = frozenset(self._inflight_executions)
request = self._inflight_requests.pop(request_id, None)
+ if task.cancelled() and request is not None:
+ self._settling_claim_ids.add(request_id)
+ self._refresh_live_claims()
interruption = self._inflight_cancellation_causes.pop(
request_id,
SETTLED_BY_INTERRUPTED,
@@ -7980,45 +8152,73 @@ class ScheduledTaskService:
RuntimeWorkLane.FAILURE_NOTICES,
)
# ``_execute_claimed_request`` already terminalizes cancellations and
- # records ordinary failures. A task cancelled before its coroutine enters
- # has no execution to settle; put that bare claim back in the queue.
+ # records ordinary failures. A task cancelled before its coroutine enters, or
+ # while its start was landing, has no execution of its own to settle: an owned
+ # task settles it, on the door rather than in this synchronous callback on the
+ # loop, and ``stop`` waits for it.
if task.cancelled():
if request is not None:
- try:
- current = self.request_store.get_run(request.id)
- if not current or current.get("status") in TERMINAL_RUN_STATUSES:
- return
- if (
- current.get("pid") is None
- ):
- self.request_store.requeue(request.id)
- return
- written_status = self.request_store.complete(
- request,
- ok=False,
- error=self._t(SETTLEMENT_I18N_KEYS[interruption]),
- interrupt_reason=interruption,
- )
- if written_status is not None and request.task_id and request.request_type in {
- "task_run",
- "scheduled",
- }:
- self._project_terminal_definition_result(
- self.request_store.get_run(request.id),
- execution_id=request.id,
- expected_status=written_status,
- )
- self.reconcile_jobs()
- except Exception:
- logger.exception(
- "Failed to terminalize canceled claimed request %s",
- request_id,
- )
+ settlement = asyncio.get_running_loop().create_task(
+ self._settle_cancelled_claim(request, interruption),
+ name=f"scheduled-claim-cancel-settlement:{request_id}",
+ )
+ self._cancel_settlements.add(settlement)
+ settlement.add_done_callback(self._cancel_settlements.discard)
return
exc = task.exception()
if exc is not None:
logger.error("Claimed request %s crashed: %r", request_id, exc, exc_info=exc)
+ async def _settle_cancelled_claim(self, request: TaskExecutionRequest, interruption: str) -> None:
+ """Settle a claim whose execution was cancelled before it ran: a claim that never
+ started goes back to the queue, a started one is terminalized as interrupted."""
+
+ try:
+ retired = await self._run_runtime_sync(
+ self._settle_cancelled_claim_in_store, request, interruption
+ )
+ except Exception:
+ logger.exception("Failed to terminalize canceled claimed request %s", request.id)
+ return
+ finally:
+ # Settled (or given up): the claim stops being live, and the request lane
+ # may now pick up the run this settlement requeued.
+ self._settling_claim_ids.discard(request.id)
+ self._refresh_live_claims()
+ self._wake_runtime_work(RuntimeWorkLane.REQUESTS)
+ if retired is _NOT_PROJECTED:
+ return
+ if retired is not None:
+ self._retire_one_shot_job(retired)
+ await self._reconcile_jobs_off_loop()
+
+ def _settle_cancelled_claim_in_store(
+ self, request: TaskExecutionRequest, interruption: str
+ ) -> Any:
+ current = self.request_store.get_run(request.id)
+ if not current or current.get("status") in TERMINAL_RUN_STATUSES:
+ return _NOT_PROJECTED
+ if current.get("pid") is None:
+ self.request_store.requeue(request.id)
+ return _NOT_PROJECTED
+ written_status = self.request_store.complete(
+ request,
+ ok=False,
+ error=self._t(SETTLEMENT_I18N_KEYS[interruption]),
+ interrupt_reason=interruption,
+ )
+ if written_status is None or not request.task_id or request.request_type not in {
+ "task_run",
+ "scheduled",
+ }:
+ return _NOT_PROJECTED
+ return self._project_terminal_definition_result(
+ self.request_store.get_run(request.id),
+ execution_id=request.id,
+ expected_status=written_status,
+ retire_job=False,
+ )
+
def _claimed_request_backend_resolution(
self,
request: TaskExecutionRequest,
@@ -8172,6 +8372,14 @@ class ScheduledTaskService:
async def _mark_execution_started_for_claimed_request(
self,
request: TaskExecutionRequest,
+ ) -> tuple[TaskExecutionRequest, bool]:
+ # One store step, on the door the claim itself runs through: a cancellation that
+ # arrives meanwhile is answered once the start (or the requeue) has landed.
+ return await self._store_step(self._start_claimed_execution, request)
+
+ def _start_claimed_execution(
+ self,
+ request: TaskExecutionRequest,
) -> tuple[TaskExecutionRequest, bool]:
observed_identity = request.observed_activation_identity
request = self.request_store.refresh_claimed_request(request)
@@ -8267,12 +8475,11 @@ class ScheduledTaskService:
# that created this Run. Only that transaction stamps the Run as
# its owner; cron fires keep the invalidation-aware fast path.
consumed_generation = task_schedule_generation(request.metadata)
- consumed_one_shot = consumed_generation is not None
- if consumed_one_shot:
- self.store.load()
- else:
- self.store.maybe_reload()
- task = self.store.get_task(request.task_id or "")
+ task = await self._store_step(
+ self._load_claimed_task,
+ request.task_id or "",
+ consumed_one_shot=consumed_generation is not None,
+ )
if task is None:
raise ValueError(f"task '{request.task_id}' not found")
task_id = task.id
@@ -8321,7 +8528,8 @@ class ScheduledTaskService:
raise ValueError("hook request requires prompt")
user_context = self._resource_user_context(request.metadata)
if request.session_policy == "create_per_run":
- session_id = self._reserve_runtime_session(
+ session_id = await self._store_step(
+ self._reserve_runtime_session,
agent_name=request.agent_name,
agent_id=request.agent_id,
deliver_key=request.deliver_key,
@@ -8403,7 +8611,9 @@ class ScheduledTaskService:
requeue_metadata[
AGENT_RUN_DELIVERY_OUTCOME_METADATA_KEY
] = result.delivery_outcome
- self.request_store.requeue(request.id, metadata=requeue_metadata)
+ await self._store_step(
+ self.request_store.requeue, request.id, metadata=requeue_metadata
+ )
else:
raise ValueError(f"unknown task request type: {request.request_type}")
except asyncio.CancelledError:
@@ -8472,7 +8682,10 @@ class ScheduledTaskService:
settled_out_of_band = False
finally:
if should_complete:
- written_status = self.request_store.complete(
+ # The settlement's store steps run on the door, in their own order, so
+ # none of them commits on the loop; the scheduler jobs follow on the loop.
+ _written_status, projected, retired = await self._store_step(
+ self._settle_claimed_request,
request,
ok=not error,
error=error,
@@ -8486,36 +8699,12 @@ class ScheduledTaskService:
stderr=stderr,
timed_out=timed_out,
escalation_run_id=escalation_run_id,
+ project_on_cancel=project_terminal_definition_on_cancel,
)
- # A STOP THAT NEVER BECAME A ``CancelledError``. ``cancel_run`` can
- # commit its flag after the work returned, so ``complete`` normalizes
- # this run to ``canceled`` while the coroutine ran to the end without
- # ever being interrupted -- and a command stamp refused by that same
- # flag then left the definition still reporting the fire BEFORE this
- # one. Read from the status ACTUALLY written rather than from
- # ``project_terminal_definition_on_cancel``, which says only that the
- # cancellation arrived the usual way: a service shutdown cancels the
- # same coroutine and settles ``interrupted``, which is a stop of the
- # SERVICE and not of this fire.
- fire_was_stopped = written_status == "canceled" and request.request_type in {
- "task_run",
- "scheduled",
- }
- if fire_was_stopped:
- # BEFORE the projection, because the escalation is claimable the
- # whole time this coroutine is still finishing up.
- self._retract_escalation_of_stopped_fire(request.id)
- if written_status is not None and (
- # The projection is the lane that settles a canceled fire.
- project_terminal_definition_on_cancel
- or fire_was_stopped
- ):
- self._project_terminal_definition_result(
- self.request_store.get_run(request.id),
- execution_id=request.id,
- expected_status=written_status,
- )
- self.reconcile_jobs()
+ if retired is not None:
+ self._retire_one_shot_job(retired)
+ if projected:
+ await self._reconcile_jobs_off_loop()
if reconcile_delivery_on_return and session_id:
manager = getattr(self.controller, "session_turns", None)
reconcile_delivery = getattr(
@@ -8561,6 +8750,63 @@ class ScheduledTaskService:
session_id,
)
+ def _settle_claimed_request(
+ self,
+ request: TaskExecutionRequest,
+ *,
+ project_on_cancel: bool,
+ **outcome: Any,
+ ) -> tuple[Optional[str], bool, Optional[str]]:
+ """A claimed execution's terminal store writes, in order, on the runtime-work
+ executor: the Run's result, a stopped fire's escalation, and the definition's
+ projection. Returns the status written, whether the projection ran, and the
+ one-shot definition it retired: the caller retires that job and reconciles the
+ scheduler jobs on the loop, which owns them."""
+
+ written_status = self.request_store.complete(request, **outcome)
+ # A STOP THAT NEVER BECAME A ``CancelledError``. ``cancel_run`` can
+ # commit its flag after the work returned, so ``complete`` normalizes
+ # this run to ``canceled`` while the coroutine ran to the end without
+ # ever being interrupted -- and a command stamp refused by that same
+ # flag then left the definition still reporting the fire BEFORE this
+ # one. Read from the status ACTUALLY written rather than from
+ # ``project_on_cancel``, which says only that the cancellation arrived
+ # the usual way: a service shutdown cancels the same coroutine and
+ # settles ``interrupted``, which is a stop of the SERVICE and not of
+ # this fire.
+ fire_was_stopped = written_status == "canceled" and request.request_type in {
+ "task_run",
+ "scheduled",
+ }
+ if fire_was_stopped:
+ # BEFORE the projection, because the escalation is claimable the
+ # whole time this coroutine is still finishing up.
+ self._retract_escalation_of_stopped_fire(request.id)
+ if written_status is None or not (
+ # The projection is the lane that settles a canceled fire.
+ project_on_cancel
+ or fire_was_stopped
+ ):
+ return written_status, False, None
+ retired = self._project_terminal_definition_result(
+ self.request_store.get_run(request.id),
+ execution_id=request.id,
+ expected_status=written_status,
+ retire_job=False,
+ )
+ return written_status, True, retired
+
+ def _load_claimed_task(self, task_id: str, *, consumed_one_shot: bool) -> Optional[ScheduledTask]:
+ """The definition a claimed task fire runs, read fresh: a scheduled one-shot is
+ retired in the same transaction that created its Run, so that fire reloads the
+ whole mirror; a cron fire keeps the invalidation-aware fast path."""
+
+ if consumed_one_shot:
+ self.store.load()
+ else:
+ self.store.maybe_reload()
+ return self.store.get_task(task_id)
+
def _runtime_default_workdir(self) -> Optional[str]:
"""The configured directory Agent work runs in, when nothing closer answered.
@@ -8675,17 +8921,17 @@ class ScheduledTaskService:
# command from the definition as it stood then, and the executor re-read the
# definition after claiming. This is the copy that will actually run.
non_owner_retired_expectation = (
- self.store._read_state(task)
+ await self._store_step(self.store._read_state, task)
if schedule_generation is None
and task.schedule_type == "at"
and task.retired_at is not None
else None
)
- self._record_executed_command(execution_id, task)
+ await self._store_step(self._record_executed_command, execution_id, task)
spawn_cwd = (
task.cwd
- or self._bound_session_workdir(task)
+ or await self._store_step(self._bound_session_workdir, task)
or self._runtime_default_workdir()
)
if not spawn_cwd:
@@ -8711,8 +8957,9 @@ class ScheduledTaskService:
cwd=spawn_cwd,
timeout_seconds=effective_timeout,
label=f"scheduled task {task.id}",
- on_spawn=lambda pid, identity: self._require_command_worker_record(
- execution_id, identity
+ # The worker record lands, on the door, before the spec is written.
+ on_spawn=lambda pid, identity: self._store_step(
+ self._require_command_worker_record, execution_id, identity
),
max_output_bytes=COMMAND_TASK_OUTPUT_CAP_BYTES,
env=background_command_env(
@@ -8798,7 +9045,7 @@ class ScheduledTaskService:
# stop raises: by the time control is here the runner has reaped
# the tree, so the stored identity now names a dead pid that the
# OS is free to hand to something else.
- self._clear_command_worker(execution_id)
+ await self._store_step(self._clear_command_worker, execution_id)
# THE ESCALATION IS COMPOSED BEFORE THE STAMP AND QUEUED BY IT. A definition
# carrying ``on_failure == "agent"`` owes the user ONE Agent turn per failed
@@ -8834,7 +9081,8 @@ class ScheduledTaskService:
parent_run_id=execution_id,
metadata=dict(task.metadata or {}),
)
- stamped = self.store.mark_task_result(
+ stamped = await self._store_step(
+ self.store.mark_task_result,
task.id,
error=error,
disable_one_shot=disable_one_shot,
@@ -8896,7 +9144,7 @@ class ScheduledTaskService:
if not error:
error = self._t(_TASK_RESULT_NOT_RECORDED_I18N_KEY)
# An ``at`` command task just disabled itself; the APScheduler job must drop.
- self.reconcile_jobs()
+ await self._reconcile_jobs_off_loop()
return TaskExecutionResult(
error=error,
session_key=task.session_key or "",
@@ -8979,6 +9227,10 @@ class ScheduledTaskService:
metadata_allows_harness_runtime,
)
+ # The fire's own copy of the definition: binding recovery edits it in place for
+ # this fire's retry, and the published mirror must never show an edit the
+ # database has not accepted (the store persists what it accepts itself).
+ task = ScheduledTask.from_dict(task.to_dict())
error: Optional[str] = None
complete_on_return = True
reconcile_delivery_on_return = False
@@ -8987,12 +9239,12 @@ class ScheduledTaskService:
session_key = task.session_key
if not metadata_allows_harness_runtime(task.metadata):
error = HARNESS_ACCESS_FORBIDDEN_CODE
- if not self.store.suspend_task(task.id, error=error):
+ if not await self._store_step(self.store.suspend_task, task.id, error=error):
logger.warning(
"Remote-origin scheduled task %s changed before it could be suspended",
task.id,
)
- self.reconcile_jobs()
+ await self._reconcile_jobs_off_loop()
return TaskExecutionResult(
error=error,
session_key=session_key,
@@ -9006,7 +9258,7 @@ class ScheduledTaskService:
# retry belongs here -- before the fire, so a cleanup that succeeds is off the
# books whatever this run does, and outside the ``try`` because it reports
# itself and must never be mistaken for the run's own failure.
- self._retry_orphaned_reservations(task)
+ await self._store_step(self._retry_orphaned_reservations, task)
if task.has_command:
# A command definition runs a subprocess instead of prompting an Agent, so
# none of the session/binding machinery below applies to it. An
@@ -9021,7 +9273,8 @@ class ScheduledTaskService:
)
try:
if task.session_policy == "create_per_run":
- session_id = self._reserve_runtime_session(
+ session_id = await self._store_step(
+ self._reserve_runtime_session,
agent_name=task.agent_name,
agent_id=agent_id,
deliver_key=task.deliver_key,
@@ -9057,7 +9310,7 @@ class ScheduledTaskService:
# re-reserve one, anything else is user-pinned and gets paused. Either
# way the user is told exactly once.
logger.error("Scheduled task %s has an unresolvable session binding: %s", task.id, exc)
- binding_change = self._recover_pinned_session_binding(task, exc)
+ binding_change = await self._store_step(self._recover_pinned_session_binding, task, exc)
error = binding_change.detail
failure_code = FAILURE_CODE_UNRESOLVABLE_TARGET
if binding_change.action == "rebound" and binding_change.new_session_id:
@@ -9104,7 +9357,8 @@ class ScheduledTaskService:
except Exception as exc:
error = str(exc)
logger.error("Scheduled task %s failed: %s", task.id, exc, exc_info=True)
- if not self.store.mark_task_result(
+ if not await self._store_step(
+ self.store.mark_task_result,
task.id,
error=error,
disable_one_shot=disable_one_shot,
@@ -9146,7 +9400,7 @@ class ScheduledTaskService:
await self._emit_binding_change(
binding_change, run_id=execution_id, run_error=error
)
- self.reconcile_jobs()
+ await self._reconcile_jobs_off_loop()
return TaskExecutionResult(
error=error,
session_key=session_key,
@@ -9187,7 +9441,9 @@ class ScheduledTaskService:
from core.services.dispatch import SOURCE_SCHEDULED, dispatch_turn_with_outcome
from core.session_turns import TurnSubmissionResult
- target_info = resolve_session_id_target(session_id) if session_id else None
+ target_info = (
+ await self._store_step(resolve_session_id_target, session_id) if session_id else None
+ )
target = target_info.session_key if target_info else parse_session_key(session_key or "")
delivery_target = self._resolve_delivery_target(
session_target=target,
@@ -9244,7 +9500,8 @@ class ScheduledTaskService:
"target_was_busy": state.target_was_busy,
}
if isinstance(state, TurnSubmissionResult) and state.delivery_status == "canceled":
- self.request_store.settle_without_result(
+ await self._store_step(
+ self.request_store.settle_without_result,
execution_id,
terminal_status="canceled",
metadata={
@@ -9316,7 +9573,8 @@ class ScheduledTaskService:
error_text = self._t(
SETTLEMENT_I18N_KEYS.get(settled_by, SETTLEMENT_I18N_KEYS[SETTLED_BY_NO_TERMINAL_RESULT])
)
- supported, _settled = self._settle_agent_run_without_result(
+ supported, _settled = await self._store_step(
+ self._settle_agent_run_without_result,
execution_id,
settled_by=settled_by,
error=error_text,
@@ -9621,23 +9879,30 @@ class ScheduledTaskService:
*,
execution_id: str,
expected_status: str,
- ) -> None:
- """Project only the exact terminal transition the caller actually won."""
+ retire_job: bool = True,
+ ) -> Optional[str]:
+ """Project only the exact terminal transition the caller actually won.
+
+ Returns the id of a one-shot definition the projection retired. With
+ ``retire_job=False`` its scheduler job is left for the caller to retire on the
+ loop (``_retire_one_shot_job``), so a caller on the runtime-work executor touches
+ only the store.
+ """
if run is None:
- return
+ return None
status = str(run.get("status") or "").strip()
if status != expected_status or status not in TERMINAL_RUN_STATUSES:
- return
+ return None
definition_id = str(
run.get("definition_id") or run.get("task_id") or ""
).strip()
if not definition_id:
- return
+ return None
self.store.maybe_reload()
task = self.store.get_task(definition_id)
if task is None:
- return
+ return None
run_session_id = str(run.get("session_id") or "").strip()
task_session_id = str(task.session_id or "").strip()
run_session_key = str(
@@ -9653,7 +9918,7 @@ class ScheduledTaskService:
execution_id,
definition_id,
)
- return
+ return None
error = None if status == "succeeded" else str(run.get("error") or "").strip()
if status != "succeeded" and not error:
metadata = run.get("metadata")
@@ -9671,7 +9936,7 @@ class ScheduledTaskService:
execution_id,
definition_id,
)
- return
+ return None
metadata = run.get("metadata")
schedule_generation = task_schedule_generation(metadata)
legacy_consumed_marker = bool(
@@ -9685,7 +9950,7 @@ class ScheduledTaskService:
execution_id,
definition_id,
)
- return
+ return None
retire_one_shot = (
str(run.get("source_kind") or "") == "scheduler"
and (
@@ -9731,26 +9996,34 @@ class ScheduledTaskService:
execution_id,
definition_id,
)
- return
+ return None
if not recorded:
logger.warning(
"Definition %s changed before terminal Run %s could be projected",
definition_id,
execution_id,
)
- return
- if retire_one_shot:
- try:
- job_id = self._job_ids.pop(task.id, task.id)
- if self.scheduler.get_job(job_id) is not None:
- self.scheduler.remove_job(job_id)
- self._one_shot_job_identities.pop(job_id, None)
- self._job_signatures.pop(task.id, None)
- except Exception:
- logger.exception(
- "Failed to retire scheduler job for terminal definition %s",
- definition_id,
- )
+ return None
+ if not retire_one_shot:
+ return None
+ if retire_job:
+ self._retire_one_shot_job(task.id)
+ return task.id
+
+ def _retire_one_shot_job(self, task_id: str) -> None:
+ """Drop a retired one-shot definition's scheduler job; the loop owns the jobs."""
+
+ try:
+ job_id = self._job_ids.pop(task_id, task_id)
+ if self.scheduler.get_job(job_id) is not None:
+ self.scheduler.remove_job(job_id)
+ self._one_shot_job_identities.pop(job_id, None)
+ self._job_signatures.pop(task_id, None)
+ except Exception:
+ logger.exception(
+ "Failed to retire scheduler job for terminal definition %s",
+ task_id,
+ )
def _settle_agent_run_without_result(
self,
@@ -9857,7 +10130,9 @@ class ScheduledTaskService:
task.agent_name,
)
task.agent_name = None
- if not self._persist_task_session_id(task, new_session_id):
+ if not self._persist_task_session_id(
+ task, new_session_id, follows_session=not settings_preserved
+ ):
# THE REBIND DID NOT LAND (HFR-268). ``new_session_id`` is left
# unset on purpose: it is what ``_execute_task`` reads to decide
# whether to retry the fire, so an unstored rebind cannot dispatch
@@ -10062,7 +10337,9 @@ class ScheduledTaskService:
return session_id, preserved
return None
- def _persist_task_session_id(self, task: ScheduledTask, session_id: str) -> bool:
+ def _persist_task_session_id(
+ self, task: ScheduledTask, session_id: str, *, follows_session: bool = False
+ ) -> bool:
"""Store the rebind. ``False`` means the guard refused it — do NOT act on it.
HFR-268. This used to swallow the conflict and return ``None``, and the comment
@@ -10077,14 +10354,16 @@ class ScheduledTaskService:
"""
try:
- self._write_task_session_id(task, session_id)
+ self._write_task_session_id(task, session_id, follows_session=follows_session)
except DefinitionWriteConflict as exc:
logger.warning("Could not persist the rebind for task %s: %s", task.id, exc)
return False
return True
- def _write_task_session_id(self, task: ScheduledTask, session_id: str) -> None:
- self.store.rebind_session(task.id, session_id)
+ def _write_task_session_id(
+ self, task: ScheduledTask, session_id: str, *, follows_session: bool = False
+ ) -> None:
+ self.store.rebind_session(task.id, session_id, follows_session=follows_session)
def _pause_task(self, task: ScheduledTask) -> None:
try:
@@ -10120,7 +10399,7 @@ class ScheduledTaskService:
longer reflects (HFR-266).
"""
- task = self.store.get_task(change.task_id)
+ task = await self._store_step(self.store.get_task, change.task_id)
if task is None:
return
recorded = task.metadata.get(BINDING_RECOVERY_METADATA_KEY) if isinstance(task.metadata, dict) else None
@@ -10164,7 +10443,8 @@ class ScheduledTaskService:
"settings_preserved": change.settings_preserved,
}
try:
- recorded_recovery = self.store.record_binding_recovery(
+ recorded_recovery = await self._store_step(
+ self.store.record_binding_recovery,
change.task_id,
recovery_payload,
binding_notice=binding_notice,
@@ -10737,8 +11017,11 @@ class ScheduledTaskService:
user_context: Any = None,
_capture_dispatch_result: bool = False,
) -> Optional[str] | TaskDispatchResult:
- target_info = resolve_session_id_target(session_id) if session_id else None
- self._require_execution_agent_access(
+ target_info = (
+ await self._store_step(resolve_session_id_target, session_id) if session_id else None
+ )
+ await self._store_step(
+ self._require_execution_agent_access,
agent_name=agent_name,
target_info=target_info,
user_context=user_context,
@@ -10886,7 +11169,9 @@ class ScheduledTaskService:
if native_session_fork is None and target_info and not str(target_info.native_session_id or "").strip():
native_session_fork = fork_metadata_from_session_metadata(getattr(target_info, "metadata", None))
- definition_name, definition_prompt = self._definition_display_fields(task_id)
+ definition_name, definition_prompt = await self._store_step(
+ self._definition_display_fields, task_id
+ )
return MessageContext(
user_id=session_target_context["user_id"],
diff --git a/tests/test_scheduled_tasks.py b/tests/test_scheduled_tasks.py
index 2a6de1662..0fc198b96 100644
--- a/tests/test_scheduled_tasks.py
+++ b/tests/test_scheduled_tasks.py
@@ -171,6 +171,16 @@ async def _fire_and_finish_scheduled_task(
await asyncio.gather(*executions)
+async def _drain_requests_settled(service: ScheduledTaskService) -> None:
+ """Drain claimable requests and wait for the executions they spawn: an execution's
+ store steps run on the runtime-work executor, so it settles after the drain returns."""
+
+ await service._drain_requests()
+ executions = tuple(service._inflight_executions.values())
+ if executions:
+ await asyncio.gather(*executions, return_exceptions=True)
+
+
def test_parse_session_key_accepts_channel_and_thread() -> None:
parsed = parse_session_key("slack::channel::C123::thread::171717.123")
@@ -493,10 +503,10 @@ def test_hfr_172_task_definition_wake_follows_commit_and_not_rollback(
release = threading.Event()
failure: list[BaseException] = []
- def blocking_save() -> None:
+ def blocking_save(*args) -> None:
entered.set()
assert release.wait(timeout=1)
- original_save()
+ original_save(*args)
def add_task() -> None:
try:
@@ -526,7 +536,7 @@ def test_hfr_172_task_definition_wake_follows_commit_and_not_rollback(
events.clear()
task = store.list_tasks()[0]
- monkeypatch.setattr(store, "_save", lambda: (_ for _ in ()).throw(OSError("disk full")))
+ monkeypatch.setattr(store, "_save", lambda *_args: (_ for _ in ()).throw(OSError("disk full")))
with pytest.raises(OSError, match="disk full"):
store.set_enabled(task.id, False)
assert events == []
@@ -1889,10 +1899,10 @@ def test_hfr_282_task_reload_waits_for_result_stamp_mirror_commit(
original_save = store._save
original_load = store._load_unlocked
- def blocked_save() -> None:
+ def blocked_save(*args) -> None:
write_entered.set()
assert release_write.wait(timeout=1)
- original_save()
+ original_save(*args)
def observed_load() -> None:
load_entered.set()
@@ -4072,6 +4082,9 @@ def test_service_lease_loss_cancels_inflight_execution(tmp_path: Path, monkeypat
assert service._owns_service_instance() is False
with pytest.raises(asyncio.CancelledError):
await execution
+ # The cancelled claim is settled by an owned task off the loop, which ``stop``
+ # waits for; this test stops nothing, so it waits itself.
+ await asyncio.gather(*tuple(service._cancel_settlements))
asyncio.run(_exercise())
@@ -4368,7 +4381,7 @@ def test_drain_requests_executes_hook_send(tmp_path: Path) -> None:
request_store=request_store,
)
- asyncio.run(service._drain_requests())
+ asyncio.run(_drain_requests_settled(service))
assert len(calls) == 1
context, message, parsed = calls[0]
@@ -9746,7 +9759,7 @@ def test_drain_requests_reserves_watch_create_per_run_before_session_validation(
service._execute_request = _execute_request # type: ignore[method-assign]
- asyncio.run(service._drain_requests())
+ asyncio.run(_drain_requests_settled(service))
assert calls == [
{
@@ -9866,7 +9879,7 @@ def test_drain_requests_records_scheduled_create_per_run_reserved_session(tmp_pa
service._execute_request = _execute_request # type: ignore[method-assign]
- asyncio.run(service._drain_requests())
+ asyncio.run(_drain_requests_settled(service))
assert calls == [
{
@@ -10050,7 +10063,7 @@ def test_drain_requests_agent_run_passes_agent_name(tmp_path: Path) -> None:
request_store=request_store,
)
- asyncio.run(service._drain_requests())
+ asyncio.run(_drain_requests_settled(service))
assert len(calls) == 1
context, message, parsed = calls[0]
@@ -10094,7 +10107,7 @@ def test_run_task_request_does_not_disable_one_shot(tmp_path: Path) -> None:
)
service = ScheduledTaskService(controller=controller, store=store, request_store=request_store)
- asyncio.run(service._drain_requests())
+ asyncio.run(_drain_requests_settled(service))
reloaded = ScheduledTaskStore(path)
updated = reloaded.get_task(task.id)
@@ -13065,7 +13078,9 @@ def test_dead_accepted_owner_converges_run_session_and_persisted_fifo(
request_store=request_store,
)
- async def _wait_for(predicate, *, timeout=1.0) -> None:
+ # Each step of the execution settles through the runtime-work executor, so allow
+ # for thread hops under a loaded suite.
+ async def _wait_for(predicate, *, timeout=5.0) -> None:
async def _poll() -> None:
while not predicate():
await asyncio.sleep(0.005)
@@ -16223,7 +16238,9 @@ def test_a_locked_database_that_refuses_release_and_record_still_recovers_the_or
# the release is entered and lifted only after the record write has failed against
# it. Nothing about the release or the record is stubbed; both hit SQLite and get
# SQLite's answer.
- holder = sqlite3_module.connect(str(db_path))
+ # The fire's store steps run on the runtime-work executor, so the fault is taken and
+ # lifted from that thread.
+ holder = sqlite3_module.connect(str(db_path), check_same_thread=False)
fault = {"armed": True, "held": False, "release_calls": 0, "record_failures": 0}
original_release = SQLiteSessionsService.release_reserved_agent_session
original_record = ScheduledTaskStore.record_orphaned_reservations
@@ -20186,3 +20203,473 @@ def test_complete_with_an_escalation_marker_stamps_no_owed_notice(
assert notice is not None and notice.get("state"), (
"the suppression leaked to ordinary failures; every failed run would go silent"
)
+
+
+def _hold_state_write_lock(held: threading.Event, release: threading.Event, released_in_time: list[bool]) -> None:
+ """Another writer holding SQLite's write lock, as a second process or a long
+ transaction would, until the loop lets it go (or two seconds pass)."""
+
+ holder = sqlite3.connect(str(paths.get_sqlite_state_path()), timeout=10)
+ try:
+ holder.execute("BEGIN IMMEDIATE")
+ held.set()
+ released_in_time.append(release.wait(2))
+ holder.rollback()
+ finally:
+ holder.close()
+
+
+def _sqlite_claimed_agent_run(tmp_path: Path, monkeypatch) -> tuple[ScheduledTaskService, TaskExecutionStore, Any]:
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ request_store = TaskExecutionStore()
+ controller = SimpleNamespace(
+ agent_service=SimpleNamespace(
+ is_backend_ready=lambda _backend: True,
+ runtime_activation_identity_for_request=Mock(return_value=None),
+ activation_registry=None,
+ )
+ )
+ service = ScheduledTaskService(
+ controller=controller,
+ store=ScheduledTaskStore(tmp_path / "scheduled_tasks.json"),
+ request_store=request_store,
+ )
+ queued = request_store.enqueue_agent_run(message="run", agent_backend="codex", session_id="ses-runtime")
+ claimed = request_store.claim(queued.id)
+ assert claimed is not None
+ return service, request_store, claimed
+
+
+def test_a_claimed_executions_store_steps_never_hold_the_loop(tmp_path: Path, monkeypatch) -> None:
+ """#2649: a claimed execution's start and terminal writes wait for SQLite's write lock and commit. The loop runs
+ every session's turns, so it stays free while another writer holds the lock, and the execution still settles."""
+
+ service, request_store, claimed = _sqlite_claimed_agent_run(tmp_path, monkeypatch)
+ service._execute_agent_run = AsyncMock( # type: ignore[assignment]
+ return_value=scheduled_tasks.AgentRunExecutionResult(error=None, complete_on_return=True)
+ )
+ held, release = threading.Event(), threading.Event()
+ released_in_time: list[bool] = []
+
+ async def _go() -> None:
+ holder = threading.Thread(target=_hold_state_write_lock, args=(held, release, released_in_time))
+ holder.start()
+ assert await asyncio.to_thread(held.wait, 5)
+ execution = asyncio.create_task(service._execute_claimed_request(claimed))
+ await asyncio.sleep(0.1)
+ # The loop lets the other writer go while the execution waits for it.
+ release.set()
+ await execution
+ await asyncio.to_thread(holder.join, 5)
+
+ asyncio.run(_go())
+ assert released_in_time == [True]
+ assert request_store.get_run(claimed.id)["status"] == "succeeded"
+
+
+def test_a_claim_cancelled_before_it_runs_is_settled_off_the_loop(tmp_path: Path, monkeypatch) -> None:
+ """#2649: the execution's done-callback is synchronous and runs on the loop. A claim cancelled before its
+ coroutine ran is settled by an owned task on the runtime-work executor instead, so the loop stays free while
+ another writer holds the lock, and the claim still goes back to the queue."""
+
+ service, request_store, claimed = _sqlite_claimed_agent_run(tmp_path, monkeypatch)
+ held, release = threading.Event(), threading.Event()
+ released_in_time: list[bool] = []
+
+ async def _go() -> None:
+ holder = threading.Thread(target=_hold_state_write_lock, args=(held, release, released_in_time))
+ holder.start()
+ assert await asyncio.to_thread(held.wait, 5)
+ service._spawn_execution(claimed, None)
+ service._inflight_executions[claimed.id].cancel()
+ await asyncio.sleep(0.1)
+ # Until its settlement lands the claim stays live, so recovery cannot reclaim the
+ # still-running row and have the settlement terminalize the new claim.
+ assert claimed.id in service._live_claimed_run_ids
+ release.set()
+ for _ in range(100):
+ if request_store.get_run(claimed.id)["status"] == "queued" and not service._cancel_settlements:
+ break
+ await asyncio.sleep(0.05)
+ await asyncio.to_thread(holder.join, 5)
+
+ asyncio.run(_go())
+ assert released_in_time == [True]
+ assert request_store.get_run(claimed.id)["status"] == "queued"
+ assert claimed.id not in service._live_claimed_run_ids
+
+
+def test_a_mirror_read_never_waits_for_a_writer_and_never_sees_its_unwritten_edit(tmp_path: Path, monkeypatch) -> None:
+ """#2649: writers hold the mirror lock across their store write, now on worker threads. The controller loop reads
+ the mirror (``get_task``, ``list_tasks``, ``reconcile_jobs``), so a read takes no lock: it answers at once, from
+ the published mirror, which a writer replaces only after its write lands."""
+ from concurrent.futures import ThreadPoolExecutor
+
+ store = ScheduledTaskStore(tmp_path / "scheduled_tasks.json")
+ task = store.add_task(
+ session_key="slack::channel::C123",
+ prompt="run once",
+ schedule_type="at",
+ run_at="2030-01-01T00:00:00+00:00",
+ timezone_name="UTC",
+ )
+ write_entered, release_write = threading.Event(), threading.Event()
+ original_save = store._save
+
+ def slow_save(*args) -> None:
+ write_entered.set()
+ assert release_write.wait(timeout=5)
+ original_save(*args)
+
+ monkeypatch.setattr(store, "_save", slow_save)
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ stamping = executor.submit(store.set_enabled, task.id, False)
+ assert write_entered.wait(timeout=1)
+ try:
+ read = executor.submit(lambda: (store.get_task(task.id), store.list_tasks()))
+ current, listed = read.result(timeout=0.5)
+ assert current.enabled is True and [item.enabled for item in listed] == [True]
+ finally:
+ release_write.set()
+ stamping.result(timeout=5)
+ assert store.get_task(task.id).enabled is False
+
+
+def test_a_reset_rebind_persists_its_whole_recovery_whatever_the_mirror_publishes_meanwhile(
+ tmp_path: Path, monkeypatch
+) -> None:
+ """#2649: binding recovery now runs on a worker while the mirror stays published to the loop. A reset rebind's
+ recorded choice (follow the rebound session, drop the stale Agent pin) is written with the binding by the store,
+ so a reload that republishes the mirror between the recovery and its write cannot drop it, and the definition the
+ loop reads is never edited before the database accepts it."""
+ from core.vibe_agents import VibeAgentStore
+ from storage.models import agent_sessions
+ from storage.sessions_service import SQLiteSessionsService
+
+ db_path = _binding_env(tmp_path, monkeypatch)
+ agent_store = VibeAgentStore(db_path)
+ try:
+ agent_store.create(name="nightly", backend="claude", model="legacy-model")
+ finally:
+ agent_store.close()
+ sessions = SQLiteSessionsService(db_path)
+ try:
+ pinned = sessions.bind_agent_session(
+ scope_key="slack::channel::C123",
+ agent_name="claude",
+ session_anchor="slack_C123:definition_republish",
+ native_session_id="native-republish",
+ )
+ finally:
+ sessions.close()
+ engine = create_sqlite_engine(db_path)
+ with engine.begin() as conn:
+ conn.execute(
+ agent_sessions.update().where(agent_sessions.c.id == pinned).values(agent_name="nightly", model="legacy-model")
+ )
+ store = ScheduledTaskStore()
+ task = store.add_task(
+ name="digest",
+ session_key="",
+ session_id=pinned,
+ session_policy="create_once",
+ agent_name="nightly",
+ prompt="send digest",
+ schedule_type="cron",
+ cron="0 * * * *",
+ timezone_name="UTC",
+ deliver_key="slack::channel::C123",
+ metadata={"session_scope_id": "slack::channel::C123"},
+ )
+ sessions = SQLiteSessionsService(db_path)
+ try:
+ assert sessions.delete_agent_sessions(scope_key="slack::channel::C123", session_anchor_prefix="slack_C123")
+ finally:
+ sessions.close()
+ agent_store = VibeAgentStore(db_path)
+ try:
+ assert agent_store.remove("nightly")
+ finally:
+ agent_store.close()
+
+ store = ScheduledTaskStore()
+ store.set_enabled(task.id, True)
+ service = _dispatching_binding_service(tmp_path, store, db_path=db_path)
+ published = store.get_task(task.id)
+ rebind = store.rebind_session
+
+ def rebind_after_a_republish(*args, **kwargs):
+ # Another executor's reload lands between the recovery and its write.
+ store.load()
+ return rebind(*args, **kwargs)
+
+ monkeypatch.setattr(store, "rebind_session", rebind_after_a_republish)
+ asyncio.run(service._execute_task(published, execution_id="exec-1", disable_one_shot=False))
+
+ rebound = ScheduledTaskStore().get_task(task.id)
+ assert rebound.session_id and rebound.session_id != pinned
+ assert rebound.agent_name is None
+ assert rebound.metadata.get(BINDING_FOLLOWS_SESSION_METADATA_KEY) is True
+ # The object the loop read before the fire was never edited in place.
+ assert published.agent_name == "nightly" and published.session_id == pinned
+
+
+
[email protected]("cancelled_during", ["_reserve_runtime_session", "_require_execution_agent_access"])
+def test_a_cancellation_during_a_per_run_fires_store_steps_is_answered_at_its_dispatch(
+ tmp_path: Path, monkeypatch, cancelled_during: str
+) -> None:
+ """#2649: a per-run fire's store steps run on the runtime-work door, but its cancellation points stay where the loop
+ had them. A cancellation that arrives while the reservation or a pre-dispatch step runs is answered at the dispatch,
+ which already holds the reserved session -- so a reservation is never stranded between steps, where nothing owns it
+ and the orphan sweep cannot find it (a per-run reservation carries no definition stamp)."""
+
+ store = ScheduledTaskStore(tmp_path / "scheduled_tasks.json")
+ task = store.add_task(
+ session_key="slack::channel::C123",
+ session_policy="create_per_run",
+ prompt="send digest",
+ schedule_type="cron",
+ cron="0 * * * *",
+ timezone_name="UTC",
+ deliver_key="slack::channel::C123",
+ )
+ dispatched: list[str] = []
+ answered: list[str] = []
+
+ async def submit_scheduled(session_id, _context, _prompt, **_kwargs):
+ dispatched.append(session_id)
+ try:
+ await asyncio.sleep(5)
+ except asyncio.CancelledError:
+ answered.append(session_id)
+ raise
+
+ service = ScheduledTaskService(
+ controller=SimpleNamespace(
+ platform_settings_managers={"slack": object()},
+ im_clients={},
+ session_turn_gate=SimpleNamespace(submit_scheduled=submit_scheduled),
+ ),
+ store=store,
+ request_store=TaskExecutionStore(tmp_path / "task_requests"),
+ )
+ entered, release = threading.Event(), threading.Event()
+ released: list[str] = []
+
+ def step(name: str, result):
+ def run(*_args, **_kwargs):
+ if name == cancelled_during:
+ entered.set()
+ assert release.wait(5)
+ return result
+
+ return run
+
+ async def build_context(*_args, **_kwargs):
+ return SimpleNamespace()
+
+ monkeypatch.setattr(service, "_reserve_runtime_session", step("_reserve_runtime_session", "ses-reserved-1"))
+ monkeypatch.setattr(
+ scheduled_tasks,
+ "resolve_session_id_target",
+ step("resolve_session_id_target", SimpleNamespace(session_key=parse_session_key("slack::channel::C123"))),
+ )
+ monkeypatch.setattr(service, "_require_execution_agent_access", step("_require_execution_agent_access", None))
+ monkeypatch.setattr(service, "_build_context", build_context)
+ monkeypatch.setattr(
+ service, "_release_reserved_session", lambda session_id, *, reason: released.append(session_id) or True
+ )
+
+ async def _go() -> None:
+ fire = asyncio.create_task(service._execute_task(task, execution_id="exec-1", disable_one_shot=False))
+ assert await asyncio.to_thread(entered.wait, 5)
+ fire.cancel()
+ await asyncio.sleep(0.05)
+ release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await fire
+
+ asyncio.run(_go())
+ assert dispatched == ["ses-reserved-1"], "the cancellation was answered before the dispatch adopted the reservation"
+ assert answered == ["ses-reserved-1"]
+ assert released == []
+
+
+def test_a_cancellation_during_a_binding_recovery_still_reports_the_change_it_committed(
+ tmp_path: Path, monkeypatch
+) -> None:
+ """#2649: the pinned-binding recovery runs on the runtime-work door. A cancellation that arrives while it runs waits
+ for it, and the change it committed still gets its result stamp, its recovery marker and its notice: the cancellation
+ is answered at the next real await, as on the loop, not between the recovery and its bookkeeping."""
+
+ _binding_env(tmp_path, monkeypatch)
+ store = ScheduledTaskStore(tmp_path / "scheduled_tasks.json")
+ task = store.add_task(
+ session_key="",
+ session_id="sesdoesnotexist",
+ session_policy="existing",
+ prompt="send digest",
+ schedule_type="cron",
+ cron="0 * * * *",
+ timezone_name="UTC",
+ deliver_key="slack::channel::C123",
+ )
+ service = _binding_service(tmp_path, store, [])
+ notices = _spy_binding_notices(service)
+ entered, release = threading.Event(), threading.Event()
+ recover = service._recover_pinned_session_binding
+
+ def slow_recovery(*args, **kwargs):
+ entered.set()
+ assert release.wait(5)
+ return recover(*args, **kwargs)
+
+ monkeypatch.setattr(service, "_recover_pinned_session_binding", slow_recovery)
+
+ async def _go() -> None:
+ fire = asyncio.create_task(service._execute_task(task, execution_id="exec-1", disable_one_shot=False))
+ assert await asyncio.to_thread(entered.wait, 5)
+ fire.cancel()
+ await asyncio.sleep(0.05)
+ release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await fire
+
+ asyncio.run(_go())
+ assert [notice.action for notice in notices] == ["failing"], "the committed binding change was never reported"
+ current = store.get_task(task.id)
+ assert current is not None and "sesdoesnotexist" in (current.last_error or "")
+ assert current.metadata.get(BINDING_RECOVERY_METADATA_KEY, {}).get("action") == "failing", (
+ "the change's recovery marker was never recorded"
+ )
+
+
+def test_an_older_mirror_capture_never_undoes_a_newer_one(tmp_path: Path) -> None:
+ """#2649: ``reconcile_jobs`` captures the mirror on a thread, so captures may complete out of order. Each is ordered
+ by the mirror generation it actually read -- not by when it was submitted -- so a capture submitted first but taken
+ after a disable applies the disable, even when a later-submitted capture of the older mirror landed before it."""
+
+ store = ScheduledTaskStore(tmp_path / "scheduled_tasks.json")
+ task = store.add_task(
+ session_key="slack::channel::C123",
+ prompt="send digest",
+ schedule_type="cron",
+ cron="0 * * * *",
+ timezone_name="UTC",
+ )
+ service = ScheduledTaskService(
+ controller=SimpleNamespace(platform_settings_managers={"slack": object()}, im_clients={}),
+ store=store,
+ request_store=TaskExecutionStore(tmp_path / "task_requests"),
+ )
+ service.scheduler = _StubScheduler()
+ capture = store.list_tasks_with_generation
+ first_waits, first_entered = threading.Event(), threading.Event()
+ calls: list[int] = []
+
+ def ordered_capture():
+ calls.append(len(calls))
+ if len(calls) == 1:
+ first_entered.set()
+ assert first_waits.wait(5)
+ return capture()
+
+ store.list_tasks_with_generation = ordered_capture # type: ignore[method-assign]
+
+ async def _go() -> None:
+ first = asyncio.create_task(service._reconcile_jobs_off_loop())
+ assert await asyncio.to_thread(first_entered.wait, 5)
+ await service._reconcile_jobs_off_loop() # submitted second, captures the enabled mirror, lands first
+ assert service.scheduler.get_job(task.id) is not None
+ store.set_enabled(task.id, False)
+ first_waits.set()
+ await first # submitted first, captures the disabled mirror, lands last
+
+ asyncio.run(_go())
+ assert service.scheduler.get_job(task.id) is None, "an older mirror capture undid the disable a newer one applied"
+
+
+class _TaskWithoutCancelCount:
+ """A running task as Python 3.10 presents it: ``cancel``, but no ``uncancel`` to count requests with."""
+
+ def __init__(self, task: "asyncio.Task[Any]") -> None:
+ self._task = task
+
+ def cancel(self, *args) -> bool:
+ return self._task.cancel(*args)
+
+
+ ("door", "step_raises", "python310"),
+ [("_store_step", True, False), ("_run_runtime_sync", True, False), ("_store_step", False, True)],
+)
+def test_a_cancellation_caught_by_a_store_step_is_answered_at_the_next_await(
+ tmp_path: Path, monkeypatch, door: str, step_raises: bool, python310: bool
+) -> None:
+ """#2649: a store step on the runtime-work door holds a cancellation that arrives while it runs until it lands, then
+ hands it back for the caller's next await -- also when the step raised (its error must not swallow the cancellation
+ and let the caller settle an ordinary failure), and on Python 3.10, whose tasks keep no cancellation count."""
+
+ service = ScheduledTaskService(
+ controller=SimpleNamespace(platform_settings_managers={}, im_clients={}),
+ store=ScheduledTaskStore(tmp_path / "scheduled_tasks.json"),
+ request_store=TaskExecutionStore(tmp_path / "task_requests"),
+ )
+ entered, release = threading.Event(), threading.Event()
+ outcomes: list[str] = []
+
+ def step() -> str:
+ entered.set()
+ assert release.wait(5)
+ if step_raises:
+ raise RuntimeError("step error")
+ return "step result"
+
+ if python310:
+ current_task = asyncio.current_task
+ monkeypatch.setattr(
+ scheduled_tasks.asyncio, "current_task", lambda *args: _TaskWithoutCancelCount(current_task(*args))
+ )
+
+ async def caller() -> None:
+ try:
+ outcomes.append(await getattr(service, door)(step))
+ except RuntimeError as exc:
+ outcomes.append(str(exc))
+ try:
+ await asyncio.sleep(5)
+ except asyncio.CancelledError:
+ outcomes.append("cancelled at the next await")
+ raise
+
+ async def _go() -> None:
+ run = asyncio.create_task(caller())
+ assert await asyncio.to_thread(entered.wait, 5)
+ run.cancel()
+ await asyncio.sleep(0.05)
+ release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await run
+
+ asyncio.run(_go())
+ assert outcomes == ["step error" if step_raises else "step result", "cancelled at the next await"]
+
+
+def test_a_timeout_that_expires_during_a_store_step_still_reads_as_a_timeout(tmp_path: Path) -> None:
+ """#2649: a cancellation a store step holds and re-arms is counted once, so an ``asyncio.timeout`` around the caller
+ still turns its own expiry into ``TimeoutError`` instead of letting a bare cancellation escape the caller."""
+
+ service = ScheduledTaskService(
+ controller=SimpleNamespace(platform_settings_managers={}, im_clients={}),
+ store=ScheduledTaskStore(tmp_path / "scheduled_tasks.json"),
+ request_store=TaskExecutionStore(tmp_path / "task_requests"),
+ )
+
+ async def _go() -> None:
+ with pytest.raises(TimeoutError):
+ async with asyncio.timeout(0.05):
+ await service._store_step(time.sleep, 0.2)
+ await asyncio.sleep(5)
+
+ asyncio.run(_go())