Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(harness): run a claimed execution's store work off the controller loop (#2654)
Call White33f6f4ca4234
diff --git a/core/command_runner.py b/core/command_runner.pyindex 045ed43fe..6f966d5fa 100644--- a/core/command_runner.py+++ b/core/command_runner.py@@ -15,6 +15,7 @@ differ, under a running runner.from __future__ import annotationsimport asyncio+import inspectimport loggingimport osimport shlex@@ -22,7 +23,7 @@ import signalimport sysfrom collections import dequefrom dataclasses import dataclass-from typing import Callable, Collection, Mapping, Optional+from typing import Awaitable, Callable, Collection, Mapping, Optionalfrom config import paths as config_pathsfrom core import watch_worker@@ -289,7 +290,9 @@ async def run_supervised_command(cwd: str,timeout_seconds: float,label: str,- on_spawn: Optional[Callable[[int, Optional[PersistedProcessIdentity]], None]] = None,+ on_spawn: Optional[+ Callable[[int, Optional[PersistedProcessIdentity]], Optional[Awaitable[None]]]+ ] = None,max_output_bytes: Optional[int] = None,env: Optional[Mapping[str, str]] = None,extra_env: Optional[Mapping[str, str]] = None,@@ -338,7 +341,11 @@ async def run_supervised_command(# other failure here rather than leaking the very orphan it is protecting# against. Still before the spec is written, so the supervisor is blocked# on stdin and nothing the user asked for has started yet.- on_spawn(process.pid, identity)+ # A callback that persists the worker may do so off the loop and hand back+ # the awaitable; the spec is not written until it has landed.+ recorded = on_spawn(process.pid, identity)+ if inspect.isawaitable(recorded):+ await recordedif process.stdin is None:await terminate_and_communicate(process, logger, label)raise RuntimeError("watch worker supervisor stdin is unavailable")diff --git a/core/scheduled_tasks.py b/core/scheduled_tasks.pyindex ac66fa0d4..43cd2dcb4 100644--- a/core/scheduled_tasks.py+++ b/core/scheduled_tasks.py@@ -1279,12 +1279,17 @@ def _serialize_task_mirror(return _locked+# A cancelled claim's settlement that projected no definition result.+_NOT_PROJECTED = object()++class ScheduledTaskStore:def __init__(self, path: Optional[Path] = None):self.path = path or (paths.get_state_dir() / "scheduled_tasks.json")self._sqlite = SQLiteBackgroundTaskStore() if path is None else Noneself._signature: Optional[_PathSignature] = None- self._tasks: Dict[str, ScheduledTask] = {}+ # The published mirror and its generation, one tuple so a reader takes both at once.+ self._published: tuple[int, Dict[str, ScheduledTask]] = (0, {})#: Set when a failed write left this mirror INCOMPLETE, cleared by the reload#: that repairs it. See ``maybe_reload`` and ``_reload_after_lost_write``.self._reload_required = False@@ -1382,24 +1387,68 @@ class ScheduledTaskStore:self.load()return True- def _save(self) -> None:+ def _save(self, tasks: Optional[Dict[str, ScheduledTask]] = None) -> None:+ """Write the file backend's mirror: ``tasks`` when a writer is about to publish+ them, else the published one."""+if self._sqlite is not None:returnself.path.parent.mkdir(parents=True, exist_ok=True)- payload = {"tasks": [task.to_dict() for task in self.list_tasks()]}+ ordered = sorted(+ (tasks if tasks is not None else self._tasks).values(),+ key=lambda item: (item.created_at, item.id),+ )+ payload = {"tasks": [task.to_dict() for task in ordered]}write_atomic(self.path, json.dumps(payload, indent=2))self._signature = _path_signature(self.path)+ # The mirror is PUBLISHED, never edited in place: a writer changes a private copy+ # (``_working_copy``), writes it, and only then swaps in a new dict+ # (``_publish``/``_unpublish``); a reload builds a new dict too. A reader therefore+ # takes no lock -- one dict reference is one consistent mirror -- so the controller+ # loop never waits behind a writer that holds ``_reload_lock`` across SQLite I/O+ # on a worker thread, and never sees an edit the database has not accepted.++ @property+ def _tasks(self) -> Dict[str, ScheduledTask]:+ return self._published[1]++ @_tasks.setter+ def _tasks(self, tasks: Dict[str, ScheduledTask]) -> None:+ # Every publication is a new generation, so a reader can tell which of two+ # mirrors it captured is newer, whatever order the captures complete in.+ self._published = (self._published[0] + 1, tasks)+def list_tasks(self) -> list[ScheduledTask]:- with self._reload_lock:- return sorted(- self._tasks.values(),- key=lambda item: (item.created_at, item.id),- )+ return self.list_tasks_with_generation()[1]++ def list_tasks_with_generation(self) -> tuple[int, list[ScheduledTask]]:+ """The published mirror, sorted, with the generation it was published as."""++ generation, tasks = self._published+ return generation, sorted(tasks.values(), key=lambda item: (item.created_at, item.id))def get_task(self, task_id: str) -> Optional[ScheduledTask]:- with self._reload_lock:- return self._tasks.get(task_id)+ return self._tasks.get(task_id)++ def _working_copy(self, task_id: str) -> Optional[ScheduledTask]:+ """A private copy of a published definition, for a writer to change."""++ task = self._tasks.get(task_id)+ return ScheduledTask.from_dict(task.to_dict()) if task is not None else None++ def _required_working_copy(self, task_id: str) -> ScheduledTask:+ task = self._working_copy(task_id)+ if task is None:+ raise KeyError(task_id)+ return task++ def _publish(self, task: ScheduledTask) -> None:+ self._tasks = {**self._tasks, task.id: task}++ def _unpublish(self, task_id: str) -> None:+ if task_id in self._tasks:+ self._tasks = {key: value for key, value in self._tasks.items() if key != task_id}def refresh_task(self, task_id: str) -> Optional[ScheduledTask]:"""Reload and read one scheduler definition under the same mirror lock."""@@ -1495,7 +1544,8 @@ class ScheduledTaskStore:"a file-backed scheduled task store cannot commit a queued run ""with the task row")- self._save()+ self._save({**self._tasks, task.id: task})+ self._publish(task)_publish_task_definitions_updated()return Trueif queued_run is None:@@ -1522,6 +1572,7 @@ class ScheduledTaskStore:self._reload_after_lost_write(task.id)raiseif landed:+ self._publish(task)_publish_task_definitions_updated()return Trueself.load()@@ -1547,7 +1598,7 @@ class ScheduledTaskStore:"stale mirror entry for %s",task_id,)- self._tasks.pop(task_id, None)+ self._unpublish(task_id)self._signature = Noneself._reload_required = True@@ -1569,7 +1620,6 @@ class ScheduledTaskStore:"""task.updated_at = _utc_now_iso()- self._tasks[task.id] = tasktry:if self._sqlite is not None:# No ``expect``: this is the create/adopt entry point (``add_task``),@@ -1583,12 +1633,14 @@ class ScheduledTaskStore:self.load()_publish_task_definitions_updated()return self._tasks[task.id]+ self._publish(task)_publish_task_definitions_updated()return task- self._save()+ self._save({**self._tasks, task.id: task})except Exception:self._reload_after_lost_write(task.id)raise+ self._publish(task)_publish_task_definitions_updated()return task@@ -1663,22 +1715,23 @@ class ScheduledTaskStore:if task_id not in self._tasks:return False- del self._tasks[task_id]try:if self._sqlite is not None:self._sqlite.remove_task(task_id)+ self._unpublish(task_id)_publish_task_definitions_updated()return True- self._save()+ self._save({key: value for key, value in self._tasks.items() if key != task_id})except Exception:self._reload_after_lost_write(task_id)raise+ self._unpublish(task_id)_publish_task_definitions_updated()return True@_serialize_task_mirrordef set_enabled(self, task_id: str, enabled: bool) -> ScheduledTask:- task = self._tasks[task_id]+ task = self._required_working_copy(task_id)if enabled and not task.enabled:require_task_resumable(task_id,@@ -1698,11 +1751,24 @@ class ScheduledTaskStore:return task@_serialize_task_mirror- def rebind_session(self, task_id: str, session_id: str) -> None:- """Host recovery changes only the binding, retaining authenticated metadata."""- task = self._tasks[task_id]+ def rebind_session(self, task_id: str, session_id: str, *, follows_session: bool = False) -> None:+ """Host recovery changes the binding, retaining authenticated metadata.++ ``follows_session`` is a reset rebind's recorded choice: the definition follows+ its rebound session's Agent from now on, so its own Agent pin is dropped and the+ marker that keeps an unrelated edit from re-pinning it is set. Applied to the+ store's copy in the same write as the binding, so neither lands without the other.+ """++ task = self._required_working_copy(task_id)expect = self._read_state(task)task.session_id = session_id+ if follows_session:+ task.metadata = {+ **(task.metadata if isinstance(task.metadata, dict) else {}),+ BINDING_FOLLOWS_SESSION_METADATA_KEY: True,+ }+ task.agent_name = Nonetask.updated_at = _utc_now_iso()if not self._write_task(task, expect):raise DefinitionWriteConflict(task_id, definition_type="scheduled task")@@ -1743,7 +1809,7 @@ class ScheduledTaskStore:ensure_harness_definition_write(user_context)ensure_agent_name_access(agent_name, user_context=user_context)- task = self._tasks[task_id]+ task = self._required_working_copy(task_id)# Captured before the first mutation: this is the state the CALLER read# (``vibe task update`` resolved Agents and Sessions from this very object),# and it is what the write below re-asserts.@@ -1809,7 +1875,7 @@ class ScheduledTaskStore:) -> bool:"""Retire the exact one-shot schedule APScheduler reported missed."""- task = self._tasks.get(task_id)+ task = self._working_copy(task_id)if (task is Noneor not task.enabled@@ -1840,7 +1906,8 @@ class ScheduledTaskStore:task.metadata.pop(COMMAND_TIMED_OUT_METADATA_KEY, None)task.metadata.pop(TASK_LAST_RESULT_STATUS_METADATA_KEY, None)task.updated_at = task.retired_at- self._save()+ self._save({**self._tasks, task.id: task})+ self._publish(task)return True@_serialize_task_mirror@@ -1861,7 +1928,7 @@ class ScheduledTaskStore:"""self.maybe_reload()- task = self._tasks.get(task_id)+ task = self._working_copy(task_id)if task is None:return Falseexpect = self._read_state(task)@@ -1883,6 +1950,7 @@ class ScheduledTaskStore:self._reload_after_lost_write(task.id)raiseif landed:+ self._publish(task)return Trueself.load()return False@@ -1918,7 +1986,7 @@ class ScheduledTaskStore:"""self.maybe_reload()- task = self._tasks.get(task_id)+ task = self._working_copy(task_id)if task is None:return Falseexpect = self._read_state(task)@@ -1979,7 +2047,7 @@ class ScheduledTaskStore:"""self.maybe_reload()- task = self._tasks.get(task_id)+ task = self._working_copy(task_id)if task is None:return Falsenon_owner_expectation = expected_non_owner_retired_one_shot@@ -2097,11 +2165,12 @@ class ScheduledTaskStore:expected_terminal_run_id=expected_terminal_run_id,)+ @_serialize_task_mirrordef suspend_task(self, task_id: str, *, error: str) -> bool:"""Atomically disable a definition before an unsafe execution boundary."""self.maybe_reload()- task = self._tasks.get(task_id)+ task = self._working_copy(task_id)if task is None:return Falseexpect = self._read_state(task)@@ -3962,6 +4031,22 @@ class _ScheduledRuntimeWorkHandler(RuntimeWorkHandler):return True+def _rearm_cancellation() -> None:+ """Deliver a cancellation the current task caught, but did not answer, at its next await.++ ``uncancel`` first (Python 3.11+, absent on 3.10, which counts no requests), so the+ request is counted once, as an ``asyncio.timeout`` around the caller expects.+ """++ current = asyncio.current_task()+ if current is None:+ return+ uncancel = getattr(current, "uncancel", None)+ if uncancel is not None:+ uncancel()+ current.cancel()++class ScheduledTaskService:"""Controller-owned runtime that executes persisted scheduled tasks."""@@ -4010,6 +4095,13 @@ class ScheduledTaskService:# job ids are unique per registration so a queued event from a replaced# DateTrigger cannot be mistaken for its successor.self._job_ids: Dict[str, str] = {}+ # Settlements of claims cancelled before they ran, started by their done-callback,+ # and the claims they settle: a claim stays live, so recovery cannot reclaim it,+ # until its settlement has landed.+ self._cancel_settlements: set["asyncio.Task[None]"] = set()+ self._settling_claim_ids: set[str] = set()+ # The store and mirror generation ``reconcile_jobs`` last applied.+ self._job_reconcile_applied: tuple[Any, int] = (None, 0)self._one_shot_job_identities: Dict[str, tuple[str, str, str, str]] = {}self._running = Falseself._watch_store_restart_count = 0@@ -4956,12 +5048,62 @@ class ScheduledTaskService:notify_after(token, delay)async def _run_runtime_sync(self, operation, /, *args, **kwargs): # noqa: ANN001, ANN202+ """The door for the service's blocking store work: the runtime-work executor.++ A store step cannot be split by cancellation, as it could not while it ran on+ the loop: a cancellation that arrives meanwhile waits for the step to land and+ is raised after it, so the caller sees exactly what committed. A step that+ raised raises its own error, with the cancellation re-armed behind it.+ """++ step, cancelled = await self._land_runtime_step(operation, *args, **kwargs)+ if cancelled and not step.cancelled():+ if step.exception() is not None:+ _rearm_cancellation()+ return step.result()+ raise asyncio.CancelledError+ return step.result()++ async def _store_step(self, operation, /, *args, **kwargs): # noqa: ANN001, ANN202+ """One store step of a claimed execution: on the runtime-work door, with the+ cancellation point it had on the loop.++ These steps ran synchronously on the loop before #2649, so a cancellation could+ land only at the execution's next real await -- the dispatch, the command, a+ delivery -- with every step before it complete and its outcome in hand. It still+ does: a cancellation that arrives while a step runs waits for the step to land,+ the step's result is returned (or its error raised), and the cancellation is+ re-armed for the next await that is not itself a store step. So a result a step+ committed (a reservation, a binding recovery, a settlement) is never dropped+ between steps, and each cancellation is answered where the execution always+ answered it.+ """++ step, cancelled = await self._land_runtime_step(operation, *args, **kwargs)+ if cancelled and not step.cancelled():+ _rearm_cancellation()+ return step.result()++ async def _land_runtime_step(self, operation, /, *args, **kwargs): # noqa: ANN001, ANN202+ """Run one store step on the runtime-work door and wait for it to land, whatever+ cancels the caller meanwhile. Returns ``(step, cancelled)``: the finished step,+ whose ``result()`` gives its outcome, and whether a cancellation was caught."""+call = partial(operation, *args, **kwargs)supervisor = getattr(self.controller, "runtime_work_supervisor", None)# Gate on the type: a stand-in controller's attribute can be callable without being awaitable.if isinstance(supervisor, RuntimeWorkSupervisor):- return await supervisor.run_sync(call)- return await asyncio.to_thread(call)+ step = asyncio.ensure_future(supervisor.run_sync(call))+ else:+ step = asyncio.ensure_future(asyncio.to_thread(call))+ cancelled = False+ while not step.done():+ try:+ # ``wait`` never cancels the step, and raises only the caller's cancellation.+ await asyncio.wait((step,))+ except asyncio.CancelledError:+ cancelled = True+ return step, cancelleddef _start_legacy_runtime_work_probes(self) -> None:if self.store.sqlite_backend is None:@@ -5260,6 +5402,10 @@ class ScheduledTaskService:await taskexcept (asyncio.CancelledError, Exception):pass+ # A claim cancelled before it ran is settled by an owned task its done-callback+ # started; it lands before shutdown returns, like an execution's own settlement.+ if self._cancel_settlements:+ await asyncio.gather(*tuple(self._cancel_settlements), return_exceptions=True)teardown = self._ensure_service_teardown_task()if teardown is not None:await teardown@@ -5271,6 +5417,7 @@ class ScheduledTaskService:self._request_recovery_unregistration_task = Noneself._inflight_executions.clear()self._request_capacity_reservations.clear()+ self._settling_claim_ids.clear()self._live_claimed_run_ids = frozenset()self._inflight_requests.clear()self._inflight_cancellation_causes.clear()@@ -5566,9 +5713,29 @@ class ScheduledTaskService:def reconcile_jobs(self) -> None:if not self._owns_service_instance():return+ self._apply_job_reconcile(self.store, *self.store.list_tasks_with_generation())++ async def _reconcile_jobs_off_loop(self) -> None:+ """``reconcile_jobs`` for a claimed execution: the mirror read (which waits on the+ store's reload lock while a scan holds it) runs on the runtime-work executor, the+ scheduler jobs are changed on the loop, which owns them."""++ if not self._owns_service_instance():+ return+ store = self.store+ self._apply_job_reconcile(store, *await self._store_step(store.list_tasks_with_generation))++ def _apply_job_reconcile(self, store: Any, generation: int, tasks: list[ScheduledTask]) -> None:+ # Captures may complete out of order once one waits on a thread: only a mirror at+ # least as new as the last one applied may change the jobs, so an older mirror+ # never undoes a newer one. Ordered by the generation each capture actually read.+ applied_store, applied_generation = self._job_reconcile_applied+ if applied_store is store and generation < applied_generation:+ return+ self._job_reconcile_applied = (store, generation)desired_ids = set()desired_job_ids = set()- for task in self.store.list_tasks():+ for task in tasks:if not task.enabled or task.retired_at is not None:continuedesired_ids.add(task.id)@@ -7951,18 +8118,23 @@ class ScheduledTaskService:self._session_lock_owners[lock_key] = request.idtask = asyncio.create_task(self._execute_claimed_request(request))self._inflight_executions[request.id] = task- self._live_claimed_run_ids = frozenset(self._inflight_executions)+ self._refresh_live_claims()self._inflight_requests[request.id] = requesttask.add_done_callback(lambda finished, rid=request.id, key=lock_key: self._on_execution_done(rid, key, finished))+ def _refresh_live_claims(self) -> None:+ self._live_claimed_run_ids = frozenset(self._inflight_executions) | frozenset(self._settling_claim_ids)+def _on_execution_done(self, request_id: str, lock_key: Optional[str], task: "asyncio.Task[Any]") -> None:self._inflight_executions.pop(request_id, None)- self._live_claimed_run_ids = frozenset(self._inflight_executions)request = self._inflight_requests.pop(request_id, None)+ if task.cancelled() and request is not None:+ self._settling_claim_ids.add(request_id)+ self._refresh_live_claims()interruption = self._inflight_cancellation_causes.pop(request_id,SETTLED_BY_INTERRUPTED,@@ -7980,45 +8152,73 @@ class ScheduledTaskService:RuntimeWorkLane.FAILURE_NOTICES,)# ``_execute_claimed_request`` already terminalizes cancellations and- # records ordinary failures. A task cancelled before its coroutine enters- # has no execution to settle; put that bare claim back in the queue.+ # records ordinary failures. A task cancelled before its coroutine enters, or+ # while its start was landing, has no execution of its own to settle: an owned+ # task settles it, on the door rather than in this synchronous callback on the+ # loop, and ``stop`` waits for it.if task.cancelled():if request is not None:- try:- current = self.request_store.get_run(request.id)- if not current or current.get("status") in TERMINAL_RUN_STATUSES:- return- if (- current.get("pid") is None- ):- self.request_store.requeue(request.id)- return- written_status = self.request_store.complete(- request,- ok=False,- error=self._t(SETTLEMENT_I18N_KEYS[interruption]),- interrupt_reason=interruption,- )- if written_status is not None and request.task_id and request.request_type in {- "task_run",- "scheduled",- }:- self._project_terminal_definition_result(- self.request_store.get_run(request.id),- execution_id=request.id,- expected_status=written_status,- )- self.reconcile_jobs()- except Exception:- logger.exception(- "Failed to terminalize canceled claimed request %s",- request_id,- )+ settlement = asyncio.get_running_loop().create_task(+ self._settle_cancelled_claim(request, interruption),+ name=f"scheduled-claim-cancel-settlement:{request_id}",+ )+ self._cancel_settlements.add(settlement)+ settlement.add_done_callback(self._cancel_settlements.discard)returnexc = task.exception()if exc is not None:logger.error("Claimed request %s crashed: %r", request_id, exc, exc_info=exc)+ async def _settle_cancelled_claim(self, request: TaskExecutionRequest, interruption: str) -> None:+ """Settle a claim whose execution was cancelled before it ran: a claim that never+ started goes back to the queue, a started one is terminalized as interrupted."""++ try:+ retired = await self._run_runtime_sync(+ self._settle_cancelled_claim_in_store, request, interruption+ )+ except Exception:+ logger.exception("Failed to terminalize canceled claimed request %s", request.id)+ return+ finally:+ # Settled (or given up): the claim stops being live, and the request lane+ # may now pick up the run this settlement requeued.+ self._settling_claim_ids.discard(request.id)+ self._refresh_live_claims()+ self._wake_runtime_work(RuntimeWorkLane.REQUESTS)+ if retired is _NOT_PROJECTED:+ return+ if retired is not None:+ self._retire_one_shot_job(retired)+ await self._reconcile_jobs_off_loop()++ def _settle_cancelled_claim_in_store(+ self, request: TaskExecutionRequest, interruption: str+ ) -> Any:+ current = self.request_store.get_run(request.id)+ if not current or current.get("status") in TERMINAL_RUN_STATUSES:+ return _NOT_PROJECTED+ if current.get("pid") is None:+ self.request_store.requeue(request.id)+ return _NOT_PROJECTED+ written_status = self.request_store.complete(+ request,+ ok=False,+ error=self._t(SETTLEMENT_I18N_KEYS[interruption]),+ interrupt_reason=interruption,+ )+ if written_status is None or not request.task_id or request.request_type not in {+ "task_run",+ "scheduled",+ }:+ return _NOT_PROJECTED+ return self._project_terminal_definition_result(+ self.request_store.get_run(request.id),+ execution_id=request.id,+ expected_status=written_status,+ retire_job=False,+ )+def _claimed_request_backend_resolution(self,request: TaskExecutionRequest,@@ -8172,6 +8372,14 @@ class ScheduledTaskService:async def _mark_execution_started_for_claimed_request(self,request: TaskExecutionRequest,+ ) -> tuple[TaskExecutionRequest, bool]:+ # One store step, on the door the claim itself runs through: a cancellation that+ # arrives meanwhile is answered once the start (or the requeue) has landed.+ return await self._store_step(self._start_claimed_execution, request)++ def _start_claimed_execution(+ self,+ request: TaskExecutionRequest,) -> tuple[TaskExecutionRequest, bool]:observed_identity = request.observed_activation_identityrequest = self.request_store.refresh_claimed_request(request)@@ -8267,12 +8475,11 @@ class ScheduledTaskService:# that created this Run. Only that transaction stamps the Run as# its owner; cron fires keep the invalidation-aware fast path.consumed_generation = task_schedule_generation(request.metadata)- consumed_one_shot = consumed_generation is not None- if consumed_one_shot:- self.store.load()- else:- self.store.maybe_reload()- task = self.store.get_task(request.task_id or "")+ task = await self._store_step(+ self._load_claimed_task,+ request.task_id or "",+ consumed_one_shot=consumed_generation is not None,+ )if task is None:raise ValueError(f"task '{request.task_id}' not found")task_id = task.id@@ -8321,7 +8528,8 @@ class ScheduledTaskService:raise ValueError("hook request requires prompt")user_context = self._resource_user_context(request.metadata)if request.session_policy == "create_per_run":- session_id = self._reserve_runtime_session(+ session_id = await self._store_step(+ self._reserve_runtime_session,agent_name=request.agent_name,agent_id=request.agent_id,deliver_key=request.deliver_key,@@ -8403,7 +8611,9 @@ class ScheduledTaskService:requeue_metadata[AGENT_RUN_DELIVERY_OUTCOME_METADATA_KEY] = result.delivery_outcome- self.request_store.requeue(request.id, metadata=requeue_metadata)+ await self._store_step(+ self.request_store.requeue, request.id, metadata=requeue_metadata+ )else:raise ValueError(f"unknown task request type: {request.request_type}")except asyncio.CancelledError:@@ -8472,7 +8682,10 @@ class ScheduledTaskService:settled_out_of_band = Falsefinally:if should_complete:- written_status = self.request_store.complete(+ # The settlement's store steps run on the door, in their own order, so+ # none of them commits on the loop; the scheduler jobs follow on the loop.+ _written_status, projected, retired = await self._store_step(+ self._settle_claimed_request,request,ok=not error,error=error,@@ -8486,36 +8699,12 @@ class ScheduledTaskService:stderr=stderr,timed_out=timed_out,escalation_run_id=escalation_run_id,+ project_on_cancel=project_terminal_definition_on_cancel,)- # A STOP THAT NEVER BECAME A ``CancelledError``. ``cancel_run`` can- # commit its flag after the work returned, so ``complete`` normalizes- # this run to ``canceled`` while the coroutine ran to the end without- # ever being interrupted -- and a command stamp refused by that same- # flag then left the definition still reporting the fire BEFORE this- # one. Read from the status ACTUALLY written rather than from- # ``project_terminal_definition_on_cancel``, which says only that the- # cancellation arrived the usual way: a service shutdown cancels the- # same coroutine and settles ``interrupted``, which is a stop of the- # SERVICE and not of this fire.- fire_was_stopped = written_status == "canceled" and request.request_type in {- "task_run",- "scheduled",- }- if fire_was_stopped:- # BEFORE the projection, because the escalation is claimable the- # whole time this coroutine is still finishing up.- self._retract_escalation_of_stopped_fire(request.id)- if written_status is not None and (- # The projection is the lane that settles a canceled fire.- project_terminal_definition_on_cancel- or fire_was_stopped- ):- self._project_terminal_definition_result(- self.request_store.get_run(request.id),- execution_id=request.id,- expected_status=written_status,- )- self.reconcile_jobs()+ if retired is not None:+ self._retire_one_shot_job(retired)+ if projected:+ await self._reconcile_jobs_off_loop()if reconcile_delivery_on_return and session_id:manager = getattr(self.controller, "session_turns", None)reconcile_delivery = getattr(@@ -8561,6 +8750,63 @@ class ScheduledTaskService:session_id,)+ def _settle_claimed_request(+ self,+ request: TaskExecutionRequest,+ *,+ project_on_cancel: bool,+ **outcome: Any,+ ) -> tuple[Optional[str], bool, Optional[str]]:+ """A claimed execution's terminal store writes, in order, on the runtime-work+ executor: the Run's result, a stopped fire's escalation, and the definition's+ projection. Returns the status written, whether the projection ran, and the+ one-shot definition it retired: the caller retires that job and reconciles the+ scheduler jobs on the loop, which owns them."""++ written_status = self.request_store.complete(request, **outcome)+ # A STOP THAT NEVER BECAME A ``CancelledError``. ``cancel_run`` can+ # commit its flag after the work returned, so ``complete`` normalizes+ # this run to ``canceled`` while the coroutine ran to the end without+ # ever being interrupted -- and a command stamp refused by that same+ # flag then left the definition still reporting the fire BEFORE this+ # one. Read from the status ACTUALLY written rather than from+ # ``project_on_cancel``, which says only that the cancellation arrived+ # the usual way: a service shutdown cancels the same coroutine and+ # settles ``interrupted``, which is a stop of the SERVICE and not of+ # this fire.+ fire_was_stopped = written_status == "canceled" and request.request_type in {+ "task_run",+ "scheduled",+ }+ if fire_was_stopped:+ # BEFORE the projection, because the escalation is claimable the+ # whole time this coroutine is still finishing up.+ self._retract_escalation_of_stopped_fire(request.id)+ if written_status is None or not (+ # The projection is the lane that settles a canceled fire.+ project_on_cancel+ or fire_was_stopped+ ):+ return written_status, False, None+ retired = self._project_terminal_definition_result(+ self.request_store.get_run(request.id),+ execution_id=request.id,+ expected_status=written_status,+ retire_job=False,+ )+ return written_status, True, retired++ def _load_claimed_task(self, task_id: str, *, consumed_one_shot: bool) -> Optional[ScheduledTask]:+ """The definition a claimed task fire runs, read fresh: a scheduled one-shot is+ retired in the same transaction that created its Run, so that fire reloads the+ whole mirror; a cron fire keeps the invalidation-aware fast path."""++ if consumed_one_shot:+ self.store.load()+ else:+ self.store.maybe_reload()+ return self.store.get_task(task_id)+def _runtime_default_workdir(self) -> Optional[str]:"""The configured directory Agent work runs in, when nothing closer answered.@@ -8675,17 +8921,17 @@ class ScheduledTaskService:# command from the definition as it stood then, and the executor re-read the# definition after claiming. This is the copy that will actually run.non_owner_retired_expectation = (- self.store._read_state(task)+ await self._store_step(self.store._read_state, task)if schedule_generation is Noneand task.schedule_type == "at"and task.retired_at is not Noneelse None)- self._record_executed_command(execution_id, task)+ await self._store_step(self._record_executed_command, execution_id, task)spawn_cwd = (task.cwd- or self._bound_session_workdir(task)+ or await self._store_step(self._bound_session_workdir, task)or self._runtime_default_workdir())if not spawn_cwd:@@ -8711,8 +8957,9 @@ class ScheduledTaskService:cwd=spawn_cwd,timeout_seconds=effective_timeout,label=f"scheduled task {task.id}",- on_spawn=lambda pid, identity: self._require_command_worker_record(- execution_id, identity+ # The worker record lands, on the door, before the spec is written.+ on_spawn=lambda pid, identity: self._store_step(+ self._require_command_worker_record, execution_id, identity),max_output_bytes=COMMAND_TASK_OUTPUT_CAP_BYTES,env=background_command_env(@@ -8798,7 +9045,7 @@ class ScheduledTaskService:# stop raises: by the time control is here the runner has reaped# the tree, so the stored identity now names a dead pid that the# OS is free to hand to something else.- self._clear_command_worker(execution_id)+ await self._store_step(self._clear_command_worker, execution_id)# THE ESCALATION IS COMPOSED BEFORE THE STAMP AND QUEUED BY IT. A definition# carrying ``on_failure == "agent"`` owes the user ONE Agent turn per failed@@ -8834,7 +9081,8 @@ class ScheduledTaskService:parent_run_id=execution_id,metadata=dict(task.metadata or {}),)- stamped = self.store.mark_task_result(+ stamped = await self._store_step(+ self.store.mark_task_result,task.id,error=error,disable_one_shot=disable_one_shot,@@ -8896,7 +9144,7 @@ class ScheduledTaskService:if not error:error = self._t(_TASK_RESULT_NOT_RECORDED_I18N_KEY)# An ``at`` command task just disabled itself; the APScheduler job must drop.- self.reconcile_jobs()+ await self._reconcile_jobs_off_loop()return TaskExecutionResult(error=error,session_key=task.session_key or "",@@ -8979,6 +9227,10 @@ class ScheduledTaskService:metadata_allows_harness_runtime,)+ # The fire's own copy of the definition: binding recovery edits it in place for+ # this fire's retry, and the published mirror must never show an edit the+ # database has not accepted (the store persists what it accepts itself).+ task = ScheduledTask.from_dict(task.to_dict())error: Optional[str] = Nonecomplete_on_return = Truereconcile_delivery_on_return = False@@ -8987,12 +9239,12 @@ class ScheduledTaskService:session_key = task.session_keyif not metadata_allows_harness_runtime(task.metadata):error = HARNESS_ACCESS_FORBIDDEN_CODE- if not self.store.suspend_task(task.id, error=error):+ if not await self._store_step(self.store.suspend_task, task.id, error=error):logger.warning("Remote-origin scheduled task %s changed before it could be suspended",task.id,)- self.reconcile_jobs()+ await self._reconcile_jobs_off_loop()return TaskExecutionResult(error=error,session_key=session_key,@@ -9006,7 +9258,7 @@ class ScheduledTaskService:# retry belongs here -- before the fire, so a cleanup that succeeds is off the# books whatever this run does, and outside the ``try`` because it reports# itself and must never be mistaken for the run's own failure.- self._retry_orphaned_reservations(task)+ await self._store_step(self._retry_orphaned_reservations, task)if task.has_command:# A command definition runs a subprocess instead of prompting an Agent, so# none of the session/binding machinery below applies to it. An@@ -9021,7 +9273,8 @@ class ScheduledTaskService:)try:if task.session_policy == "create_per_run":- session_id = self._reserve_runtime_session(+ session_id = await self._store_step(+ self._reserve_runtime_session,agent_name=task.agent_name,agent_id=agent_id,deliver_key=task.deliver_key,@@ -9057,7 +9310,7 @@ class ScheduledTaskService:# re-reserve one, anything else is user-pinned and gets paused. Either# way the user is told exactly once.logger.error("Scheduled task %s has an unresolvable session binding: %s", task.id, exc)- binding_change = self._recover_pinned_session_binding(task, exc)+ binding_change = await self._store_step(self._recover_pinned_session_binding, task, exc)error = binding_change.detailfailure_code = FAILURE_CODE_UNRESOLVABLE_TARGETif binding_change.action == "rebound" and binding_change.new_session_id:@@ -9104,7 +9357,8 @@ class ScheduledTaskService:except Exception as exc:error = str(exc)logger.error("Scheduled task %s failed: %s", task.id, exc, exc_info=True)- if not self.store.mark_task_result(+ if not await self._store_step(+ self.store.mark_task_result,task.id,error=error,disable_one_shot=disable_one_shot,@@ -9146,7 +9400,7 @@ class ScheduledTaskService:await self._emit_binding_change(binding_change, run_id=execution_id, run_error=error)- self.reconcile_jobs()+ await self._reconcile_jobs_off_loop()return TaskExecutionResult(error=error,session_key=session_key,@@ -9187,7 +9441,9 @@ class ScheduledTaskService:from core.services.dispatch import SOURCE_SCHEDULED, dispatch_turn_with_outcomefrom core.session_turns import TurnSubmissionResult- target_info = resolve_session_id_target(session_id) if session_id else None+ target_info = (+ await self._store_step(resolve_session_id_target, session_id) if session_id else None+ )target = target_info.session_key if target_info else parse_session_key(session_key or "")delivery_target = self._resolve_delivery_target(session_target=target,@@ -9244,7 +9500,8 @@ class ScheduledTaskService:"target_was_busy": state.target_was_busy,}if isinstance(state, TurnSubmissionResult) and state.delivery_status == "canceled":- self.request_store.settle_without_result(+ await self._store_step(+ self.request_store.settle_without_result,execution_id,terminal_status="canceled",metadata={@@ -9316,7 +9573,8 @@ class ScheduledTaskService:error_text = self._t(SETTLEMENT_I18N_KEYS.get(settled_by, SETTLEMENT_I18N_KEYS[SETTLED_BY_NO_TERMINAL_RESULT]))- supported, _settled = self._settle_agent_run_without_result(+ supported, _settled = await self._store_step(+ self._settle_agent_run_without_result,execution_id,settled_by=settled_by,error=error_text,@@ -9621,23 +9879,30 @@ class ScheduledTaskService:*,execution_id: str,expected_status: str,- ) -> None:- """Project only the exact terminal transition the caller actually won."""+ retire_job: bool = True,+ ) -> Optional[str]:+ """Project only the exact terminal transition the caller actually won.++ Returns the id of a one-shot definition the projection retired. With+ ``retire_job=False`` its scheduler job is left for the caller to retire on the+ loop (``_retire_one_shot_job``), so a caller on the runtime-work executor touches+ only the store.+ """if run is None:- return+ return Nonestatus = str(run.get("status") or "").strip()if status != expected_status or status not in TERMINAL_RUN_STATUSES:- return+ return Nonedefinition_id = str(run.get("definition_id") or run.get("task_id") or "").strip()if not definition_id:- return+ return Noneself.store.maybe_reload()task = self.store.get_task(definition_id)if task is None:- return+ return Nonerun_session_id = str(run.get("session_id") or "").strip()task_session_id = str(task.session_id or "").strip()run_session_key = str(@@ -9653,7 +9918,7 @@ class ScheduledTaskService:execution_id,definition_id,)- return+ return Noneerror = None if status == "succeeded" else str(run.get("error") or "").strip()if status != "succeeded" and not error:metadata = run.get("metadata")@@ -9671,7 +9936,7 @@ class ScheduledTaskService:execution_id,definition_id,)- return+ return Nonemetadata = run.get("metadata")schedule_generation = task_schedule_generation(metadata)legacy_consumed_marker = bool(@@ -9685,7 +9950,7 @@ class ScheduledTaskService:execution_id,definition_id,)- return+ return Noneretire_one_shot = (str(run.get("source_kind") or "") == "scheduler"and (@@ -9731,26 +9996,34 @@ class ScheduledTaskService:execution_id,definition_id,)- return+ return Noneif not recorded:logger.warning("Definition %s changed before terminal Run %s could be projected",definition_id,execution_id,)- return- if retire_one_shot:- try:- job_id = self._job_ids.pop(task.id, task.id)- if self.scheduler.get_job(job_id) is not None:- self.scheduler.remove_job(job_id)- self._one_shot_job_identities.pop(job_id, None)- self._job_signatures.pop(task.id, None)- except Exception:- logger.exception(- "Failed to retire scheduler job for terminal definition %s",- definition_id,- )+ return None+ if not retire_one_shot:+ return None+ if retire_job:+ self._retire_one_shot_job(task.id)+ return task.id++ def _retire_one_shot_job(self, task_id: str) -> None:+ """Drop a retired one-shot definition's scheduler job; the loop owns the jobs."""++ try:+ job_id = self._job_ids.pop(task_id, task_id)+ if self.scheduler.get_job(job_id) is not None:+ self.scheduler.remove_job(job_id)+ self._one_shot_job_identities.pop(job_id, None)+ self._job_signatures.pop(task_id, None)+ except Exception:+ logger.exception(+ "Failed to retire scheduler job for terminal definition %s",+ task_id,+ )def _settle_agent_run_without_result(self,@@ -9857,7 +10130,9 @@ class ScheduledTaskService:task.agent_name,)task.agent_name = None- if not self._persist_task_session_id(task, new_session_id):+ if not self._persist_task_session_id(+ task, new_session_id, follows_session=not settings_preserved+ ):# THE REBIND DID NOT LAND (HFR-268). ``new_session_id`` is left# unset on purpose: it is what ``_execute_task`` reads to decide# whether to retry the fire, so an unstored rebind cannot dispatch@@ -10062,7 +10337,9 @@ class ScheduledTaskService:return session_id, preservedreturn None- def _persist_task_session_id(self, task: ScheduledTask, session_id: str) -> bool:+ def _persist_task_session_id(+ self, task: ScheduledTask, session_id: str, *, follows_session: bool = False+ ) -> bool:"""Store the rebind. ``False`` means the guard refused it — do NOT act on it.HFR-268. This used to swallow the conflict and return ``None``, and the comment@@ -10077,14 +10354,16 @@ class ScheduledTaskService:"""try:- self._write_task_session_id(task, session_id)+ self._write_task_session_id(task, session_id, follows_session=follows_session)except DefinitionWriteConflict as exc:logger.warning("Could not persist the rebind for task %s: %s", task.id, exc)return Falsereturn True- def _write_task_session_id(self, task: ScheduledTask, session_id: str) -> None:- self.store.rebind_session(task.id, session_id)+ def _write_task_session_id(+ self, task: ScheduledTask, session_id: str, *, follows_session: bool = False+ ) -> None:+ self.store.rebind_session(task.id, session_id, follows_session=follows_session)def _pause_task(self, task: ScheduledTask) -> None:try:@@ -10120,7 +10399,7 @@ class ScheduledTaskService:longer reflects (HFR-266)."""- task = self.store.get_task(change.task_id)+ task = await self._store_step(self.store.get_task, change.task_id)if task is None:returnrecorded = task.metadata.get(BINDING_RECOVERY_METADATA_KEY) if isinstance(task.metadata, dict) else None@@ -10164,7 +10443,8 @@ class ScheduledTaskService:"settings_preserved": change.settings_preserved,}try:- recorded_recovery = self.store.record_binding_recovery(+ recorded_recovery = await self._store_step(+ self.store.record_binding_recovery,change.task_id,recovery_payload,binding_notice=binding_notice,@@ -10737,8 +11017,11 @@ class ScheduledTaskService:user_context: Any = None,_capture_dispatch_result: bool = False,) -> Optional[str] | TaskDispatchResult:- target_info = resolve_session_id_target(session_id) if session_id else None- self._require_execution_agent_access(+ target_info = (+ await self._store_step(resolve_session_id_target, session_id) if session_id else None+ )+ await self._store_step(+ self._require_execution_agent_access,agent_name=agent_name,target_info=target_info,user_context=user_context,@@ -10886,7 +11169,9 @@ class ScheduledTaskService:if native_session_fork is None and target_info and not str(target_info.native_session_id or "").strip():native_session_fork = fork_metadata_from_session_metadata(getattr(target_info, "metadata", None))- definition_name, definition_prompt = self._definition_display_fields(task_id)+ definition_name, definition_prompt = await self._store_step(+ self._definition_display_fields, task_id+ )return MessageContext(user_id=session_target_context["user_id"],diff --git a/tests/test_scheduled_tasks.py b/tests/test_scheduled_tasks.pyindex 2a6de1662..0fc198b96 100644--- a/tests/test_scheduled_tasks.py+++ b/tests/test_scheduled_tasks.py@@ -171,6 +171,16 @@ async def _fire_and_finish_scheduled_task(await asyncio.gather(*executions)+async def _drain_requests_settled(service: ScheduledTaskService) -> None:+ """Drain claimable requests and wait for the executions they spawn: an execution's+ store steps run on the runtime-work executor, so it settles after the drain returns."""++ await service._drain_requests()+ executions = tuple(service._inflight_executions.values())+ if executions:+ await asyncio.gather(*executions, return_exceptions=True)++def test_parse_session_key_accepts_channel_and_thread() -> None:parsed = parse_session_key("slack::channel::C123::thread::171717.123")@@ -493,10 +503,10 @@ def test_hfr_172_task_definition_wake_follows_commit_and_not_rollback(release = threading.Event()failure: list[BaseException] = []- def blocking_save() -> None:+ def blocking_save(*args) -> None:entered.set()assert release.wait(timeout=1)- original_save()+ original_save(*args)def add_task() -> None:try:@@ -526,7 +536,7 @@ def test_hfr_172_task_definition_wake_follows_commit_and_not_rollback(events.clear()task = store.list_tasks()[0]- monkeypatch.setattr(store, "_save", lambda: (_ for _ in ()).throw(OSError("disk full")))+ monkeypatch.setattr(store, "_save", lambda *_args: (_ for _ in ()).throw(OSError("disk full")))with pytest.raises(OSError, match="disk full"):store.set_enabled(task.id, False)assert events == []@@ -1889,10 +1899,10 @@ def test_hfr_282_task_reload_waits_for_result_stamp_mirror_commit(original_save = store._saveoriginal_load = store._load_unlocked- def blocked_save() -> None:+ def blocked_save(*args) -> None:write_entered.set()assert release_write.wait(timeout=1)- original_save()+ original_save(*args)def observed_load() -> None:load_entered.set()@@ -4072,6 +4082,9 @@ def test_service_lease_loss_cancels_inflight_execution(tmp_path: Path, monkeypatassert service._owns_service_instance() is Falsewith pytest.raises(asyncio.CancelledError):await execution+ # The cancelled claim is settled by an owned task off the loop, which ``stop``+ # waits for; this test stops nothing, so it waits itself.+ await asyncio.gather(*tuple(service._cancel_settlements))asyncio.run(_exercise())@@ -4368,7 +4381,7 @@ def test_drain_requests_executes_hook_send(tmp_path: Path) -> None:request_store=request_store,)- asyncio.run(service._drain_requests())+ asyncio.run(_drain_requests_settled(service))assert len(calls) == 1context, message, parsed = calls[0]@@ -9746,7 +9759,7 @@ def test_drain_requests_reserves_watch_create_per_run_before_session_validation(service._execute_request = _execute_request # type: ignore[method-assign]- asyncio.run(service._drain_requests())+ asyncio.run(_drain_requests_settled(service))assert calls == [{@@ -9866,7 +9879,7 @@ def test_drain_requests_records_scheduled_create_per_run_reserved_session(tmp_paservice._execute_request = _execute_request # type: ignore[method-assign]- asyncio.run(service._drain_requests())+ asyncio.run(_drain_requests_settled(service))assert calls == [{@@ -10050,7 +10063,7 @@ def test_drain_requests_agent_run_passes_agent_name(tmp_path: Path) -> None:request_store=request_store,)- asyncio.run(service._drain_requests())+ asyncio.run(_drain_requests_settled(service))assert len(calls) == 1context, message, parsed = calls[0]@@ -10094,7 +10107,7 @@ def test_run_task_request_does_not_disable_one_shot(tmp_path: Path) -> None:)service = ScheduledTaskService(controller=controller, store=store, request_store=request_store)- asyncio.run(service._drain_requests())+ asyncio.run(_drain_requests_settled(service))reloaded = ScheduledTaskStore(path)updated = reloaded.get_task(task.id)@@ -13065,7 +13078,9 @@ def test_dead_accepted_owner_converges_run_session_and_persisted_fifo(request_store=request_store,)- async def _wait_for(predicate, *, timeout=1.0) -> None:+ # Each step of the execution settles through the runtime-work executor, so allow+ # for thread hops under a loaded suite.+ async def _wait_for(predicate, *, timeout=5.0) -> None:async def _poll() -> None:while not predicate():await asyncio.sleep(0.005)@@ -16223,7 +16238,9 @@ def test_a_locked_database_that_refuses_release_and_record_still_recovers_the_or# the release is entered and lifted only after the record write has failed against# it. Nothing about the release or the record is stubbed; both hit SQLite and get# SQLite's answer.- holder = sqlite3_module.connect(str(db_path))+ # The fire's store steps run on the runtime-work executor, so the fault is taken and+ # lifted from that thread.+ holder = sqlite3_module.connect(str(db_path), check_same_thread=False)fault = {"armed": True, "held": False, "release_calls": 0, "record_failures": 0}original_release = SQLiteSessionsService.release_reserved_agent_sessionoriginal_record = ScheduledTaskStore.record_orphaned_reservations@@ -20186,3 +20203,473 @@ def test_complete_with_an_escalation_marker_stamps_no_owed_notice(assert notice is not None and notice.get("state"), ("the suppression leaked to ordinary failures; every failed run would go silent")+++def _hold_state_write_lock(held: threading.Event, release: threading.Event, released_in_time: list[bool]) -> None:+ """Another writer holding SQLite's write lock, as a second process or a long+ transaction would, until the loop lets it go (or two seconds pass)."""++ holder = sqlite3.connect(str(paths.get_sqlite_state_path()), timeout=10)+ try:+ holder.execute("BEGIN IMMEDIATE")+ held.set()+ released_in_time.append(release.wait(2))+ holder.rollback()+ finally:+ holder.close()+++def _sqlite_claimed_agent_run(tmp_path: Path, monkeypatch) -> tuple[ScheduledTaskService, TaskExecutionStore, Any]:+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))+ request_store = TaskExecutionStore()+ controller = SimpleNamespace(+ agent_service=SimpleNamespace(+ is_backend_ready=lambda _backend: True,+ runtime_activation_identity_for_request=Mock(return_value=None),+ activation_registry=None,+ )+ )+ service = ScheduledTaskService(+ controller=controller,+ store=ScheduledTaskStore(tmp_path / "scheduled_tasks.json"),+ request_store=request_store,+ )+ queued = request_store.enqueue_agent_run(message="run", agent_backend="codex", session_id="ses-runtime")+ claimed = request_store.claim(queued.id)+ assert claimed is not None+ return service, request_store, claimed+++def test_a_claimed_executions_store_steps_never_hold_the_loop(tmp_path: Path, monkeypatch) -> None:+ """#2649: a claimed execution's start and terminal writes wait for SQLite's write lock and commit. The loop runs+ every session's turns, so it stays free while another writer holds the lock, and the execution still settles."""++ service, request_store, claimed = _sqlite_claimed_agent_run(tmp_path, monkeypatch)+ service._execute_agent_run = AsyncMock( # type: ignore[assignment]+ return_value=scheduled_tasks.AgentRunExecutionResult(error=None, complete_on_return=True)+ )+ held, release = threading.Event(), threading.Event()+ released_in_time: list[bool] = []++ async def _go() -> None:+ holder = threading.Thread(target=_hold_state_write_lock, args=(held, release, released_in_time))+ holder.start()+ assert await asyncio.to_thread(held.wait, 5)+ execution = asyncio.create_task(service._execute_claimed_request(claimed))+ await asyncio.sleep(0.1)+ # The loop lets the other writer go while the execution waits for it.+ release.set()+ await execution+ await asyncio.to_thread(holder.join, 5)++ asyncio.run(_go())+ assert released_in_time == [True]+ assert request_store.get_run(claimed.id)["status"] == "succeeded"+++def test_a_claim_cancelled_before_it_runs_is_settled_off_the_loop(tmp_path: Path, monkeypatch) -> None:+ """#2649: the execution's done-callback is synchronous and runs on the loop. A claim cancelled before its+ coroutine ran is settled by an owned task on the runtime-work executor instead, so the loop stays free while+ another writer holds the lock, and the claim still goes back to the queue."""++ service, request_store, claimed = _sqlite_claimed_agent_run(tmp_path, monkeypatch)+ held, release = threading.Event(), threading.Event()+ released_in_time: list[bool] = []++ async def _go() -> None:+ holder = threading.Thread(target=_hold_state_write_lock, args=(held, release, released_in_time))+ holder.start()+ assert await asyncio.to_thread(held.wait, 5)+ service._spawn_execution(claimed, None)+ service._inflight_executions[claimed.id].cancel()+ await asyncio.sleep(0.1)+ # Until its settlement lands the claim stays live, so recovery cannot reclaim the+ # still-running row and have the settlement terminalize the new claim.+ assert claimed.id in service._live_claimed_run_ids+ release.set()+ for _ in range(100):+ if request_store.get_run(claimed.id)["status"] == "queued" and not service._cancel_settlements:+ break+ await asyncio.sleep(0.05)+ await asyncio.to_thread(holder.join, 5)++ asyncio.run(_go())+ assert released_in_time == [True]+ assert request_store.get_run(claimed.id)["status"] == "queued"+ assert claimed.id not in service._live_claimed_run_ids+++def test_a_mirror_read_never_waits_for_a_writer_and_never_sees_its_unwritten_edit(tmp_path: Path, monkeypatch) -> None:+ """#2649: writers hold the mirror lock across their store write, now on worker threads. The controller loop reads+ the mirror (``get_task``, ``list_tasks``, ``reconcile_jobs``), so a read takes no lock: it answers at once, from+ the published mirror, which a writer replaces only after its write lands."""+ from concurrent.futures import ThreadPoolExecutor++ store = ScheduledTaskStore(tmp_path / "scheduled_tasks.json")+ task = store.add_task(+ session_key="slack::channel::C123",+ prompt="run once",+ schedule_type="at",+ run_at="2030-01-01T00:00:00+00:00",+ timezone_name="UTC",+ )+ write_entered, release_write = threading.Event(), threading.Event()+ original_save = store._save++ def slow_save(*args) -> None:+ write_entered.set()+ assert release_write.wait(timeout=5)+ original_save(*args)++ monkeypatch.setattr(store, "_save", slow_save)+ with ThreadPoolExecutor(max_workers=2) as executor:+ stamping = executor.submit(store.set_enabled, task.id, False)+ assert write_entered.wait(timeout=1)+ try:+ read = executor.submit(lambda: (store.get_task(task.id), store.list_tasks()))+ current, listed = read.result(timeout=0.5)+ assert current.enabled is True and [item.enabled for item in listed] == [True]+ finally:+ release_write.set()+ stamping.result(timeout=5)+ assert store.get_task(task.id).enabled is False+++def test_a_reset_rebind_persists_its_whole_recovery_whatever_the_mirror_publishes_meanwhile(+ tmp_path: Path, monkeypatch+) -> None:+ """#2649: binding recovery now runs on a worker while the mirror stays published to the loop. A reset rebind's+ recorded choice (follow the rebound session, drop the stale Agent pin) is written with the binding by the store,+ so a reload that republishes the mirror between the recovery and its write cannot drop it, and the definition the+ loop reads is never edited before the database accepts it."""+ from core.vibe_agents import VibeAgentStore+ from storage.models import agent_sessions+ from storage.sessions_service import SQLiteSessionsService++ db_path = _binding_env(tmp_path, monkeypatch)+ agent_store = VibeAgentStore(db_path)+ try:+ agent_store.create(name="nightly", backend="claude", model="legacy-model")+ finally:+ agent_store.close()+ sessions = SQLiteSessionsService(db_path)+ try:+ pinned = sessions.bind_agent_session(+ scope_key="slack::channel::C123",+ agent_name="claude",+ session_anchor="slack_C123:definition_republish",+ native_session_id="native-republish",+ )+ finally:+ sessions.close()+ engine = create_sqlite_engine(db_path)+ with engine.begin() as conn:+ conn.execute(+ agent_sessions.update().where(agent_sessions.c.id == pinned).values(agent_name="nightly", model="legacy-model")+ )+ store = ScheduledTaskStore()+ task = store.add_task(+ name="digest",+ session_key="",+ session_id=pinned,+ session_policy="create_once",+ agent_name="nightly",+ prompt="send digest",+ schedule_type="cron",+ cron="0 * * * *",+ timezone_name="UTC",+ deliver_key="slack::channel::C123",+ metadata={"session_scope_id": "slack::channel::C123"},+ )+ sessions = SQLiteSessionsService(db_path)+ try:+ assert sessions.delete_agent_sessions(scope_key="slack::channel::C123", session_anchor_prefix="slack_C123")+ finally:+ sessions.close()+ agent_store = VibeAgentStore(db_path)+ try:+ assert agent_store.remove("nightly")+ finally:+ agent_store.close()++ store = ScheduledTaskStore()+ store.set_enabled(task.id, True)+ service = _dispatching_binding_service(tmp_path, store, db_path=db_path)+ published = store.get_task(task.id)+ rebind = store.rebind_session++ def rebind_after_a_republish(*args, **kwargs):+ # Another executor's reload lands between the recovery and its write.+ store.load()+ return rebind(*args, **kwargs)++ monkeypatch.setattr(store, "rebind_session", rebind_after_a_republish)+ asyncio.run(service._execute_task(published, execution_id="exec-1", disable_one_shot=False))++ rebound = ScheduledTaskStore().get_task(task.id)+ assert rebound.session_id and rebound.session_id != pinned+ assert rebound.agent_name is None+ assert rebound.metadata.get(BINDING_FOLLOWS_SESSION_METADATA_KEY) is True+ # The object the loop read before the fire was never edited in place.+ assert published.agent_name == "nightly" and published.session_id == pinned+++[email protected]("cancelled_during", ["_reserve_runtime_session", "_require_execution_agent_access"])+def test_a_cancellation_during_a_per_run_fires_store_steps_is_answered_at_its_dispatch(+ tmp_path: Path, monkeypatch, cancelled_during: str+) -> None:+ """#2649: a per-run fire's store steps run on the runtime-work door, but its cancellation points stay where the loop+ had them. A cancellation that arrives while the reservation or a pre-dispatch step runs is answered at the dispatch,+ which already holds the reserved session -- so a reservation is never stranded between steps, where nothing owns it+ and the orphan sweep cannot find it (a per-run reservation carries no definition stamp)."""++ store = ScheduledTaskStore(tmp_path / "scheduled_tasks.json")+ task = store.add_task(+ session_key="slack::channel::C123",+ session_policy="create_per_run",+ prompt="send digest",+ schedule_type="cron",+ cron="0 * * * *",+ timezone_name="UTC",+ deliver_key="slack::channel::C123",+ )+ dispatched: list[str] = []+ answered: list[str] = []++ async def submit_scheduled(session_id, _context, _prompt, **_kwargs):+ dispatched.append(session_id)+ try:+ await asyncio.sleep(5)+ except asyncio.CancelledError:+ answered.append(session_id)+ raise++ service = ScheduledTaskService(+ controller=SimpleNamespace(+ platform_settings_managers={"slack": object()},+ im_clients={},+ session_turn_gate=SimpleNamespace(submit_scheduled=submit_scheduled),+ ),+ store=store,+ request_store=TaskExecutionStore(tmp_path / "task_requests"),+ )+ entered, release = threading.Event(), threading.Event()+ released: list[str] = []++ def step(name: str, result):+ def run(*_args, **_kwargs):+ if name == cancelled_during:+ entered.set()+ assert release.wait(5)+ return result++ return run++ async def build_context(*_args, **_kwargs):+ return SimpleNamespace()++ monkeypatch.setattr(service, "_reserve_runtime_session", step("_reserve_runtime_session", "ses-reserved-1"))+ monkeypatch.setattr(+ scheduled_tasks,+ "resolve_session_id_target",+ step("resolve_session_id_target", SimpleNamespace(session_key=parse_session_key("slack::channel::C123"))),+ )+ monkeypatch.setattr(service, "_require_execution_agent_access", step("_require_execution_agent_access", None))+ monkeypatch.setattr(service, "_build_context", build_context)+ monkeypatch.setattr(+ service, "_release_reserved_session", lambda session_id, *, reason: released.append(session_id) or True+ )++ async def _go() -> None:+ fire = asyncio.create_task(service._execute_task(task, execution_id="exec-1", disable_one_shot=False))+ assert await asyncio.to_thread(entered.wait, 5)+ fire.cancel()+ await asyncio.sleep(0.05)+ release.set()+ with pytest.raises(asyncio.CancelledError):+ await fire++ asyncio.run(_go())+ assert dispatched == ["ses-reserved-1"], "the cancellation was answered before the dispatch adopted the reservation"+ assert answered == ["ses-reserved-1"]+ assert released == []+++def test_a_cancellation_during_a_binding_recovery_still_reports_the_change_it_committed(+ tmp_path: Path, monkeypatch+) -> None:+ """#2649: the pinned-binding recovery runs on the runtime-work door. A cancellation that arrives while it runs waits+ for it, and the change it committed still gets its result stamp, its recovery marker and its notice: the cancellation+ is answered at the next real await, as on the loop, not between the recovery and its bookkeeping."""++ _binding_env(tmp_path, monkeypatch)+ store = ScheduledTaskStore(tmp_path / "scheduled_tasks.json")+ task = store.add_task(+ session_key="",+ session_id="sesdoesnotexist",+ session_policy="existing",+ prompt="send digest",+ schedule_type="cron",+ cron="0 * * * *",+ timezone_name="UTC",+ deliver_key="slack::channel::C123",+ )+ service = _binding_service(tmp_path, store, [])+ notices = _spy_binding_notices(service)+ entered, release = threading.Event(), threading.Event()+ recover = service._recover_pinned_session_binding++ def slow_recovery(*args, **kwargs):+ entered.set()+ assert release.wait(5)+ return recover(*args, **kwargs)++ monkeypatch.setattr(service, "_recover_pinned_session_binding", slow_recovery)++ async def _go() -> None:+ fire = asyncio.create_task(service._execute_task(task, execution_id="exec-1", disable_one_shot=False))+ assert await asyncio.to_thread(entered.wait, 5)+ fire.cancel()+ await asyncio.sleep(0.05)+ release.set()+ with pytest.raises(asyncio.CancelledError):+ await fire++ asyncio.run(_go())+ assert [notice.action for notice in notices] == ["failing"], "the committed binding change was never reported"+ current = store.get_task(task.id)+ assert current is not None and "sesdoesnotexist" in (current.last_error or "")+ assert current.metadata.get(BINDING_RECOVERY_METADATA_KEY, {}).get("action") == "failing", (+ "the change's recovery marker was never recorded"+ )+++def test_an_older_mirror_capture_never_undoes_a_newer_one(tmp_path: Path) -> None:+ """#2649: ``reconcile_jobs`` captures the mirror on a thread, so captures may complete out of order. Each is ordered+ by the mirror generation it actually read -- not by when it was submitted -- so a capture submitted first but taken+ after a disable applies the disable, even when a later-submitted capture of the older mirror landed before it."""++ store = ScheduledTaskStore(tmp_path / "scheduled_tasks.json")+ task = store.add_task(+ session_key="slack::channel::C123",+ prompt="send digest",+ schedule_type="cron",+ cron="0 * * * *",+ timezone_name="UTC",+ )+ service = ScheduledTaskService(+ controller=SimpleNamespace(platform_settings_managers={"slack": object()}, im_clients={}),+ store=store,+ request_store=TaskExecutionStore(tmp_path / "task_requests"),+ )+ service.scheduler = _StubScheduler()+ capture = store.list_tasks_with_generation+ first_waits, first_entered = threading.Event(), threading.Event()+ calls: list[int] = []++ def ordered_capture():+ calls.append(len(calls))+ if len(calls) == 1:+ first_entered.set()+ assert first_waits.wait(5)+ return capture()++ store.list_tasks_with_generation = ordered_capture # type: ignore[method-assign]++ async def _go() -> None:+ first = asyncio.create_task(service._reconcile_jobs_off_loop())+ assert await asyncio.to_thread(first_entered.wait, 5)+ await service._reconcile_jobs_off_loop() # submitted second, captures the enabled mirror, lands first+ assert service.scheduler.get_job(task.id) is not None+ store.set_enabled(task.id, False)+ first_waits.set()+ await first # submitted first, captures the disabled mirror, lands last++ asyncio.run(_go())+ assert service.scheduler.get_job(task.id) is None, "an older mirror capture undid the disable a newer one applied"+++class _TaskWithoutCancelCount:+ """A running task as Python 3.10 presents it: ``cancel``, but no ``uncancel`` to count requests with."""++ def __init__(self, task: "asyncio.Task[Any]") -> None:+ self._task = task++ def cancel(self, *args) -> bool:+ return self._task.cancel(*args)+++ ("door", "step_raises", "python310"),+ [("_store_step", True, False), ("_run_runtime_sync", True, False), ("_store_step", False, True)],+)+def test_a_cancellation_caught_by_a_store_step_is_answered_at_the_next_await(+ tmp_path: Path, monkeypatch, door: str, step_raises: bool, python310: bool+) -> None:+ """#2649: a store step on the runtime-work door holds a cancellation that arrives while it runs until it lands, then+ hands it back for the caller's next await -- also when the step raised (its error must not swallow the cancellation+ and let the caller settle an ordinary failure), and on Python 3.10, whose tasks keep no cancellation count."""++ service = ScheduledTaskService(+ controller=SimpleNamespace(platform_settings_managers={}, im_clients={}),+ store=ScheduledTaskStore(tmp_path / "scheduled_tasks.json"),+ request_store=TaskExecutionStore(tmp_path / "task_requests"),+ )+ entered, release = threading.Event(), threading.Event()+ outcomes: list[str] = []++ def step() -> str:+ entered.set()+ assert release.wait(5)+ if step_raises:+ raise RuntimeError("step error")+ return "step result"++ if python310:+ current_task = asyncio.current_task+ monkeypatch.setattr(+ scheduled_tasks.asyncio, "current_task", lambda *args: _TaskWithoutCancelCount(current_task(*args))+ )++ async def caller() -> None:+ try:+ outcomes.append(await getattr(service, door)(step))+ except RuntimeError as exc:+ outcomes.append(str(exc))+ try:+ await asyncio.sleep(5)+ except asyncio.CancelledError:+ outcomes.append("cancelled at the next await")+ raise++ async def _go() -> None:+ run = asyncio.create_task(caller())+ assert await asyncio.to_thread(entered.wait, 5)+ run.cancel()+ await asyncio.sleep(0.05)+ release.set()+ with pytest.raises(asyncio.CancelledError):+ await run++ asyncio.run(_go())+ assert outcomes == ["step error" if step_raises else "step result", "cancelled at the next await"]+++def test_a_timeout_that_expires_during_a_store_step_still_reads_as_a_timeout(tmp_path: Path) -> None:+ """#2649: a cancellation a store step holds and re-arms is counted once, so an ``asyncio.timeout`` around the caller+ still turns its own expiry into ``TimeoutError`` instead of letting a bare cancellation escape the caller."""++ service = ScheduledTaskService(+ controller=SimpleNamespace(platform_settings_managers={}, im_clients={}),+ store=ScheduledTaskStore(tmp_path / "scheduled_tasks.json"),+ request_store=TaskExecutionStore(tmp_path / "task_requests"),+ )++ async def _go() -> None:+ with pytest.raises(TimeoutError):+ async with asyncio.timeout(0.05):+ await service._store_step(time.sleep, 0.2)+ await asyncio.sleep(5)++ asyncio.run(_go())