Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(runtime): commit off-loop activation owners outside the boundary, then confirm (#2655) (#2662)

Call White92b22ee30cd6
diff --git a/core/controller.py b/core/controller.py
index 6fa11fea3..1eeaba68b 100644
--- a/core/controller.py
+++ b/core/controller.py
@@ -2178,6 +2178,9 @@ class Controller:
try:
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
+ # This thread runs the loop: the one thread an activation commit may hold
+ # the runtime boundary across its I/O on.
+ self.runtime_activation.bind_loop_thread()
# Names whatever blocks this loop while it blocks; see core/loop_watchdog.py.
self._loop_watchdog = EventLoopWatchdog(self._loop)
self._loop_watchdog.start()
diff --git a/core/runtime_activation.py b/core/runtime_activation.py
index 823b6d685..1b20eb6c1 100644
--- a/core/runtime_activation.py
+++ b/core/runtime_activation.py
@@ -8,18 +8,28 @@ Backend lifecycle locks remain outermost. Callers attach, replace, or retire a
generation while holding the adapter's lifecycle lock, then use the short
synchronous boundary here for the final predicate or authoritative commit. No
cleanup await belongs inside this boundary.
+
+The controller loop takes the boundary synchronously, so off its thread the
+boundary is only ever held for constant-time in-memory sections: an owner commit
+made off the loop uses ``commit_then_confirm``, never a form that holds the
+boundary across the commit's I/O (see docs/plans/runtime-activation-boundary.md).
"""
from __future__ import annotations
+import contextlib
import itertools
+import logging
import threading
+import traceback
from collections import OrderedDict
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Callable, Generic, Iterator, TypeVar
+logger = logging.getLogger(__name__)
+
_T = TypeVar("_T")
_RETIRED_DIAGNOSTIC_LIMIT = 64
@@ -86,6 +96,13 @@ class RuntimeActivationRegistry:
tuple[str, str], RuntimeActivationIdentity
] = OrderedDict()
self._generations = itertools.count(1)
+ self._loop_thread: int | None = None
+
+ def bind_loop_thread(self) -> None:
+ """Record the calling thread as the controller loop's, where alone a commit may
+ hold the boundary across its I/O."""
+
+ self._loop_thread = threading.get_ident()
@staticmethod
def _target_key(backend: str, resource_key: str) -> tuple[str, str]:
@@ -180,8 +197,11 @@ class RuntimeActivationRegistry:
Generation validation and ``commit`` execute under the same target
boundary. A cleanup-first loser receives ``admitted=False`` and must use
its existing exact recovery path; ``commit`` is never called.
+
+ Controller loop thread only: off it, use ``commit_then_confirm``.
"""
+ self._report_off_loop_holder("commit_if_current")
key = self._target_key(identity.backend, identity.resource_key)
with identity._boundary.lock:
if not self._is_current_locked(key, identity):
@@ -200,12 +220,63 @@ class RuntimeActivationRegistry:
and release it only after commit. A false value means cleanup already
retired or replaced the observed generation; callers may persist a
queued/waiting owner, but must not promote it to starting or active.
+
+ Controller loop thread only: off it, use ``commit_then_confirm``.
"""
+ self._report_off_loop_holder("hold_if_current")
key = self._target_key(identity.backend, identity.resource_key)
with identity._boundary.lock:
yield self._is_current_locked(key, identity)
+ def commit_then_confirm(
+ self,
+ identity: RuntimeActivationIdentity,
+ commit: Callable[[], _T],
+ *,
+ compensate: Callable[[_T], object] | None,
+ ) -> RuntimeActivationCommit[_T]:
+ """Run one owner commit off the controller loop, never holding the boundary
+ across it.
+
+ The loop takes the boundary synchronously -- to retire a generation, or to
+ admit a Session Turn inside a SQLite transaction that already holds the
+ writer -- so a commit that held it across its own I/O would make the loop
+ wait for that I/O, or for SQLite's ``busy_timeout`` when both wait on each
+ other. Here the boundary is held only to admit and to confirm:
+
+ 1. admit: unless the generation is live, ``admitted=False`` and ``commit``
+ is never called;
+ 2. ``commit``, with no boundary held;
+ 3. confirm: if the generation is no longer live -- a retirement reserved or
+ finished meanwhile -- the commit lost, ``compensate(value)`` undoes it,
+ and the result is ``admitted=False``. Pass ``None`` only when the
+ caller's not-admitted path already undoes the commit.
+
+ A retirement never waits for the commit: one whose final snapshot runs
+ after the commit lands sees its owner, and one that does not has the commit
+ compensated. A compensation that fails is logged and the result is still
+ ``admitted=False``, so nothing proceeds on a retired generation; the
+ owner's recovery reclaims what the compensation left.
+ """
+
+ if not self.is_current(identity):
+ return RuntimeActivationCommit(admitted=False)
+ value = commit()
+ if self.is_current(identity):
+ return RuntimeActivationCommit(admitted=True, value=value)
+ if compensate is not None:
+ try:
+ compensate(value)
+ except Exception:
+ logger.exception(
+ "Failed to undo a commit that lost to the retirement of %s generation %s; "
+ "its owner's recovery reclaims it",
+ identity.backend,
+ identity.generation,
+ )
+ return RuntimeActivationCommit(admitted=False)
+
def reserve_retirement(
self,
identity: RuntimeActivationIdentity,
@@ -277,6 +348,24 @@ class RuntimeActivationRegistry:
return False
return self.finish_retirement(reservation, retire=True)
+ def _report_off_loop_holder(self, form: str) -> None:
+ loop_thread = self._loop_thread
+ if loop_thread is None or threading.get_ident() == loop_thread:
+ return
+ internal = {__file__, contextlib.__file__}
+ caller = next(
+ (frame for frame in reversed(traceback.extract_stack()[:-1]) if frame.filename not in internal),
+ None,
+ )
+ logger.error(
+ "%s held the runtime activation boundary on thread %r, off the controller loop, "
+ "which takes the boundary synchronously and would wait for this commit; "
+ "use commit_then_confirm. Caller: %s",
+ form,
+ threading.current_thread().name,
+ f"{caller.filename}:{caller.lineno} in {caller.name}" if caller else "unknown",
+ )
+
def tracked_target_count(self) -> int:
with self._index_lock:
return len(self._current)
diff --git a/core/scheduled_tasks.py b/core/scheduled_tasks.py
index c7176fd70..41e4c89a1 100644
--- a/core/scheduled_tasks.py
+++ b/core/scheduled_tasks.py
@@ -8354,9 +8354,12 @@ class ScheduledTaskService:
if registry is None or identity is None:
return self.request_store.claim(pending.id)
- committed = registry.commit_if_current(
+ # On the runtime-work door, off the loop: the claim never holds the boundary
+ # across its write. A claim that lost to a retirement goes back to the queue.
+ committed = registry.commit_then_confirm(
identity,
lambda: self.request_store.claim(pending.id),
+ compensate=lambda claimed: claimed is not None and self.request_store.requeue(claimed.id),
)
request = committed.value if committed.admitted else None
if request is not None:
@@ -8410,9 +8413,13 @@ class ScheduledTaskService:
activation_registry = self._runtime_activation_registry()
if activation_registry is not None and activation_identity is not None:
- committed = activation_registry.commit_if_current(
+ # Off the loop, like the claim. A start that lost to a retirement needs no
+ # compensation of its own: the requeue below gives the claim back and
+ # clears the marker with it.
+ committed = activation_registry.commit_then_confirm(
activation_identity,
commit_start,
+ compensate=None,
)
if not committed.admitted:
self.request_store.requeue(request.id)
diff --git a/docs/plans/runtime-activation-boundary.md b/docs/plans/runtime-activation-boundary.md
new file mode 100644
index 000000000..9ac2d65e5
--- /dev/null
+++ b/docs/plans/runtime-activation-boundary.md
@@ -0,0 +1,176 @@
+# Runtime activation boundary off the controller loop (#2655)
+
+Status: accepted (option 3), implemented in the PR that adds this note.
+
+## Background
+
+`RuntimeActivationRegistry` (`core/runtime_activation.py`) gives every runtime generation an identity. Each identity has a boundary: a `threading.RLock`, a phase (`live`, `retiring`, `retired`) and a reservation serial. All of it is process memory; nothing is persisted.
+
+The boundary is used in two ways.
+
+- **Short sections.** These touch memory only:
+ - `attach`, `current` and `is_current`;
+ - `reserve_retirement` and `finish_retirement`;
+ - `retire_if_current`, which runs its predicate between the reserve and the finish, without the lock.
+- **Commits under the lock.** `commit_if_current(identity, commit)` runs `commit()` while holding the lock, and `hold_if_current` keeps the lock for the caller's whole `with` body. That body is usually a SQLite transaction.
+
+This design exists so that an owner commit for a generation cannot race that generation's retirement. A commit either lands before retirement takes its final ownership snapshot, so the snapshot sees it, or it is refused.
+
+## Problem
+
+Commits under the lock happen on two threads.
+
+**Off the loop.** Only two holders keep the boundary across SQLite I/O on a worker thread. Both are reached through the scheduler's runtime-work door.
+
+- **W1, the scheduler claim.** `_claim_pending_request` → `request_store.claim` → `claim_pending_run`. It runs `BEGIN IMMEDIATE`, then a read, then an update from queued to running.
+- **W2, the execution start.** `_start_claimed_execution` → `mark_execution_started`. It sets the pid marker where the run is running and `pid IS NULL`.
+
+**On the loop.** Every other holder that does I/O under the lock runs on the loop thread, and none of them awaits inside the lock:
+
+- the Session Turn admissions in `core/session_turns.py`, including `drain_delivery_queue`'s in-transaction hold;
+- native start and agent-initiated turns in `modules/agents/service.py`;
+- SessionActivity writes in `core/session_activities.py`.
+
+The loop also takes the boundary for short sections:
+
+- retirement:
+ - Codex `_stop_runtime`;
+ - OpenCode `_on_generation_stopping`;
+ - Claude cleanup, idle eviction and native migration;
+- `is_current` before every Claude native write;
+- `current` and `attach`.
+
+Because the lock is an `RLock` and the loop never awaits inside it, loop-side holders never make the loop wait on each other. The loop waits on the boundary only while W1 or W2 holds it. That produces two failures.
+
+1. **Retirement waits for a contended commit.** A retirement on the loop that reaches the boundary while W1 or W2 is committing under SQLite contention waits for that commit, up to the 5 s `busy_timeout`. This is the failure #2655 describes.
+
+2. **A lock-order inversion stalls the loop, with no retirement involved.** The two sides take the same two locks in opposite orders:
+ - W1 and W2 take the boundary, then the SQLite writer.
+ - `drain_delivery_queue` on the loop takes the writer (`reserve_write_lock`), then the boundary (`_claim_fifo_batch_in_transaction` → `owner_factory` → `hold_if_current`).
+
+ When both resolve to the same identity, the loop blocks on the `RLock`, which has no timeout. It stays blocked until the worker's `busy_timeout` expires and the worker gives up its claim.
+
+ This is common in practice. OpenCode resolves every unbound request to its current runtime, and Codex shares one identity per working-directory generation.
+
+ A probe used the real registry and two SQLite connections ordered as above. The loop thread blocked for 5.35 s, and the worker's claim failed with `database is locked`. The scheduler claim has run on the worker since #1173, so this happens on master today and in released versions.
+
+## Goal
+
+The controller loop never waits on a lock that another thread holds across blocking I/O.
+
+Equivalently: **off the loop, the boundary is held only for constant-time in-memory sections.** It may be held across I/O only on the loop thread. There, it is subject to the general controller-loop rule like any other SQLite work on the loop.
+
+## Options
+
+### 1. Persisted generation CAS (from #2655)
+
+Commits would carry a persisted generation, and retirement would bump it transactionally. Rejected, for three reasons.
+
+- **The generation is process-local** (`itertools.count(1)`) and repeats after a restart, so a persisted key needs a restart-unique part and a schema change.
+- **The bump at retirement is itself a contended SQLite write.**
+ - On the loop, it brings back SQLite-on-loop.
+ - Off the loop, it puts an await before the fence lands. That breaks Codex's rule that "the retirement is reserved before anything awaits", unless the in-memory phase flip stays synchronous as well — and then the CAS adds nothing for in-memory owners.
+- **Several loop-side owners commit memory as well as SQLite under the boundary**, and a database CAS cannot fence memory:
+ - native start mutates the gate and creates loop tasks;
+ - the agent-initiated turn must not yield;
+ - activities update the identity maps that retirement predicates read.
+
+### 2. Never take the boundary on the loop
+
+This would move every loop-side acquisition to a worker:
+- retirement;
+- the `is_current` check before each Claude write;
+- `current` and `attach`;
+- the Session Turn holds inside `drain_delivery_queue`'s transaction;
+- native start, the agent-initiated turn and activities.
+
+Several of these are tied to the loop by construction: `get_running_loop().create_task` runs inside the commit, the receiver's activity API is synchronous, and `begin_agent_initiated_turn` is documented as non-yielding. Rejected: the change is very wide, and the risk is high.
+
+### 3. Off the loop, commit outside the boundary and confirm afterwards (recommended)
+
+Change only the two off-loop holders, W1 and W2. Each one becomes:
+
+1. **Admit.** Under the boundary, refuse unless the identity is live. This is unchanged: a retirement already reserved refuses the commit before it runs.
+2. **Commit.** Run the SQLite write with no boundary held.
+3. **Confirm.** Under the boundary, check that the identity is still current and live.
+ - If it is, the commit stands.
+ - If it is not, the commit lost to a retirement. Compensate with the existing `requeue`, which moves the run from running back to queued and clears `pid`. Then report "not admitted", so the execution is skipped exactly as it is today.
+
+The registry owns this rule as one new method, `commit_then_confirm(identity, commit, *, compensate)`. `compensate(value)` undoes a commit that lost. A caller passes `None` only when its own not-admitted path already undoes the commit: W2's existing `requeue` gives the claim back and clears the marker in the same write.
+
+`commit_if_current` and `hold_if_current` keep their meaning but are now loop-only, because their commit holds the boundary. The controller binds its loop thread to the registry (`bind_loop_thread`). Either method called on any other thread logs an error that names its caller, so a new off-loop holder cannot bring the stall back unnoticed.
+
+**Why this is correct**, case by case:
+
+- **The retirement reserves before admission.** The commit is refused before it runs. Unchanged.
+- **The commit lands before the retirement's final snapshot.**
+ - Codex `_generation_drained` and Claude idle eviction read durable ownership through `RuntimeOwnershipProvider`. If that predicate counts the run as an owner, the retirement declines, as it does today.
+ - The confirm step then sees `live`, and the run proceeds. If the reservation is still open when the confirm step runs, it sees `retiring` and compensates, even though the retirement is about to abort. That compensated retry is a valid outcome: the run goes back to the queue and is claimed again once the generation is live.
+ - Only a confirm step that runs before the reservation, or after the abort, guarantees that this attempt proceeds.
+ - If the predicate does not count the run, the retirement completes, and the next case applies.
+ - Correctness therefore does not depend on what any predicate sees.
+- **The commit lands after the snapshot, or the predicate is in memory only** (OpenCode `lambda: True`, Claude cleanup). The retirement completes, and the confirm step sees the identity retired and compensates.
+ - Today the retirement would have waited for the commit and retired anyway, leaving a claimed run whose start W2 then refuses.
+ - Both designs end with the run requeued and nothing executing on the retired generation; the new one gets there a step earlier.
+- **A retirement is reserved and then aborted while the commit is in flight.** The confirm step either compensates when it did not need to (safe: the run is reclaimed later) or sees `live`.
+
+**What changes:**
+- A retirement no longer waits for an in-flight W1 or W2 commit.
+- Two tests pinned the wait, and both are restated as outcome assertions: the cleanup no longer waits, and nothing starts on a generation retired before its claim or start was confirmed.
+ - The pid-marker test (`test_hfr_137_fallback_pid_marker_first_blocks_retirement_until_commit`) becomes `test_hfr_137_pid_marker_that_loses_to_cleanup_during_its_write_is_requeued`.
+ - The claim-first test becomes `test_hfr_137_claim_that_loses_to_cleanup_during_its_write_is_given_back`. The claim used to be admitted and then requeued by the start; now it is given back by its own confirm step, one step earlier.
+ - The cleanup-first test is unchanged.
+- A run may be `running` for a moment before it is compensated. Graceful retirement treats that as an owner and declines, which is the conservative choice.
+
+**If the compensating write fails**, the step still reports "not admitted", so nothing executes on the retired generation. What happens to the run afterwards differs:
+- **W1.** The registry logs the failure. The claim is left running without a pid and is not a live claim, so the claimed-run recovery requeues it, and the run is retried.
+- **W2.** The requeue on the not-admitted path raises from the start step, so the execution never begins. But the marker it failed to clear stays on the row. Startup recovery (`recover_processing_runs`) and the stale-run sweep read a marker as an execution that may have begun, so they settle the run as interrupted, with its failure notice, instead of retrying it. On master the same double failure leaves no marker, and the run is retried.
+ - The marker is the commit itself, so no durable rollback exists short of a persisted generation, which is the rejected option 1.
+ - This is a deliberate trade-off: removing the loop stall costs a visible interrupted run in a doubly rare case. A retirement must land between W2's write and its confirm step, and the write that follows must then fail. Nothing ever runs on the retired generation, and nothing is lost silently.
+
+**What `requeue` does**, since it is the compensation (`mark_run_queued_from_running`). It is a status-guarded update, running to queued, that:
+- clears `started_at` and `pid`, bumps `updated_at`, and leaves the metadata untouched;
+- publishes one `runs.updated` event, which wakes the request, callback, failure-notice and stale-run lanes and refreshes the UI.
+
+It bumps no attempt or retry counter, sends no notification and touches no Delivery. Failure notices are driven by `owed_failure_notice` metadata, which it does not write. So a compensation does not count as a failed attempt. It is the same write and event the existing not-admitted path of W2 already makes.
+
+**What it removes:**
+- Failure 1: off the loop, the boundary is held only for constant-time sections, so a retirement on the loop never waits on I/O.
+- Failure 2: W1 and W2 no longer hold the boundary while they wait for the writer, so the inversion cannot form.
+
+**What it leaves (not in scope here):**
+- **Loop-side holders still do SQLite I/O under the boundary.** They cannot make the loop wait on another thread, but they are SQLite-on-loop under the controller-loop rule. That belongs to the #2630 family.
+- **W1 and W2 resolve their identity on the worker by reading loop-owned adapter state.** For example, Claude iterates `claude_sessions.values()`. A concurrent mutation can raise `dictionary changed size during iteration`. `AgentService` catches the error and fails closed: the resolution is treated as inconclusive, and the claim stays queued and is retried. That is correct but noisy. Moving the in-memory part of the resolution to the loop is an optional follow-up.
+
+## Invariants each caller keeps
+
+| Caller | Thread | Invariant | After this change |
+| --- | --- | --- | --- |
+| W1 scheduler claim | worker | A queued run does not become running against a retiring or retired generation | Admit, commit, confirm; `compensate` requeues the claim |
+| W2 execution start | worker | No execution starts on a generation retired before its start was confirmed | Admit, commit, confirm; the existing not-admitted `requeue` clears the marker (a failed clear settles the run as interrupted, see above) |
+| Session Turn admissions | loop | A non-admitted owner is persisted as queued, never promoted | Unchanged: `hold_if_current` |
+| Native start / agent-initiated turn | loop | A retired generation does not bind the gate or the durable Turn | Unchanged: `commit_if_current` |
+| SessionActivity writes | loop | A stale generation does not rebuild or complete an owner | Unchanged: `commit_if_current` |
+| Codex retirement | loop | Reserved before anything awaits; drained check made inside the reservation; aborts on failure | Unchanged |
+| Claude cleanup / idle eviction / migration | loop | Durable Turn and Activity ownership vetoes reclamation | Unchanged |
+| OpenCode retirement | loop | No fenced commit is lost when an unbound generation retires | Unchanged |
+
+## Test design
+
+1. **Inversion.** A worker in W1 waits for the SQLite writer, which a transaction on the loop thread holds before that transaction takes the boundary (`drain_delivery_queue`'s order). The loop thread must acquire the boundary at once (under 50 ms). The claim then commits after the loop's transaction, or is retried. Neither side waits 5 s.
+2. **Retirement during a commit.** W1's or W2's write is held in flight. Meanwhile a cleanup runs `retire_if_current`, which must return at once. When the write lands, the confirm step compensates: the run is queued and nothing starts. What holds the write does not matter, because no boundary is held during it; the tests hold it with an event.
+3. **Admission first, graceful retirement.** The claim is confirmed before the retirement reserves. A retirement whose snapshot sees the owner declines, and the run proceeds (unchanged behavior). If the reservation overlaps the confirm step instead, the result is a compensated retry, as above. The test orders the two explicitly so that it is not scheduling-dependent.
+4. **Compensation fails.** The compensating `requeue` raises. Each test asserts the run's eventual state on a real SQLite store:
+ - W1: the claim is refused and the failure logged, and the claimed-run recovery puts the run back in the queue.
+ - W2: the execution never starts, and recovery settles the run as interrupted.
+5. **Off-loop guard.** With the loop thread bound, `commit_if_current` and `hold_if_current` called from a worker thread each log an error that names the caller; on the loop thread they log nothing.
+
+On master, test 1 fails for the intended reason: the loop waits for the claim's busy timeout, 2 s in the test. The two restated tests in design 2 fail there because the cleanup waits for the write. Test 4 fails because the lost claim is admitted.
+
+## Plan
+
+- [x] Registry: add `commit_then_confirm`, and document `commit_if_current` and `hold_if_current` as loop-only.
+- [x] Scheduler: move W1 (`_claim_pending_request`) and W2 (`_start_claimed_execution`) to it, with `requeue` as the compensation.
+- [x] Tests: restate the pinned wait-order tests as outcome tests, and add test designs 1–5.
+- [x] Guard: the controller binds its loop thread, and `commit_if_current` or `hold_if_current` off that thread logs an error naming the caller.
+- [x] Follow-up issue (#2664): W1 and W2 resolve identity from loop-owned adapter state on the worker. A concurrent mutation surfaces as `dictionary changed size during iteration`; it fails closed and is retried, but it is noisy.
diff --git a/tests/scenarios/harness_failure_recovery/catalog.yaml b/tests/scenarios/harness_failure_recovery/catalog.yaml
index 2382a44ea..be1a809e9 100644
--- a/tests/scenarios/harness_failure_recovery/catalog.yaml
+++ b/tests/scenarios/harness_failure_recovery/catalog.yaml
@@ -2420,7 +2420,7 @@ scenarios:
kind: concurrency
layer: contract
backend: shared
- test: tests/test_runtime_activation.py::test_hfr_137_claim_first_requeues_when_cleanup_wins_before_pid
+ test: tests/test_runtime_activation.py::test_hfr_137_claim_that_loses_to_cleanup_during_its_write_is_given_back
- id: HFR-138
name: ownership handoffs stay visible in one SQLite generation
status: covered
diff --git a/tests/test_runtime_activation.py b/tests/test_runtime_activation.py
index 4be19cde1..93c79d315 100644
--- a/tests/test_runtime_activation.py
+++ b/tests/test_runtime_activation.py
@@ -1,7 +1,11 @@
from __future__ import annotations
import asyncio
+import logging
+import sqlite3
import threading
+import time
+from contextlib import closing
from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock
@@ -13,7 +17,8 @@ from core.runtime_activation import (
RuntimeActivationRegistry,
RuntimeActivationResolution,
)
-from core.scheduled_tasks import ScheduledTaskService, TaskExecutionRequest
+from core.runtime_recovery import FallbackRequestRecoveryHandler
+from core.scheduled_tasks import ScheduledTaskService, TaskExecutionRequest, TaskExecutionStore
from core.session_activities import SessionActivityRegistry
from modules.agents.base import AGENT_RUNTIME_TURN_KEY, AGENT_RUNTIME_TURN_TOKEN
from modules.agents.claude_agent import ClaudeAgent
@@ -929,14 +934,15 @@ def test_hfr_137_cleanup_first_rejects_fallback_claim() -> None:
assert claim_calls == []
-def test_hfr_137_claim_first_requeues_when_cleanup_wins_before_pid() -> None:
- """HFR-137: a pre-PID claim survives cleanup through exact Run requeue."""
+def test_hfr_137_claim_that_loses_to_cleanup_during_its_write_is_given_back() -> None:
+ """HFR-137, #2655: a claim writes with no boundary held, so a cleanup that starts meanwhile retires the generation
+ at once instead of waiting for the write. The claim, confirmed after it lands, finds its generation retired and is
+ given back to the queue, so nothing ever starts on the retired generation."""
registry = RuntimeActivationRegistry()
identity = registry.attach("claude", "runtime-1")
claim_entered = threading.Event()
allow_claim = threading.Event()
- retirement_started = threading.Event()
pending = TaskExecutionRequest(
id="run-1",
request_type="agent_run",
@@ -947,7 +953,6 @@ def test_hfr_137_claim_first_requeues_when_cleanup_wins_before_pid() -> None:
request_type=pending.request_type,
agent_backend=pending.agent_backend,
)
- mark_calls: list[str] = []
requeued: list[str] = []
def claim(_request_id: str) -> TaskExecutionRequest:
@@ -957,8 +962,6 @@ def test_hfr_137_claim_first_requeues_when_cleanup_wins_before_pid() -> None:
request_store = SimpleNamespace(
claim=claim,
- refresh_claimed_request=lambda request: request,
- mark_execution_started=lambda request_id: mark_calls.append(request_id) or True,
requeue=lambda request_id: requeued.append(request_id),
)
service = _fallback_service(registry, request_store, identity)
@@ -968,32 +971,19 @@ def test_hfr_137_claim_first_requeues_when_cleanup_wins_before_pid() -> None:
claim_thread = threading.Thread(
target=lambda: claims.append(service._claim_pending_request(pending))
)
-
- def retire() -> None:
- retirement_started.set()
- retired.append(registry.retire_if_current(identity, lambda: True))
-
- retirement_thread = threading.Thread(target=retire)
+ retirement_thread = threading.Thread(
+ target=lambda: retired.append(registry.retire_if_current(identity, lambda: True))
+ )
claim_thread.start()
assert claim_entered.wait(timeout=2)
retirement_thread.start()
- assert retirement_started.wait(timeout=2)
- allow_claim.set()
-
- _join(claim_thread)
+ # The cleanup does not wait for the claim's write.
_join(retirement_thread)
-
- assert claims == [claimed_request]
- assert claimed_request.observed_activation_identity == identity
assert retired == [True]
+ allow_claim.set()
+ _join(claim_thread)
- request, started = asyncio.run(
- service._mark_execution_started_for_claimed_request(claimed_request)
- )
-
- assert request is claimed_request
- assert started is False
- assert mark_calls == []
+ assert claims == [None]
assert requeued == [pending.id]
@@ -1055,33 +1045,32 @@ def test_hfr_137_backend_drain_requeues_existing_pre_pid_claim() -> None:
assert requeued == [request.id]
-def test_hfr_137_fallback_pid_marker_first_blocks_retirement_until_commit() -> None:
- """HFR-137: PID admission completes before a competing cleanup can retire."""
+def test_hfr_137_pid_marker_that_loses_to_cleanup_during_its_write_is_requeued() -> None:
+ """HFR-137, #2655: the start marker writes with no boundary held, so a cleanup that starts meanwhile retires the
+ generation at once instead of waiting for the write. The start, confirmed after the marker lands, finds its
+ generation retired: the execution does not start, and the claim goes back to the queue, marker cleared."""
registry = RuntimeActivationRegistry()
identity = registry.attach("claude", "runtime-1")
marker_entered = threading.Event()
allow_marker = threading.Event()
- retirement_started = threading.Event()
- order: list[str] = []
request = TaskExecutionRequest(
id="run-1",
request_type="agent_run",
agent_backend="claude",
observed_activation_identity=identity,
)
+ requeued: list[str] = []
def mark_execution_started(_request_id: str) -> bool:
- order.append("marker-entered")
marker_entered.set()
assert allow_marker.wait(timeout=2)
- order.append("marker-finished")
return True
request_store = SimpleNamespace(
refresh_claimed_request=lambda current: current,
mark_execution_started=mark_execution_started,
- requeue=lambda _request_id: pytest.fail("admitted marker was requeued"),
+ requeue=lambda request_id: requeued.append(request_id),
)
service = _fallback_service(registry, request_store, identity)
marked: list[tuple[TaskExecutionRequest, bool]] = []
@@ -1092,26 +1081,209 @@ def test_hfr_137_fallback_pid_marker_first_blocks_retirement_until_commit() -> N
asyncio.run(service._mark_execution_started_for_claimed_request(request))
)
)
-
- def retire() -> None:
- retirement_started.set()
-
- def final_predicate() -> bool:
- order.append("retire-predicate")
- return True
-
- retired.append(registry.retire_if_current(identity, final_predicate))
-
- retirement_thread = threading.Thread(target=retire)
+ retirement_thread = threading.Thread(
+ target=lambda: retired.append(registry.retire_if_current(identity, lambda: True))
+ )
marker_thread.start()
assert marker_entered.wait(timeout=2)
retirement_thread.start()
- assert retirement_started.wait(timeout=2)
+ # The cleanup does not wait for the marker's write.
+ _join(retirement_thread)
+ assert retired == [True]
allow_marker.set()
-
_join(marker_thread)
- _join(retirement_thread)
- assert marked == [(request, True)]
- assert retired == [True]
- assert order == ["marker-entered", "marker-finished", "retire-predicate"]
+ assert marked == [(request, False)]
+ assert requeued == [request.id]
+
+
+def test_a_claim_waiting_for_the_sqlite_writer_never_holds_the_boundary_the_loop_needs(tmp_path) -> None:
+ """#2655: the scheduler claims off the loop, and its claim waits for SQLite's writer. The loop admits Session Turns
+ inside a transaction that already holds the writer, and only then takes the activation boundary. A claim that held
+ the boundary while it waited for the writer would make the loop wait for SQLite's busy timeout. It never holds it
+ there, so the loop takes the boundary at once, and the claim lands when the loop's transaction ends."""
+
+ db_path = tmp_path / "state.sqlite"
+ with closing(sqlite3.connect(db_path)) as setup:
+ setup.execute("CREATE TABLE claims (run_id TEXT)")
+ setup.commit()
+ registry = RuntimeActivationRegistry()
+ identity = registry.attach("claude", "runtime-1")
+ pending = TaskExecutionRequest(id="run-1", request_type="agent_run", agent_backend="claude")
+ claim_writing = threading.Event()
+
+ def claim(request_id: str) -> TaskExecutionRequest:
+ with closing(sqlite3.connect(db_path, timeout=2)) as conn:
+ claim_writing.set()
+ conn.execute("BEGIN IMMEDIATE")
+ conn.execute("INSERT INTO claims VALUES (?)", (request_id,))
+ conn.commit()
+ return pending
+
+ request_store = SimpleNamespace(
+ claim=claim,
+ requeue=lambda _request_id: pytest.fail("a claim made on a live generation was given back"),
+ )
+ service = _fallback_service(registry, request_store, identity)
+ claims: list[TaskExecutionRequest | None] = []
+ errors: list[BaseException] = []
+
+ def run_claim() -> None:
+ try:
+ claims.append(service._claim_pending_request(pending))
+ except Exception as exc:
+ errors.append(exc)
+
+ with closing(sqlite3.connect(db_path, timeout=2)) as loop_transaction:
+ loop_transaction.execute("BEGIN IMMEDIATE")
+ claim_thread = threading.Thread(target=run_claim)
+ claim_thread.start()
+ assert claim_writing.wait(timeout=2)
+ time.sleep(0.1) # the claim now waits for the writer this transaction holds
+ started = time.monotonic()
+ with registry.hold_if_current(identity) as admitted:
+ waited = time.monotonic() - started
+ loop_transaction.commit()
+ _join(claim_thread)
+
+ assert waited < 0.05, f"the loop waited {waited:.2f}s on a claim that held the boundary while waiting for SQLite"
+ assert admitted is True
+ assert errors == []
+ assert claims == [pending]
+
+
+def test_a_claim_confirmed_before_a_graceful_cleanup_keeps_its_generation() -> None:
+ """#2655: a claim confirmed while its generation was live stands. A graceful cleanup whose final snapshot sees the
+ claimed Run as an owner declines, and the Run then starts on that generation."""
+
+ registry = RuntimeActivationRegistry()
+ identity = registry.attach("claude", "runtime-1")
+ pending = TaskExecutionRequest(id="run-1", request_type="agent_run", agent_backend="claude")
+ owners: set[str] = set()
+ request_store = SimpleNamespace(
+ claim=lambda request_id: owners.add(request_id) or pending,
+ refresh_claimed_request=lambda current: current,
+ mark_execution_started=lambda _request_id: True,
+ requeue=lambda _request_id: pytest.fail("a confirmed claim was given back"),
+ )
+ service = _fallback_service(registry, request_store, identity)
+
+ claimed = service._claim_pending_request(pending)
+
+ assert claimed is pending
+ assert registry.retire_if_current(identity, lambda: not owners) is False
+ assert asyncio.run(service._mark_execution_started_for_claimed_request(claimed)) == (claimed, True)
+
+
+def _sqlite_request_store(tmp_path, monkeypatch) -> tuple[TaskExecutionStore, TaskExecutionRequest]:
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ request_store = TaskExecutionStore()
+ queued = request_store.enqueue_agent_run(message="run", agent_backend="claude", session_id="ses-runtime")
+ return request_store, queued
+
+
+def _failing_requeue(_request_id: str) -> None:
+ raise sqlite3.OperationalError("database is locked")
+
+
+def test_a_claim_whose_give_back_fails_starts_nothing_and_is_requeued_by_recovery(
+ tmp_path, monkeypatch, caplog
+) -> None:
+ """#2655: a claim that lost to a cleanup is given back to the queue. When that write fails too, the claim is still
+ refused, so nothing starts on the retired generation, and the failure is logged. The claim it left has no start
+ marker, so the claimed-run recovery puts the Run back in the queue, to run again."""
+
+ request_store, queued = _sqlite_request_store(tmp_path, monkeypatch)
+ registry = RuntimeActivationRegistry()
+ identity = registry.attach("claude", "runtime-1")
+
+ def claim_while_cleanup_lands(request_id: str) -> TaskExecutionRequest | None:
+ assert registry.retire_if_current(identity, lambda: True)
+ return request_store.claim(request_id)
+
+ service = _fallback_service(
+ registry,
+ SimpleNamespace(claim=claim_while_cleanup_lands, requeue=_failing_requeue),
+ identity,
+ )
+
+ with caplog.at_level(logging.ERROR, logger="core.runtime_activation"):
+ assert service._claim_pending_request(queued) is None
+ assert "lost to the retirement of claude" in caplog.text
+ left = request_store.get_run(queued.id)
+ assert left["status"] == "running" and left["pid"] is None
+
+ recovery = FallbackRequestRecoveryHandler(request_store.sqlite_backend)
+ items, _has_more = recovery.scan(limit=10, occupied=frozenset())
+ assert [recovery.process_sync(item) for item in items] == [True]
+ assert request_store.get_run(queued.id)["status"] == "queued"
+
+
+def test_a_start_whose_give_back_fails_never_runs_and_settles_as_interrupted(tmp_path, monkeypatch) -> None:
+ """#2655: a start marker that lost to a cleanup is cleared by giving the claim back. When that write fails too, the
+ execution never starts. The marker it left reads, to recovery, as an execution that may have begun, so recovery
+ settles the Run as interrupted, with its failure notice, rather than running it again: the cost of a failed write
+ racing a retirement, which only a persisted generation could undo (see the design note)."""
+
+ request_store, queued = _sqlite_request_store(tmp_path, monkeypatch)
+ registry = RuntimeActivationRegistry()
+ identity = registry.attach("claude", "runtime-1")
+ claimed = request_store.claim(queued.id)
+ assert claimed is not None
+ claimed.observed_activation_identity = identity
+
+ def mark_while_cleanup_lands(request_id: str) -> bool:
+ assert registry.retire_if_current(identity, lambda: True)
+ return request_store.mark_execution_started(request_id)
+
+ service = _fallback_service(
+ registry,
+ SimpleNamespace(
+ refresh_claimed_request=lambda current: current,
+ mark_execution_started=mark_while_cleanup_lands,
+ requeue=_failing_requeue,
+ ),
+ identity,
+ )
+
+ with pytest.raises(sqlite3.OperationalError):
+ asyncio.run(service._mark_execution_started_for_claimed_request(claimed))
+ assert request_store.get_run(queued.id)["pid"] is not None
+
+ request_store.recover_processing()
+ settled = request_store.get_run(queued.id)
+ assert settled["status"] == "failed" and settled["pid"] is None
+
+
+def test_a_boundary_held_off_the_loop_thread_is_reported_with_its_caller(caplog) -> None:
+ """#2655: off the controller loop thread, ``commit_if_current`` and ``hold_if_current`` would hold the boundary
+ across their commit's I/O while the loop takes it synchronously. Such a call is logged as an error naming its
+ caller, so a new off-loop holder cannot bring the loop stall back unnoticed."""
+
+ registry = RuntimeActivationRegistry()
+ identity = registry.attach("claude", "runtime-1")
+ registry.bind_loop_thread() # this thread plays the controller loop
+
+ def holder_on_a_worker_commits() -> None:
+ registry.commit_if_current(identity, lambda: None)
+
+ def holder_on_a_worker_holds() -> None:
+ with registry.hold_if_current(identity):
+ pass
+
+ with caplog.at_level(logging.ERROR, logger="core.runtime_activation"):
+ registry.commit_if_current(identity, lambda: None)
+ with registry.hold_if_current(identity):
+ pass
+ assert not caplog.records, "a holder on the loop thread was reported"
+ for target in (holder_on_a_worker_commits, holder_on_a_worker_holds):
+ worker = threading.Thread(target=target)
+ worker.start()
+ _join(worker)
+
+ reports = [record.getMessage() for record in caplog.records if record.levelno == logging.ERROR]
+ assert len(reports) == 2
+ assert "commit_if_current" in reports[0] and "in holder_on_a_worker_commits" in reports[0]
+ assert "hold_if_current" in reports[1] and "in holder_on_a_worker_holds" in reports[1]
+
+
diff --git a/tests/test_service_readiness.py b/tests/test_service_readiness.py
index 795d273ae..c07a6eff3 100644
--- a/tests/test_service_readiness.py
+++ b/tests/test_service_readiness.py
@@ -26,6 +26,7 @@ import pytest
from core import controller as controller_module
from core import internal_server
from core.controller import Controller
+from core.runtime_activation import RuntimeActivationRegistry
REPO_ROOT = Path(__file__).resolve().parents[1]
#: Everything shipped to a user's machine. `tests/` is excluded deliberately:
@@ -99,6 +100,8 @@ def startup(monkeypatch):
controller.config = SimpleNamespace()
controller.enabled_platforms = ["slack"]
controller.show_git_checkpoint_service = SimpleNamespace(start=lambda: events.append("show checkpoints"))
+ # `run()` binds the loop thread it starts to the activation registry itself.
+ controller.runtime_activation = RuntimeActivationRegistry()
controller._shutdown_requested = False
controller._im_run_exception = None
controller.cleanup_sync = Mock()