Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(session-turns): leave a head held for explicit retry to its owner (#2781)

Call Whited8bd182d776e
diff --git a/core/runtime_recovery.py b/core/runtime_recovery.py
index 17e28f0a0..3c89d650b 100644
--- a/core/runtime_recovery.py
+++ b/core/runtime_recovery.py
@@ -29,10 +29,24 @@ class SessionDeliveryRecoveryHandler:
occupied=occupied,
cursor=cursor,
)
- return [
- RuntimeWorkItem(observation.session_id, observation)
+ items = [
+ # A head held for its owner's explicit retry only moves the cursor:
+ # re-entering it would find nothing to start, back off and rescan
+ # it every second until that retry.
+ RuntimeWorkItem(
+ observation.session_id,
+ observation,
+ cursor_only=observation.awaits_explicit_retry,
+ )
for observation in observations
- ], has_more
+ ]
+ if not has_more:
+ # On the last page a held head must not leave the cursor behind it:
+ # no worker re-arms the lane for it, so the next wake would scan
+ # only the Sessions after it and miss earlier ones.
+ while items and items[-1].cursor_only:
+ items.pop()
+ return items, has_more
async def process(self, item: RuntimeWorkItem) -> bool:
return await self.manager.recover_runtime_delivery_observation(
diff --git a/core/session_turns.py b/core/session_turns.py
index 206978f0e..bf94efc72 100644
--- a/core/session_turns.py
+++ b/core/session_turns.py
@@ -265,6 +265,8 @@ class RuntimeDeliveryObservation:
predecessor_terminal_evidence_kind: str | None = None
predecessor_terminal_evidence_json: str | None = None
predecessor_terminal_at: str | None = None
+ # An open head held for its owner's explicit retry: no recovery can start it.
+ awaits_explicit_retry: bool = False
def _start_replay_count(deliveries: list[dict[str, Any]]) -> int:
@@ -4829,10 +4831,14 @@ class SessionTurnManager:
with self._sqlite_engine().connect() as conn:
# Most drains find nothing to claim -- every Turn end asks. Those answer
# from a read, as the claim below would, without taking the writer (#2706).
- claimable = (
- delivery_store.active_turn(conn, session_id) is None
- and delivery_store.claimable_fifo_head(conn, session_id) is not None
+ # A head held for its owner's explicit retry is no claim either: only
+ # that retry starts it.
+ head = (
+ delivery_store.claimable_fifo_head(conn, session_id)
+ if delivery_store.active_turn(conn, session_id) is None
+ else None
)
+ claimable = head is not None and not delivery_store.requires_explicit_start_retry(head)
if not claimable:
return False
with self._sqlite_engine().begin() as conn:
@@ -6107,6 +6113,7 @@ class SessionTurnManager:
delivery_rows.c.current_expected_native_turn_id.label(
"delivery_expected_native_turn_id"
),
+ delivery_rows.c.delivery_history_json,
literal(None).label("turn_id"),
literal(None).label("turn_state"),
literal(None).label("turn_version"),
@@ -6382,6 +6389,10 @@ class SessionTurnManager:
predecessor_terminal_at=(
str(row.get("predecessor_terminal_at") or "") or None
),
+ awaits_explicit_retry=(
+ row["kind"] == "open_head"
+ and delivery_store.requires_explicit_start_retry(row)
+ ),
)
for row in selected
]
diff --git a/docs/plans/controller-loop-sqlite-writers.md b/docs/plans/controller-loop-sqlite-writers.md
index 139806255..791b6acb0 100644
--- a/docs/plans/controller-loop-sqlite-writers.md
+++ b/docs/plans/controller-loop-sqlite-writers.md
@@ -112,6 +112,24 @@ What it shows:
- **Every other site stayed under 135 ms per write**, on the same disk and under the same load. That includes `persist_agent_message` (#2725, parked until evidence shows a need) and path B's terminal write.
- **Coverage.** The soak exercised the Vibey and Model Hub paths. It did not exercise IM transports, the Claude or Codex receivers, or Harness Runs. Sites that review found and that this soak did not reach stay listed below until a soak measures them.
+**Soak 2** (2026-10-17, the owner's host on gh-v3.2.2rc2, about 3 h 10 min of everyday use: Claude, Codex and Vibey Turns, Harness and IM). The 08:50 summary counted 12221 writes at 23 sites. There were no loop stalls of 2 s or more and no `database is locked`. The leading sites, by total loop time:
+
+| Site (caller) | Writes | Total | Max |
+| --- | --- | --- | --- |
+| `drain_delivery_queue` (`recover_runtime_delivery_observation`) | 9967 | 91.5 s | 506 ms |
+| `persist_agent_message` (`emit_agent_message` `:3934`) | 841 | 2.7 s | 403 ms |
+| `persist_agent_message` (`emit_agent_message` `:3286`) | 699 | 2.6 s | 482 ms |
+| `on_native_start` (`commit_native_start`) | 72 | 1.7 s | 475 ms |
+| `upsert_activity` (`_persist_activity`) | 78 | 1.3 s | 489 ms |
+| `bind_agent_session_by_id` (`v2_sessions.bind_agent_session_by_id`) | 74 | 1.3 s | 598 ms |
+| `_admit_p1` (`deliver`) | 112 | 1.0 s | 396 ms |
+
+The other sixteen sites each took under 1 s of loop time in total.
+
+What it shows:
+- **One queued head took 82% of all loop writes.** It had failed to start three times and was held until its owner retried it explicitly. No automatic start can claim such a head. Even so, every runtime-work scan reported it as an open head, and the drain's read-only precheck counted it as claimable. Each scan therefore took SQLite's writer on the loop, the claim refused the head, and the lane backed off for a second and scanned it again, at about 0.9 writes a second. A held head now only moves the scan's cursor, and the drain's precheck no longer counts it as claimable.
+- **Every other site averaged under 30 ms per write.** Single writes reached 400–600 ms on the host's disk. `persist_agent_message` (#2725) is the largest of them, at 5.3 s over 1540 writes.
+
The per-Turn admissions all hold the boundary inside the transaction, so they are loop-side boundary holders and fall under B's split analysis:
- `_admit_p0`, `_admit_p1`, `_admit_p3`, `_promote_fifo_head`;
- `drain_delivery_queue`, `_resume_linked_control_successor`.
diff --git a/tests/test_runtime_recovery.py b/tests/test_runtime_recovery.py
index 5864649bb..e6940b245 100644
--- a/tests/test_runtime_recovery.py
+++ b/tests/test_runtime_recovery.py
@@ -936,3 +936,123 @@ async def test_cancel_abandons_unknown_start_without_replaying(
assert turn["terminal_evidence_kind"] == "runtime_gone"
assert delivery["state"] == "retired"
assert successors == []
+
+
+def _held_for_explicit_retry(conn, delivery_id: str) -> None:
+ """The durable hold a third pre-write start failure leaves: queued until its owner retries it."""
+
+ history = {
+ "events": [
+ {"kind": "start", "outcome": "claimed", "attempt_id": "atm-held", "position": 0, "turn_id": "turn-held"},
+ {
+ "kind": "start",
+ "outcome": "not_written",
+ "attempt_id": None,
+ "turn_id": None,
+ "receipt": {
+ "kind": "definitive_prewrite_failure",
+ "reason": "no_terminal_result",
+ "requires_explicit_retry": True,
+ },
+ },
+ ]
+ }
+ conn.execute(
+ update(delivery_store.message_deliveries)
+ .where(delivery_store.message_deliveries.c.id == delivery_id)
+ .values(delivery_history_json=json.dumps(history))
+ )
+
+
+async def test_a_head_held_for_explicit_retry_is_left_to_its_owner(tmp_path: Path, monkeypatch) -> None:
+ """A queued head that only its owner's explicit retry can start is not recovery work.
+
+ On a real host one such head was re-entered about once a second for hours:
+ each scan reported it as an open head, the drain took SQLite's writer on the
+ controller loop, the claim refused it, and the lane backed off one second
+ and scanned it again -- 9967 writes in 3h10m.
+ """
+
+ import core.session_turns as session_turns_module
+
+ engine = _engine(tmp_path)
+ with engine.begin() as conn:
+ _session(conn, "ses-held")
+ _delivery(conn, "delivery-held", "ses-held")
+ _held_for_explicit_retry(conn, "delivery-held")
+ manager = SessionTurnManager(SimpleNamespace())
+ manager._engine = engine
+ writer_reservations: list[str] = []
+ real_reserve = session_turns_module.reserve_write_lock
+ monkeypatch.setattr(
+ session_turns_module,
+ "reserve_write_lock",
+ lambda conn: writer_reservations.append("drain") or real_reserve(conn),
+ )
+ processed: list[str] = []
+
+ class _ObservedHandler(SessionDeliveryRecoveryHandler):
+ async def process(self, item):
+ processed.append(item.partition_key)
+ return await super().process(item)
+
+ supervisor = RuntimeWorkSupervisor(reconcile_interval=0.01, lane_capacity=1, retry_backoff=0.01)
+ supervisor.register(RuntimeWorkLane.SESSION_DELIVERIES, _ObservedHandler(manager))
+ await supervisor.activate()
+ await asyncio.sleep(0.5)
+ await supervisor.stop()
+
+ assert processed == []
+ assert writer_reservations == []
+ # A drain asked directly, as at a Turn end, answers from a read too.
+ assert await manager.drain_delivery_queue("ses-held") is False
+ assert writer_reservations == []
+ with engine.connect() as conn:
+ assert delivery_store.get_delivery(conn, "delivery-held")["state"] == "queued"
+ engine.dispose()
+
+
+async def test_a_held_head_on_the_last_page_leaves_the_next_wake_a_full_scan(tmp_path: Path) -> None:
+ """A wake for a Session that sorts before a held head still finds its recovery work."""
+
+ engine = _engine(tmp_path)
+ with engine.begin() as conn:
+ _session(conn, "ses-z-held")
+ _delivery(conn, "delivery-held", "ses-z-held")
+ _held_for_explicit_retry(conn, "delivery-held")
+ manager = SessionTurnManager(SimpleNamespace())
+ manager._engine = engine
+ scanned = asyncio.Event()
+ processed: list[str] = []
+ recovered = asyncio.Event()
+
+ class _ObservedHandler(SessionDeliveryRecoveryHandler):
+ def scan(self, **kwargs):
+ result = super().scan(**kwargs)
+ scanned.set()
+ return result
+
+ async def process(self, item):
+ processed.append(item.partition_key)
+ recovered.set()
+ return True
+
+ # Only a wake can scan: reconciliation is too far away to help.
+ supervisor = RuntimeWorkSupervisor(reconcile_interval=3600, lane_capacity=1)
+ supervisor.register(RuntimeWorkLane.SESSION_DELIVERIES, _ObservedHandler(manager))
+ await supervisor.activate()
+ await asyncio.wait_for(scanned.wait(), timeout=1)
+ await asyncio.sleep(0.05)
+ with engine.begin() as conn:
+ _session(conn, "ses-a")
+ _delivery(conn, "delivery-a", "ses-a")
+ supervisor.notify(RuntimeWorkLane.SESSION_DELIVERIES)
+ try:
+ await asyncio.wait_for(recovered.wait(), timeout=1)
+ finally:
+ await supervisor.stop()
+ engine.dispose()
+
+ assert processed == ["ses-a"]