Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(session): bound startup delivery recovery to unresolved work (#2713)

Call White7cebdfe40bc5
diff --git a/core/session_turns.py b/core/session_turns.py
index 8cad2ea8b..f1e5b3342 100644
--- a/core/session_turns.py
+++ b/core/session_turns.py
@@ -6790,8 +6790,8 @@ class SessionTurnManager:
for turn in turns:
turn_id = str(turn["id"])
target_session = str(turn["session_id"])
- with self._sqlite_engine().begin() as conn:
- reserve_write_lock(conn)
+ # A read: ``_terminalize_durable_turn`` rechecks it under the writer.
+ with self._sqlite_engine().connect() as conn:
latest = delivery_store.get_turn(conn, turn_id)
terminal_stop_receipt = bool(
latest is not None
@@ -7122,7 +7122,7 @@ class SessionTurnManager:
if result.state != "reserved":
recovered.append(target_session)
with self._sqlite_engine().connect() as conn:
- terminal_run_owners = delivery_store.terminal_turn_agent_run_owners(
+ terminal_run_owners = delivery_store.recoverable_terminal_turn_agent_run_owners(
conn,
session_id,
)
@@ -7187,8 +7187,7 @@ class SessionTurnManager:
recoverable_owners.append((terminal_turn, run_ids))
for terminal_turn, run_ids in recoverable_owners:
self._settle_agent_run_ids_from_terminal_turn(run_ids, terminal_turn)
- with self._sqlite_engine().begin() as conn:
- reserve_write_lock(conn)
+ with self._sqlite_engine().connect() as conn:
queued_without_owner = set(
delivery_store.queued_session_ids_without_live_turns(conn, session_id)
)
@@ -7203,17 +7202,25 @@ class SessionTurnManager:
with self._sqlite_engine().begin() as conn:
reserve_write_lock(conn)
live = delivery_store.session_ids_with_live_turns(conn)
- owned = delivery_store.session_ids_with_turn_history(conn)
- idle = owned - live
- if idle:
- cleared = conn.execute(
- update(agent_sessions)
- .where(agent_sessions.c.id.in_(idle))
- .where(agent_sessions.c.status == "active")
- .where(agent_sessions.c.agent_status == "running")
- .values(agent_status="idle")
+ # Only the sessions projected ``running`` that have Turn history and no
+ # live Turn, each checked by an indexed lookup -- not every session that
+ # ever ran a Turn, a set that grows with history (#2706).
+ # Seeks ``ix_agent_sessions_agent_status`` (agent_status, status).
+ stale_running = (
+ update(agent_sessions)
+ .where(agent_sessions.c.agent_status == "running")
+ .where(agent_sessions.c.status == "active")
+ .where(
+ exists(
+ select(session_turn_rows.c.id).where(
+ session_turn_rows.c.session_id == agent_sessions.c.id
+ )
+ )
)
- _ = cleared.rowcount
+ )
+ if live:
+ stale_running = stale_running.where(agent_sessions.c.id.notin_(live))
+ conn.execute(stale_running.values(agent_status="idle"))
if live:
running = conn.execute(
update(agent_sessions)
@@ -9579,7 +9586,7 @@ class SessionTurnManager:
return
try:
with self._sqlite_engine().connect() as conn:
- has_history = session_id in delivery_store.session_ids_with_turn_history(conn)
+ has_history = delivery_store.session_has_turn_history(conn, session_id)
except Exception:
logger.debug(
"could not inspect durable Turn history for restored poll %s",
diff --git a/storage/alembic/versions/20261016_0068_agent_sessions_agent_status_index.py b/storage/alembic/versions/20261016_0068_agent_sessions_agent_status_index.py
new file mode 100644
index 000000000..f47ef8004
--- /dev/null
+++ b/storage/alembic/versions/20261016_0068_agent_sessions_agent_status_index.py
@@ -0,0 +1,60 @@
+"""index a Session by its projected agent status
+
+At every start, durable-delivery recovery clears the ``running`` projection of
+Sessions whose Turns have all ended (``SessionTurnManager.project_durable_agent_status``).
+No index led with ``agent_status``, so finding the projected-running Sessions
+walked every active Session through ``ix_agent_sessions_status_activity
+(status=?)`` and filtered each one, under SQLite's write lock (#2706)::
+
+ SEARCH agent_sessions USING INDEX ix_agent_sessions_status_activity (status=?)
+
+That grows with every Session ever kept, not with the work in hand. Leading
+with ``agent_status`` makes it a seek on the few Sessions projected running:
+``SEARCH agent_sessions USING INDEX ix_agent_sessions_agent_status
+(agent_status=? AND status=?)``.
+
+Index only: downgrade drops it, and an older build reads the same rows without
+it.
+
+Revision ID: 20261016_0068
+Revises: 20261016_0067
+Create Date: 2026-10-16
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+revision = "20261016_0068"
+down_revision = "20261016_0067"
+branch_labels = None
+depends_on = None
+MIGRATION_SAFETY = "additive"
+
+_INDEX = "ix_agent_sessions_agent_status"
+# Exported so the head-schema repair path in ``storage/migrations.py`` executes THIS
+# statement rather than a retyped copy.
+DROP_INDEX_SQL = f"drop index if exists {_INDEX}"
+CREATE_INDEX_SQL = f"create index {_INDEX} on agent_sessions (agent_status, status)"
+
+
+def _tables(bind) -> set[str]:
+ return {
+ str(row[0])
+ for row in bind.exec_driver_sql("select name from sqlite_master where type = 'table'")
+ }
+
+
+def upgrade() -> None:
+ bind = op.get_bind()
+ if "agent_sessions" not in _tables(bind):
+ return
+ bind.exec_driver_sql(DROP_INDEX_SQL)
+ bind.exec_driver_sql(CREATE_INDEX_SQL)
+
+
+def downgrade() -> None:
+ bind = op.get_bind()
+ if "agent_sessions" not in _tables(bind):
+ return
+ bind.exec_driver_sql(DROP_INDEX_SQL)
diff --git a/storage/message_deliveries.py b/storage/message_deliveries.py
index ca0577b49..05072a5af 100644
--- a/storage/message_deliveries.py
+++ b/storage/message_deliveries.py
@@ -8,7 +8,7 @@ from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Any, Iterable, Literal
-from sqlalchemy import Select, and_, func, or_, select, update
+from sqlalchemy import Select, and_, func, literal_column, or_, select, text, union, union_all, update
from sqlalchemy.engine import Connection
from core.delivery_target import normalize_message_kind
@@ -1880,16 +1880,37 @@ def terminalize_turn(
return {"changed": settled is not None, "turn": settled or turn}
+# The predicates of the partial indexes that migration 0044 built over live rows
+# only, verbatim. Session Turns and Deliveries accumulate with every Turn ever run,
+# and SQLite uses a partial index only when the query states its predicate -- never
+# through bound parameters -- so the reads that must stay bounded by live work spell
+# them out (#2706).
+_LIVE_TURN_INDEX_PREDICATE = "state in ('starting','active')"
+_WAITING_TURN_INDEX_PREDICATE = "state = 'waiting'"
+_LIVE_DELIVERY_INDEX_PREDICATE = (
+ "state in ('reserved','queued','claimed','pending_steer','steering','reconciling_steer')"
+)
+
+
def recovery_turns(conn: Connection, session_id: str | None = None) -> list[dict[str, Any]]:
- query = select(session_turns).where(session_turns.c.state.in_(("waiting",) + TURN_OWNER_STATES))
- if session_id:
- query = query.where(session_turns.c.session_id == session_id)
- return [dict(row) for row in conn.execute(query.order_by(session_turns.c.created_at, session_turns.c.id)).mappings()]
+ # One arm per partial index: an OR of the two predicates would scan the table.
+ arms = []
+ for predicate in (_WAITING_TURN_INDEX_PREDICATE, _LIVE_TURN_INDEX_PREDICATE):
+ arm = select(session_turns).where(text(predicate))
+ if session_id:
+ arm = arm.where(session_turns.c.session_id == session_id)
+ arms.append(arm)
+ turns = union_all(*arms).subquery()
+ return [
+ dict(row)
+ for row in conn.execute(select(turns).order_by(turns.c.created_at, turns.c.id)).mappings()
+ ]
def unresolved_deliveries(conn: Connection, session_id: str | None = None) -> list[dict[str, Any]]:
query = select(message_deliveries).where(
- message_deliveries.c.state.in_(FENCE_STATES + ("claimed",))
+ message_deliveries.c.state.in_(FENCE_STATES + ("claimed",)),
+ text(_LIVE_DELIVERY_INDEX_PREDICATE),
)
if session_id:
query = query.where(message_deliveries.c.session_id == session_id)
@@ -1902,7 +1923,10 @@ def recoverable_reservations(
) -> list[dict[str, Any]]:
"""Pre-write producer reservations safe to re-enter through admission."""
- query = select(message_deliveries).where(message_deliveries.c.state == "reserved")
+ query = select(message_deliveries).where(
+ message_deliveries.c.state == "reserved",
+ text(_LIVE_DELIVERY_INDEX_PREDICATE),
+ )
if session_id:
query = query.where(message_deliveries.c.session_id == session_id)
return [
@@ -1917,13 +1941,18 @@ def session_ids_with_live_turns(conn: Connection) -> set[str]:
return {
str(value)
for value in conn.execute(
- select(session_turns.c.session_id).where(session_turns.c.state.in_(TURN_OWNER_STATES)).distinct()
+ select(session_turns.c.session_id).where(text(_LIVE_TURN_INDEX_PREDICATE)).distinct()
).scalars()
}
-def session_ids_with_turn_history(conn: Connection) -> set[str]:
- return {str(value) for value in conn.execute(select(session_turns.c.session_id).distinct()).scalars()}
+def session_has_turn_history(conn: Connection, session_id: str) -> bool:
+ return (
+ conn.execute(
+ select(session_turns.c.id).where(session_turns.c.session_id == session_id).limit(1)
+ ).first()
+ is not None
+ )
def active_runtime_session_ids_for_backend(conn: Connection, backend: str) -> set[str]:
@@ -2202,12 +2231,38 @@ def retire_queued_with_run(
return retire_queued(conn, session_id, delivery_id)
-def terminal_turn_agent_run_owners(
+def recoverable_terminal_turn_agent_run_owners(
conn: Connection,
session_id: str | None = None,
) -> list[tuple[dict[str, Any], list[str]]]:
- """Return terminal Turns and their accepted Agent Run participants."""
+ """Terminal Turns that may still owe their accepted Agent Runs a settlement.
+
+ Recovery acts on a terminal Turn only when one of its accepted Runs is still
+ queued or running, or carries a pending owed failure notice (the legacy per-Run
+ notice it re-attributes). Every other terminal Turn is history: reading them all,
+ as this used to, grew with every Turn ever run and held the controller loop at
+ start (#2706). So the read starts from those Runs, through indexed seeks
+ (``ix_agent_runs_status_created``, ``ix_agent_runs_owed_notice``), and joins
+ outward to the terminal Turns that accepted them. Each Turn still comes back with
+ ALL its accepted participants.
+ """
+
+ from storage.background import NOTICE_PENDING, OWED_NOTICE_STATE_SQL, RUN_STATUS_ALIASES
+ live_statuses = sorted(
+ {"queued", "running"}
+ | {raw for raw, public in RUN_STATUS_ALIASES.items() if public in {"queued", "running"}}
+ )
+ candidate_deliveries = union(
+ select(agent_runs.c.delivery_id).where(agent_runs.c.status.in_(live_statuses)),
+ # Verbatim, so the planner matches ``ix_agent_runs_owed_notice``'s expression.
+ select(agent_runs.c.delivery_id).where(literal_column(OWED_NOTICE_STATE_SQL) == NOTICE_PENDING),
+ )
+ recoverable_turns = (
+ select(message_deliveries.c.turn_id)
+ .where(message_deliveries.c.id.in_(candidate_deliveries))
+ .where(message_deliveries.c.state == "accepted")
+ )
query = (
select(
session_turns.c.id.label("turn_id"),
@@ -2225,6 +2280,7 @@ def terminal_turn_agent_run_owners(
.join(agent_runs, agent_runs.c.delivery_id == message_deliveries.c.id)
.where(session_turns.c.state == "terminal")
.where(message_deliveries.c.state == "accepted")
+ .where(session_turns.c.id.in_(recoverable_turns))
)
if session_id:
query = query.where(session_turns.c.session_id == session_id)
diff --git a/storage/migrations.py b/storage/migrations.py
index 25cbd4c87..b21cf3464 100644
--- a/storage/migrations.py
+++ b/storage/migrations.py
@@ -915,6 +915,18 @@ def _ensure_messages_query_indexes(conn: sqlite3.Connection, tables: set[str]) -
"create index if not exists ix_agent_sessions_scope_status_pinned_activity "
"on agent_sessions (scope_id, status, pinned, last_active_at, created_at, id)"
)
+ status_index = import_module(
+ "storage.alembic.versions.20261016_0068_agent_sessions_agent_status_index"
+ )
+ status_row = conn.execute(
+ "select sql from sqlite_master where type = 'index' and name = ?",
+ (status_index._INDEX,),
+ ).fetchone()
+ if status_row is None or _normalized_index_sql(status_row[0]) != _normalized_index_sql(
+ status_index.CREATE_INDEX_SQL
+ ):
+ conn.execute(status_index.DROP_INDEX_SQL)
+ conn.execute(status_index.CREATE_INDEX_SQL)
if "messages" not in tables:
return
transcript_index = import_module(
diff --git a/storage/models.py b/storage/models.py
index 52ad07763..1fa242e7f 100644
--- a/storage/models.py
+++ b/storage/models.py
@@ -228,6 +228,8 @@ agent_sessions = Table(
Index("ix_agent_sessions_scope_anchor_workdir", "scope_id", "session_anchor", "workdir"),
Index("ix_agent_sessions_backend_variant", "agent_backend", "agent_variant"),
Index("ix_agent_sessions_status_activity", "status", "last_active_at"),
+ # The Sessions projected running, for the startup projection repair (migration 0068).
+ Index("ix_agent_sessions_agent_status", "agent_status", "status"),
Index("ix_agent_sessions_visibility", "visibility"),
Index("ix_agent_sessions_scope_status_activity", "scope_id", "status", "last_active_at", "created_at", "id"),
Index(
diff --git a/tests/test_agent_context_seq_migration.py b/tests/test_agent_context_seq_migration.py
index d51c14443..eddeaa974 100644
--- a/tests/test_agent_context_seq_migration.py
+++ b/tests/test_agent_context_seq_migration.py
@@ -56,7 +56,7 @@ def test_context_seq_upgrade_and_downgrade_keep_released_rows(tmp_path):
migrations.run_migrations(path)
with sqlite3.connect(path) as conn:
- assert conn.execute("SELECT version_num FROM alembic_version").fetchone() == ("20261016_0067",)
+ assert conn.execute("SELECT version_num FROM alembic_version").fetchone() == ("20261016_0068",)
assert_released_rows_unchanged(conn)
assert _index_sql(conn, "ix_agent_events_turn_sequence_id") is not None
for table, (first, same_session, other_session) in _IDS.items():
diff --git a/tests/test_session_delivery_fsm.py b/tests/test_session_delivery_fsm.py
index 564b70c7c..862fd1d24 100644
--- a/tests/test_session_delivery_fsm.py
+++ b/tests/test_session_delivery_fsm.py
@@ -8354,3 +8354,47 @@ def test_legacy_effective_author_survives_queue_collect_merge_hydrate(
public = delivery_store.public_delivery_payload(row)
assert "_memory_user_id" not in public["metadata"]
assert "resource_user_context" not in public["metadata"]
+
+
+def test_startup_recovery_reads_growing_tables_only_through_index_seeks(tmp_path: Path, sqlite_db_factory) -> None:
+ """#2706: durable-delivery recovery runs on the controller loop at every start, before producers are admitted.
+ ``agent_runs``, ``session_turns`` and ``message_deliveries`` grow with every Turn ever run, and ``agent_sessions``
+ with every Session kept; a read of their history held the loop at start. Every statement recovery issues must reach them by an index seek, or by walking a
+ partial index that holds only live rows -- never by scanning the table or a full index."""
+
+ from sqlalchemy import event
+
+ db_path = sqlite_db_factory(tmp_path / "recovery.sqlite")
+ engine = create_sqlite_engine(db_path)
+ _seed_session(engine)
+ manager = SessionTurnManager(_Controller(), build_context=_context)
+ manager._engine = engine
+ statements: list[tuple[str, object]] = []
+
+ def capture(_conn, _cursor, statement, parameters, _context, _executemany):
+ if statement.lstrip().upper().startswith(("SELECT", "UPDATE", "WITH")):
+ statements.append((statement, parameters))
+
+ event.listen(engine, "before_cursor_execute", capture)
+ try:
+ asyncio.run(manager.recover_durable_delivery_state())
+ finally:
+ event.remove(engine, "before_cursor_execute", capture)
+ engine.dispose()
+
+ growing = ("agent_runs", "session_turns", "message_deliveries", "agent_sessions")
+ live_only_indexes = {
+ "uq_session_turns_live_session",
+ "uq_session_turns_waiting_successor",
+ "ix_message_deliveries_session_order",
+ }
+ scans: list[str] = []
+ with closing(sqlite3.connect(db_path)) as raw:
+ for statement, parameters in statements:
+ for row in raw.execute("EXPLAIN QUERY PLAN " + statement, parameters):
+ detail = str(row[-1])
+ words = detail.split()
+ if detail.startswith("SCAN ") and words[1] in growing and words[-1] not in live_only_indexes:
+ scans.append(f"{detail} <- {' '.join(statement.split())}")
+ assert statements, "recovery issued no statements, so this test proves nothing"
+ assert not scans, "startup recovery scans a growing table:\n" + "\n".join(scans)
diff --git a/tests/test_skill_observability_migration.py b/tests/test_skill_observability_migration.py
index 797c9a5ce..fb6179068 100644
--- a/tests/test_skill_observability_migration.py
+++ b/tests/test_skill_observability_migration.py
@@ -35,7 +35,7 @@ def test_skill_schema_upgrade_downgrade_preserves_existing_events(tmp_path):
migrations.run_migrations(path)
with sqlite3.connect(path) as conn:
assert conn.execute(f"SELECT {columns} FROM agent_events ORDER BY id").fetchall() == original
- assert conn.execute("SELECT version_num FROM alembic_version").fetchone() == ("20261016_0067",)
+ assert conn.execute("SELECT version_num FROM alembic_version").fetchone() == ("20261016_0068",)
assert conn.execute("SELECT count(*) FROM skill_usage_daily").fetchone() == (0,)
assert {row[2] for row in conn.execute("PRAGMA foreign_key_list(skill_usage_daily)")} == {
"scopes",
diff --git a/tests/test_sqlite_state_migration.py b/tests/test_sqlite_state_migration.py
index a9691d4eb..dc29af1c2 100644
--- a/tests/test_sqlite_state_migration.py
+++ b/tests/test_sqlite_state_migration.py
@@ -38,7 +38,7 @@ from vibe.message_types import build_partial_index_predicate
pytestmark = pytest.mark.no_sqlite_template
-HEAD_REVISION = "20261016_0067"
+HEAD_REVISION = "20261016_0068"
# ``storage.models`` builds a bare ``MetaData()``, so its foreign keys are unnamed and
# Alembic cannot re-emit them when batch mode recreates a table. Every migration that
# rebuilds one therefore passes this convention; the rebuild simulated below has to pass
@@ -3605,6 +3605,7 @@ def test_run_migrations_repairs_head_indexes_before_stamping_head(tmp_path: Path
conn.execute("drop index if exists ix_messages_vibey_recovery_input")
conn.execute("drop index if exists ix_message_deliveries_message")
conn.execute("drop index if exists ix_agent_sessions_scope_status_activity")
+ conn.execute("drop index if exists ix_agent_sessions_agent_status")
conn.execute("drop index if exists ix_messages_session_created_id")
conn.execute("drop index if exists ix_messages_session_type_created_id")
conn.execute("drop index if exists ix_messages_platform_session_created_id")
@@ -3622,6 +3623,7 @@ def test_run_migrations_repairs_head_indexes_before_stamping_head(tmp_path: Path
with sqlite3.connect(db_path) as conn:
assert "context_seq is null" in _index_sql(conn, "ix_messages_vibey_recovery_input")
assert "message_id is not null" in _index_sql(conn, "ix_message_deliveries_message")
+ assert "(agent_status, status)" in _index_sql(conn, "ix_agent_sessions_agent_status")
run_migrations(db_path)
diff --git a/tests/test_web_push_provider_invalidation_migration.py b/tests/test_web_push_provider_invalidation_migration.py
index 832ef51a0..1b8c08120 100644
--- a/tests/test_web_push_provider_invalidation_migration.py
+++ b/tests/test_web_push_provider_invalidation_migration.py
@@ -69,7 +69,7 @@ def test_upgrade_keeps_legacy_disabled_failure_opted_out(tmp_path):
web_push_service.mark_send_failure(conn, endpoint=enabled["endpoint"], disable=True)
with sqlite3.connect(path) as conn:
assert conn.execute("select version_num from alembic_version").fetchone() == (
- "20261016_0067",
+ "20261016_0068",
)
command.downgrade(migrations.alembic_config(path), "20260907_0061")