Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(session): bound startup delivery recovery to unresolved work (#2713)
Call White7cebdfe40bc5
diff --git a/core/session_turns.py b/core/session_turns.pyindex 8cad2ea8b..f1e5b3342 100644--- a/core/session_turns.py+++ b/core/session_turns.py@@ -6790,8 +6790,8 @@ class SessionTurnManager:for turn in turns:turn_id = str(turn["id"])target_session = str(turn["session_id"])- with self._sqlite_engine().begin() as conn:- reserve_write_lock(conn)+ # A read: ``_terminalize_durable_turn`` rechecks it under the writer.+ with self._sqlite_engine().connect() as conn:latest = delivery_store.get_turn(conn, turn_id)terminal_stop_receipt = bool(latest is not None@@ -7122,7 +7122,7 @@ class SessionTurnManager:if result.state != "reserved":recovered.append(target_session)with self._sqlite_engine().connect() as conn:- terminal_run_owners = delivery_store.terminal_turn_agent_run_owners(+ terminal_run_owners = delivery_store.recoverable_terminal_turn_agent_run_owners(conn,session_id,)@@ -7187,8 +7187,7 @@ class SessionTurnManager:recoverable_owners.append((terminal_turn, run_ids))for terminal_turn, run_ids in recoverable_owners:self._settle_agent_run_ids_from_terminal_turn(run_ids, terminal_turn)- with self._sqlite_engine().begin() as conn:- reserve_write_lock(conn)+ with self._sqlite_engine().connect() as conn:queued_without_owner = set(delivery_store.queued_session_ids_without_live_turns(conn, session_id))@@ -7203,17 +7202,25 @@ class SessionTurnManager:with self._sqlite_engine().begin() as conn:reserve_write_lock(conn)live = delivery_store.session_ids_with_live_turns(conn)- owned = delivery_store.session_ids_with_turn_history(conn)- idle = owned - live- if idle:- cleared = conn.execute(- update(agent_sessions)- .where(agent_sessions.c.id.in_(idle))- .where(agent_sessions.c.status == "active")- .where(agent_sessions.c.agent_status == "running")- .values(agent_status="idle")+ # Only the sessions projected ``running`` that have Turn history and no+ # live Turn, each checked by an indexed lookup -- not every session that+ # ever ran a Turn, a set that grows with history (#2706).+ # Seeks ``ix_agent_sessions_agent_status`` (agent_status, status).+ stale_running = (+ update(agent_sessions)+ .where(agent_sessions.c.agent_status == "running")+ .where(agent_sessions.c.status == "active")+ .where(+ exists(+ select(session_turn_rows.c.id).where(+ session_turn_rows.c.session_id == agent_sessions.c.id+ )+ ))- _ = cleared.rowcount+ )+ if live:+ stale_running = stale_running.where(agent_sessions.c.id.notin_(live))+ conn.execute(stale_running.values(agent_status="idle"))if live:running = conn.execute(update(agent_sessions)@@ -9579,7 +9586,7 @@ class SessionTurnManager:returntry:with self._sqlite_engine().connect() as conn:- has_history = session_id in delivery_store.session_ids_with_turn_history(conn)+ has_history = delivery_store.session_has_turn_history(conn, session_id)except Exception:logger.debug("could not inspect durable Turn history for restored poll %s",diff --git a/storage/alembic/versions/20261016_0068_agent_sessions_agent_status_index.py b/storage/alembic/versions/20261016_0068_agent_sessions_agent_status_index.pynew file mode 100644index 000000000..f47ef8004--- /dev/null+++ b/storage/alembic/versions/20261016_0068_agent_sessions_agent_status_index.py@@ -0,0 +1,60 @@+"""index a Session by its projected agent status++At every start, durable-delivery recovery clears the ``running`` projection of+Sessions whose Turns have all ended (``SessionTurnManager.project_durable_agent_status``).+No index led with ``agent_status``, so finding the projected-running Sessions+walked every active Session through ``ix_agent_sessions_status_activity+(status=?)`` and filtered each one, under SQLite's write lock (#2706)::++ SEARCH agent_sessions USING INDEX ix_agent_sessions_status_activity (status=?)++That grows with every Session ever kept, not with the work in hand. Leading+with ``agent_status`` makes it a seek on the few Sessions projected running:+``SEARCH agent_sessions USING INDEX ix_agent_sessions_agent_status+(agent_status=? AND status=?)``.++Index only: downgrade drops it, and an older build reads the same rows without+it.++Revision ID: 20261016_0068+Revises: 20261016_0067+Create Date: 2026-10-16+"""++from __future__ import annotations++from alembic import op++revision = "20261016_0068"+down_revision = "20261016_0067"+branch_labels = None+depends_on = None+MIGRATION_SAFETY = "additive"++_INDEX = "ix_agent_sessions_agent_status"+# Exported so the head-schema repair path in ``storage/migrations.py`` executes THIS+# statement rather than a retyped copy.+DROP_INDEX_SQL = f"drop index if exists {_INDEX}"+CREATE_INDEX_SQL = f"create index {_INDEX} on agent_sessions (agent_status, status)"+++def _tables(bind) -> set[str]:+ return {+ str(row[0])+ for row in bind.exec_driver_sql("select name from sqlite_master where type = 'table'")+ }+++def upgrade() -> None:+ bind = op.get_bind()+ if "agent_sessions" not in _tables(bind):+ return+ bind.exec_driver_sql(DROP_INDEX_SQL)+ bind.exec_driver_sql(CREATE_INDEX_SQL)+++def downgrade() -> None:+ bind = op.get_bind()+ if "agent_sessions" not in _tables(bind):+ return+ bind.exec_driver_sql(DROP_INDEX_SQL)diff --git a/storage/message_deliveries.py b/storage/message_deliveries.pyindex ca0577b49..05072a5af 100644--- a/storage/message_deliveries.py+++ b/storage/message_deliveries.py@@ -8,7 +8,7 @@ from collections.abc import Mappingfrom datetime import datetime, timezonefrom typing import Any, Iterable, Literal-from sqlalchemy import Select, and_, func, or_, select, update+from sqlalchemy import Select, and_, func, literal_column, or_, select, text, union, union_all, updatefrom sqlalchemy.engine import Connectionfrom core.delivery_target import normalize_message_kind@@ -1880,16 +1880,37 @@ def terminalize_turn(return {"changed": settled is not None, "turn": settled or turn}+# The predicates of the partial indexes that migration 0044 built over live rows+# only, verbatim. Session Turns and Deliveries accumulate with every Turn ever run,+# and SQLite uses a partial index only when the query states its predicate -- never+# through bound parameters -- so the reads that must stay bounded by live work spell+# them out (#2706).+_LIVE_TURN_INDEX_PREDICATE = "state in ('starting','active')"+_WAITING_TURN_INDEX_PREDICATE = "state = 'waiting'"+_LIVE_DELIVERY_INDEX_PREDICATE = (+ "state in ('reserved','queued','claimed','pending_steer','steering','reconciling_steer')"+)++def recovery_turns(conn: Connection, session_id: str | None = None) -> list[dict[str, Any]]:- query = select(session_turns).where(session_turns.c.state.in_(("waiting",) + TURN_OWNER_STATES))- if session_id:- query = query.where(session_turns.c.session_id == session_id)- return [dict(row) for row in conn.execute(query.order_by(session_turns.c.created_at, session_turns.c.id)).mappings()]+ # One arm per partial index: an OR of the two predicates would scan the table.+ arms = []+ for predicate in (_WAITING_TURN_INDEX_PREDICATE, _LIVE_TURN_INDEX_PREDICATE):+ arm = select(session_turns).where(text(predicate))+ if session_id:+ arm = arm.where(session_turns.c.session_id == session_id)+ arms.append(arm)+ turns = union_all(*arms).subquery()+ return [+ dict(row)+ for row in conn.execute(select(turns).order_by(turns.c.created_at, turns.c.id)).mappings()+ ]def unresolved_deliveries(conn: Connection, session_id: str | None = None) -> list[dict[str, Any]]:query = select(message_deliveries).where(- message_deliveries.c.state.in_(FENCE_STATES + ("claimed",))+ message_deliveries.c.state.in_(FENCE_STATES + ("claimed",)),+ text(_LIVE_DELIVERY_INDEX_PREDICATE),)if session_id:query = query.where(message_deliveries.c.session_id == session_id)@@ -1902,7 +1923,10 @@ def recoverable_reservations() -> list[dict[str, Any]]:"""Pre-write producer reservations safe to re-enter through admission."""- query = select(message_deliveries).where(message_deliveries.c.state == "reserved")+ query = select(message_deliveries).where(+ message_deliveries.c.state == "reserved",+ text(_LIVE_DELIVERY_INDEX_PREDICATE),+ )if session_id:query = query.where(message_deliveries.c.session_id == session_id)return [@@ -1917,13 +1941,18 @@ def session_ids_with_live_turns(conn: Connection) -> set[str]:return {str(value)for value in conn.execute(- select(session_turns.c.session_id).where(session_turns.c.state.in_(TURN_OWNER_STATES)).distinct()+ select(session_turns.c.session_id).where(text(_LIVE_TURN_INDEX_PREDICATE)).distinct()).scalars()}-def session_ids_with_turn_history(conn: Connection) -> set[str]:- return {str(value) for value in conn.execute(select(session_turns.c.session_id).distinct()).scalars()}+def session_has_turn_history(conn: Connection, session_id: str) -> bool:+ return (+ conn.execute(+ select(session_turns.c.id).where(session_turns.c.session_id == session_id).limit(1)+ ).first()+ is not None+ )def active_runtime_session_ids_for_backend(conn: Connection, backend: str) -> set[str]:@@ -2202,12 +2231,38 @@ def retire_queued_with_run(return retire_queued(conn, session_id, delivery_id)-def terminal_turn_agent_run_owners(+def recoverable_terminal_turn_agent_run_owners(conn: Connection,session_id: str | None = None,) -> list[tuple[dict[str, Any], list[str]]]:- """Return terminal Turns and their accepted Agent Run participants."""+ """Terminal Turns that may still owe their accepted Agent Runs a settlement.++ Recovery acts on a terminal Turn only when one of its accepted Runs is still+ queued or running, or carries a pending owed failure notice (the legacy per-Run+ notice it re-attributes). Every other terminal Turn is history: reading them all,+ as this used to, grew with every Turn ever run and held the controller loop at+ start (#2706). So the read starts from those Runs, through indexed seeks+ (``ix_agent_runs_status_created``, ``ix_agent_runs_owed_notice``), and joins+ outward to the terminal Turns that accepted them. Each Turn still comes back with+ ALL its accepted participants.+ """++ from storage.background import NOTICE_PENDING, OWED_NOTICE_STATE_SQL, RUN_STATUS_ALIASES+ live_statuses = sorted(+ {"queued", "running"}+ | {raw for raw, public in RUN_STATUS_ALIASES.items() if public in {"queued", "running"}}+ )+ candidate_deliveries = union(+ select(agent_runs.c.delivery_id).where(agent_runs.c.status.in_(live_statuses)),+ # Verbatim, so the planner matches ``ix_agent_runs_owed_notice``'s expression.+ select(agent_runs.c.delivery_id).where(literal_column(OWED_NOTICE_STATE_SQL) == NOTICE_PENDING),+ )+ recoverable_turns = (+ select(message_deliveries.c.turn_id)+ .where(message_deliveries.c.id.in_(candidate_deliveries))+ .where(message_deliveries.c.state == "accepted")+ )query = (select(session_turns.c.id.label("turn_id"),@@ -2225,6 +2280,7 @@ def terminal_turn_agent_run_owners(.join(agent_runs, agent_runs.c.delivery_id == message_deliveries.c.id).where(session_turns.c.state == "terminal").where(message_deliveries.c.state == "accepted")+ .where(session_turns.c.id.in_(recoverable_turns)))if session_id:query = query.where(session_turns.c.session_id == session_id)diff --git a/storage/migrations.py b/storage/migrations.pyindex 25cbd4c87..b21cf3464 100644--- a/storage/migrations.py+++ b/storage/migrations.py@@ -915,6 +915,18 @@ def _ensure_messages_query_indexes(conn: sqlite3.Connection, tables: set[str]) -"create index if not exists ix_agent_sessions_scope_status_pinned_activity ""on agent_sessions (scope_id, status, pinned, last_active_at, created_at, id)")+ status_index = import_module(+ "storage.alembic.versions.20261016_0068_agent_sessions_agent_status_index"+ )+ status_row = conn.execute(+ "select sql from sqlite_master where type = 'index' and name = ?",+ (status_index._INDEX,),+ ).fetchone()+ if status_row is None or _normalized_index_sql(status_row[0]) != _normalized_index_sql(+ status_index.CREATE_INDEX_SQL+ ):+ conn.execute(status_index.DROP_INDEX_SQL)+ conn.execute(status_index.CREATE_INDEX_SQL)if "messages" not in tables:returntranscript_index = import_module(diff --git a/storage/models.py b/storage/models.pyindex 52ad07763..1fa242e7f 100644--- a/storage/models.py+++ b/storage/models.py@@ -228,6 +228,8 @@ agent_sessions = Table(Index("ix_agent_sessions_scope_anchor_workdir", "scope_id", "session_anchor", "workdir"),Index("ix_agent_sessions_backend_variant", "agent_backend", "agent_variant"),Index("ix_agent_sessions_status_activity", "status", "last_active_at"),+ # The Sessions projected running, for the startup projection repair (migration 0068).+ Index("ix_agent_sessions_agent_status", "agent_status", "status"),Index("ix_agent_sessions_visibility", "visibility"),Index("ix_agent_sessions_scope_status_activity", "scope_id", "status", "last_active_at", "created_at", "id"),Index(diff --git a/tests/test_agent_context_seq_migration.py b/tests/test_agent_context_seq_migration.pyindex d51c14443..eddeaa974 100644--- a/tests/test_agent_context_seq_migration.py+++ b/tests/test_agent_context_seq_migration.py@@ -56,7 +56,7 @@ def test_context_seq_upgrade_and_downgrade_keep_released_rows(tmp_path):migrations.run_migrations(path)with sqlite3.connect(path) as conn:- assert conn.execute("SELECT version_num FROM alembic_version").fetchone() == ("20261016_0067",)+ assert conn.execute("SELECT version_num FROM alembic_version").fetchone() == ("20261016_0068",)assert_released_rows_unchanged(conn)assert _index_sql(conn, "ix_agent_events_turn_sequence_id") is not Nonefor table, (first, same_session, other_session) in _IDS.items():diff --git a/tests/test_session_delivery_fsm.py b/tests/test_session_delivery_fsm.pyindex 564b70c7c..862fd1d24 100644--- a/tests/test_session_delivery_fsm.py+++ b/tests/test_session_delivery_fsm.py@@ -8354,3 +8354,47 @@ def test_legacy_effective_author_survives_queue_collect_merge_hydrate(public = delivery_store.public_delivery_payload(row)assert "_memory_user_id" not in public["metadata"]assert "resource_user_context" not in public["metadata"]+++def test_startup_recovery_reads_growing_tables_only_through_index_seeks(tmp_path: Path, sqlite_db_factory) -> None:+ """#2706: durable-delivery recovery runs on the controller loop at every start, before producers are admitted.+ ``agent_runs``, ``session_turns`` and ``message_deliveries`` grow with every Turn ever run, and ``agent_sessions``+ with every Session kept; a read of their history held the loop at start. Every statement recovery issues must reach them by an index seek, or by walking a+ partial index that holds only live rows -- never by scanning the table or a full index."""++ from sqlalchemy import event++ db_path = sqlite_db_factory(tmp_path / "recovery.sqlite")+ engine = create_sqlite_engine(db_path)+ _seed_session(engine)+ manager = SessionTurnManager(_Controller(), build_context=_context)+ manager._engine = engine+ statements: list[tuple[str, object]] = []++ def capture(_conn, _cursor, statement, parameters, _context, _executemany):+ if statement.lstrip().upper().startswith(("SELECT", "UPDATE", "WITH")):+ statements.append((statement, parameters))++ event.listen(engine, "before_cursor_execute", capture)+ try:+ asyncio.run(manager.recover_durable_delivery_state())+ finally:+ event.remove(engine, "before_cursor_execute", capture)+ engine.dispose()++ growing = ("agent_runs", "session_turns", "message_deliveries", "agent_sessions")+ live_only_indexes = {+ "uq_session_turns_live_session",+ "uq_session_turns_waiting_successor",+ "ix_message_deliveries_session_order",+ }+ scans: list[str] = []+ with closing(sqlite3.connect(db_path)) as raw:+ for statement, parameters in statements:+ for row in raw.execute("EXPLAIN QUERY PLAN " + statement, parameters):+ detail = str(row[-1])+ words = detail.split()+ if detail.startswith("SCAN ") and words[1] in growing and words[-1] not in live_only_indexes:+ scans.append(f"{detail} <- {' '.join(statement.split())}")+ assert statements, "recovery issued no statements, so this test proves nothing"+ assert not scans, "startup recovery scans a growing table:\n" + "\n".join(scans)diff --git a/tests/test_skill_observability_migration.py b/tests/test_skill_observability_migration.pyindex 797c9a5ce..fb6179068 100644--- a/tests/test_skill_observability_migration.py+++ b/tests/test_skill_observability_migration.py@@ -35,7 +35,7 @@ def test_skill_schema_upgrade_downgrade_preserves_existing_events(tmp_path):migrations.run_migrations(path)with sqlite3.connect(path) as conn:assert conn.execute(f"SELECT {columns} FROM agent_events ORDER BY id").fetchall() == original- assert conn.execute("SELECT version_num FROM alembic_version").fetchone() == ("20261016_0067",)+ assert conn.execute("SELECT version_num FROM alembic_version").fetchone() == ("20261016_0068",)assert conn.execute("SELECT count(*) FROM skill_usage_daily").fetchone() == (0,)assert {row[2] for row in conn.execute("PRAGMA foreign_key_list(skill_usage_daily)")} == {"scopes",diff --git a/tests/test_sqlite_state_migration.py b/tests/test_sqlite_state_migration.pyindex a9691d4eb..dc29af1c2 100644--- a/tests/test_sqlite_state_migration.py+++ b/tests/test_sqlite_state_migration.py@@ -38,7 +38,7 @@ from vibe.message_types import build_partial_index_predicatepytestmark = pytest.mark.no_sqlite_template-HEAD_REVISION = "20261016_0067"+HEAD_REVISION = "20261016_0068"# ``storage.models`` builds a bare ``MetaData()``, so its foreign keys are unnamed and# Alembic cannot re-emit them when batch mode recreates a table. Every migration that# rebuilds one therefore passes this convention; the rebuild simulated below has to pass@@ -3605,6 +3605,7 @@ def test_run_migrations_repairs_head_indexes_before_stamping_head(tmp_path: Pathconn.execute("drop index if exists ix_messages_vibey_recovery_input")conn.execute("drop index if exists ix_message_deliveries_message")conn.execute("drop index if exists ix_agent_sessions_scope_status_activity")+ conn.execute("drop index if exists ix_agent_sessions_agent_status")conn.execute("drop index if exists ix_messages_session_created_id")conn.execute("drop index if exists ix_messages_session_type_created_id")conn.execute("drop index if exists ix_messages_platform_session_created_id")@@ -3622,6 +3623,7 @@ def test_run_migrations_repairs_head_indexes_before_stamping_head(tmp_path: Pathwith sqlite3.connect(db_path) as conn:assert "context_seq is null" in _index_sql(conn, "ix_messages_vibey_recovery_input")assert "message_id is not null" in _index_sql(conn, "ix_message_deliveries_message")+ assert "(agent_status, status)" in _index_sql(conn, "ix_agent_sessions_agent_status")run_migrations(db_path)diff --git a/tests/test_web_push_provider_invalidation_migration.py b/tests/test_web_push_provider_invalidation_migration.pyindex 832ef51a0..1b8c08120 100644--- a/tests/test_web_push_provider_invalidation_migration.py+++ b/tests/test_web_push_provider_invalidation_migration.py@@ -69,7 +69,7 @@ def test_upgrade_keeps_legacy_disabled_failure_opted_out(tmp_path):web_push_service.mark_send_failure(conn, endpoint=enabled["endpoint"], disable=True)with sqlite3.connect(path) as conn:assert conn.execute("select version_num from alembic_version").fetchone() == (- "20261016_0067",+ "20261016_0068",)command.downgrade(migrations.alembic_config(path), "20260907_0061")