Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
feat(storage): count the SQLite write transactions the controller loop runs (#2710) (#2745)
Call Whiteb7131c8eba47
diff --git a/core/controller.py b/core/controller.pyindex d7b0a160b..1d156d4af 100644--- a/core/controller.py+++ b/core/controller.py@@ -52,6 +52,7 @@ from core.vibe_agents import VibeAgent, VibeAgentStorefrom core.blocking import run_blockingfrom vibe.i18n import get_supported_languages, t as i18n_tfrom core.loop_watchdog import EventLoopWatchdog+from storage import loop_writersfrom vibe.runtime import current_process_owns_service_instance, mark_service_instance_startedif TYPE_CHECKING:@@ -2181,6 +2182,8 @@ class Controller:# This thread runs the loop: the one thread an activation commit may hold# the runtime boundary across its I/O on.self.runtime_activation.bind_loop_thread()+ # The thread whose SQLite write transactions are counted (#2710).+ loop_writers.bind_loop_thread()# Names whatever blocks this loop while it blocks; see core/loop_watchdog.py.self._loop_watchdog = EventLoopWatchdog(self._loop)self._loop_watchdog.start()@@ -2219,6 +2222,9 @@ class Controller:except Exception as e:logger.error(f"Error in main run loop: {e}", exc_info=True)finally:+ # The loop no longer serves Sessions; cleanup that reuses its thread is+ # not a loop writer (#2710).+ loop_writers.unbind_loop_thread()self.cleanup_sync()if not getattr(self, "_shutdown_tainted", False):self._service_lock_safe_to_release = True@@ -2704,6 +2710,7 @@ class Controller:except Exception as e:logger.warning("OpenCode server cleanup failed: %s", e)+ loop_writers.log_summary(final=True)logger.info("Controller cleanup (sync) complete")def _model_hub_snapshot_refresh_completed(self) -> None:diff --git a/core/session_turns.py b/core/session_turns.pyindex 3a1227f40..54e79ac87 100644--- a/core/session_turns.py+++ b/core/session_turns.py@@ -65,7 +65,7 @@ from core.services.agent_steering import (result as steer_result,steer_active_turn,)-from storage import messages_service+from storage import loop_writers, messages_servicefrom storage import message_deliveries as delivery_storefrom storage.agent_transcript import recovered_watch_idsfrom storage.agent_session_rows import reserve_write_lock@@ -638,6 +638,7 @@ def _try_reserve_write_lock(conn: Connection) -> bool:"""``reserve_write_lock`` without waiting: ``False`` when another connection holdsSQLite's writer, instead of blocking the caller for the busy timeout."""+ loop_writers.mark_never_waits(conn)previous = conn.exec_driver_sql("PRAGMA busy_timeout").scalar()conn.exec_driver_sql("PRAGMA busy_timeout = 0")try:diff --git a/docs/plans/controller-loop-sqlite-writers.md b/docs/plans/controller-loop-sqlite-writers.mdindex c9b85b6f8..f74f0bc76 100644--- a/docs/plans/controller-loop-sqlite-writers.md+++ b/docs/plans/controller-loop-sqlite-writers.md@@ -90,7 +90,7 @@ Because producers are gated and the function already yields, moving the remainin## Other writer sites on the loop (scope only)-This list holds the sites known so far and is **not exhaustive**: writers are reached through many indirections, and two review rounds each found more. The complete inventory will be measured at runtime rather than reviewed by hand (#2710), and this section will cite it then.+This list holds the sites known so far and is **not exhaustive**: writers are reached through many indirections, and two review rounds each found more. The complete inventory is measured at runtime rather than reviewed by hand (#2710). `storage/loop_writers.py` counts every write transaction the controller loop runs, by code location, and logs the ranked list ("Controller-loop SQLite writers") every five minutes while writes arrive and at shutdown. This section will cite that list after a soak.The per-Turn admissions all hold the boundary inside the transaction, so they are loop-side boundary holders and fall under B's split analysis:- `_admit_p0`, `_admit_p1`, `_admit_p3`, `_promote_fifo_head`;diff --git a/storage/agent_session_rows.py b/storage/agent_session_rows.pyindex 25e5c4bc3..485c08535 100644--- a/storage/agent_session_rows.py+++ b/storage/agent_session_rows.py@@ -10,6 +10,7 @@ from typing import Anyfrom sqlalchemy import Connection, func, select, update+from storage import loop_writersfrom storage.models import agent_sessions, scope_settingsfrom storage.session_reclaim import (OVERRIDABLE_SETTING_COLUMNS,@@ -202,6 +203,9 @@ def reserve_write_lock(conn: Connection) -> None:if conn.connection.dbapi_connection.in_transaction:conn.exec_driver_sql(_WRITE_LOCK_RESERVATION_SQL)return+ # A timed-out wait for the writer leaves no transaction, but it held the+ # caller for the busy timeout: the loop-writer census still counts it.+ loop_writers.mark_write_attempt(conn)conn.exec_driver_sql("BEGIN IMMEDIATE")diff --git a/storage/db.py b/storage/db.pyindex 9c28be208..4f7d6119a 100644--- a/storage/db.py+++ b/storage/db.py@@ -7,6 +7,7 @@ from sqlalchemy import Engine, create_engine, eventfrom sqlalchemy.engine import URLfrom config import paths+from storage import loop_writersdef escape_sql_like(value: str) -> str:@@ -47,7 +48,14 @@ def create_sqlite_engine(db_path: Path | None = None, *, read_only: bool = FalseURL.create("sqlite", database=path.as_uri(), query={"mode": "ro", "uri": "true"})if read_only else sqlite_url(path))- engine = create_engine(url, future=True, hide_parameters=True)+ # Every connection is a ``CensusConnection`` so the loop-writer census (#2710)+ # can time a loop transaction to the end of SQLite's own commit.+ engine = create_engine(+ url,+ future=True,+ hide_parameters=True,+ connect_args={"factory": loop_writers.CensusConnection},+ )@event.listens_for(engine, "connect")def _set_sqlite_pragmas(dbapi_connection, _connection_record) -> None:@@ -58,6 +66,7 @@ def create_sqlite_engine(db_path: Path | None = None, *, read_only: bool = Falsecursor.execute("PRAGMA busy_timeout = 5000")cursor.close()+ loop_writers.watch_engine(engine)return enginediff --git a/storage/loop_writers.py b/storage/loop_writers.pynew file mode 100644index 000000000..a3a375368--- /dev/null+++ b/storage/loop_writers.py@@ -0,0 +1,238 @@+"""Measure which SQLite write transactions run on the controller loop (#2710).++A write on the controller loop stalls every Session while it waits for SQLite's+writer or for its own commit, and the writers are reached through so many+indirections that no reviewed list of them stays complete. So the engine factory+counts them instead. ``storage.db.create_sqlite_engine`` opens every connection+as a ``CensusConnection`` and listens to the pool, never to the connection: a+SQLAlchemy connection event would put every statement of every thread on the+slower event-dispatch path.++- A pool checkout on the controller loop's thread starts a record; a checkout on+ any other thread does nothing.+- ``CensusConnection.commit`` and ``rollback`` end it. They keep only the+ transactions that wrote (the connection is in a transaction only after a write+ or ``BEGIN IMMEDIATE``) or that tried to take the writer and failed+ (``mark_write_attempt``). They add the loop time since the record started, up to+ the end of SQLite's own commit or rollback (``fsync`` included), to the totals of+ the code location that ended the transaction, and start the next record.++The location is two frames: the first outside SQLAlchemy, ``contextlib`` and this+module, and its caller, so a write behind a shared transaction helper still names+who used the helper. The walk is bounded and keyed by ``(filename, line, name)``;+nothing is formatted until a report. The totals live in memory only. A location is+logged once when it first appears, a ranked summary at most every+``SUMMARY_INTERVAL_SECONDS`` while new writes arrive, and a complete one at shutdown.++Not counted: writes SQLite runs outside a transaction, such as DDL and+``PRAGMA journal_mode`` (#2747).++A write that cannot wait for another writer, because it takes the writer with a+zero busy timeout (``_try_reserve_write_lock``), is marked with+``mark_never_waits`` and counted apart: it cannot stall the loop on another+process's writer.+"""++from __future__ import annotations++import contextlib+import logging+import os+import sqlite3+import sys+import threading+import time+from typing import Any, Callable++import sqlalchemy+from sqlalchemy import Engine, event++logger = logging.getLogger(__name__)++SUMMARY_INTERVAL_SECONDS = 300.0+# Sites listed in a summary, by total loop time.+SUMMARY_SITES = 25+# Frames inspected per write while looking for the first location outside the+# internal modules, and locations kept from there.+_MAX_FRAMES = 16+_SITE_DEPTH = 2++_Frame = tuple[str, int, str]+_Site = tuple[_Frame, ...]+++_INTERNAL_PREFIXES = (+ os.path.dirname(os.path.abspath(sqlalchemy.__file__)) + os.sep,+ os.path.abspath(contextlib.__file__),+ os.path.abspath(__file__),+)+_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + os.sep++_lock = threading.Lock()+_loop_thread: int | None = None+# site -> [writes, total seconds, max seconds, never-waits writes]+_totals: dict[_Site, list[float]] = {}+_changed_since_summary = False+_last_summary_at = 0.0+++def bind_loop_thread(thread_id: int | None = None) -> None:+ """Record the controller loop's thread; only transactions opened on it are counted."""++ global _loop_thread+ _loop_thread = threading.get_ident() if thread_id is None else thread_id+++def unbind_loop_thread() -> None:+ """Stop counting: the loop no longer serves Sessions, so its writes stall none."""++ global _loop_thread+ _loop_thread = None+++class CensusConnection(sqlite3.Connection):+ """A sqlite3 connection that ends the census's record of a loop transaction."""++ _census: list[Any] | None = None++ def commit(self) -> None:+ self._end(super().commit)++ def rollback(self) -> None:+ self._end(super().rollback)++ def _end(self, finish: Callable[[], None]) -> None:+ record = self._census+ if record is None or threading.get_ident() != _loop_thread:+ finish()+ return+ wrote = self.in_transaction or record[2]+ try:+ finish()+ except BaseException:+ if self.in_transaction:+ # A commit that failed with the transaction still open (a deferred+ # constraint, say): the rollback that follows ends it, and counts it once.+ raise+ self._close(record, wrote)+ raise+ self._close(record, wrote)++ def _close(self, record: list[Any], wrote: bool) -> None:+ now = time.perf_counter()+ # The next transaction on this checkout starts where this one ended.+ self._census = [now, False, False]+ if wrote:+ _count(_site(sys._getframe(1)), now - record[0], never_waits=record[1])+++def watch_engine(engine: Engine) -> None:+ event.listen(engine, "checkout", _on_checkout)+++def _record(conn: Any) -> list[Any] | None:+ return getattr(conn.connection.dbapi_connection, "_census", None)+++def mark_never_waits(conn: Any) -> None:+ """Mark the loop transaction on ``conn`` as one that never waits for another writer."""++ record = _record(conn)+ if record is not None:+ record[1] = True+++def mark_write_attempt(conn: Any) -> None:+ """Count the loop transaction on ``conn`` even if taking the writer fails.++ A ``BEGIN IMMEDIATE`` that times out waiting for another writer leaves no+ transaction behind, yet the loop waited the whole busy timeout for it.+ """++ record = _record(conn)+ if record is not None:+ record[2] = True+++def _on_checkout(dbapi_connection: Any, _record: Any, _proxy: Any) -> None:+ if threading.get_ident() != _loop_thread or not isinstance(dbapi_connection, CensusConnection):+ return+ dbapi_connection._census = [time.perf_counter(), False, False]+++def _site(frame: Any) -> _Site:+ found: list[_Frame] = []+ for _ in range(_MAX_FRAMES):+ if frame is None:+ break+ code = frame.f_code+ if not code.co_filename.startswith(_INTERNAL_PREFIXES):+ found.append((code.co_filename, frame.f_lineno, code.co_name))+ if len(found) == _SITE_DEPTH:+ break+ frame = frame.f_back+ return tuple(found)+++def _count(site: _Site, elapsed: float, *, never_waits: bool) -> None:+ global _changed_since_summary+ with _lock:+ totals = _totals.get(site)+ first = totals is None+ if first:+ totals = _totals[site] = [0, 0.0, 0.0, 0]+ totals[0] += 1+ totals[1] += elapsed+ if elapsed > totals[2]:+ totals[2] = elapsed+ if never_waits:+ totals[3] += 1+ _changed_since_summary = True+ if first:+ logger.info(+ "Controller-loop SQLite write site: %s (%.1f ms%s)",+ _format_site(site),+ elapsed * 1000,+ ", never waits" if never_waits else "",+ )+ if time.monotonic() - _last_summary_at >= SUMMARY_INTERVAL_SECONDS:+ log_summary()+++def log_summary(*, final: bool = False) -> None:+ """Log the write sites ranked by total loop time.++ A periodic summary lists the top ``SUMMARY_SITES`` and is skipped when nothing+ was written since the last one. The final one at shutdown lists every site, so+ the inventory survives log rotation, and it reports even a run with no writes.+ """++ global _changed_since_summary, _last_summary_at+ with _lock:+ if not (_changed_since_summary or final):+ return+ _changed_since_summary = False+ _last_summary_at = time.monotonic()+ ranked = sorted(_totals.items(), key=lambda item: item[1][1], reverse=True)+ writes = sum(int(totals[0]) for _site, totals in ranked)+ lines = [+ f"{int(count)} writes, {total * 1000:.0f} ms total, {peak * 1000:.1f} ms max"+ f"{f', {int(never)} never wait' if never else ''}: {_format_site(site)}"+ for site, (count, total, peak, never) in (ranked if final else ranked[:SUMMARY_SITES])+ ]+ logger.info(+ "Controller-loop SQLite writers%s: %d writes at %d sites, by total loop time%s%s",+ " at shutdown" if final else "",+ writes,+ len(ranked),+ f" (top {SUMMARY_SITES})" if len(lines) < len(ranked) else "",+ "".join(f"\n {line}" for line in lines),+ )+++def _format_site(site: _Site) -> str:+ if not site:+ return "unknown"+ return " <- ".join(+ f"{filename.removeprefix(_REPO_ROOT)}:{line} in {name}" for filename, line, name in site+ )diff --git a/tests/test_loop_writers.py b/tests/test_loop_writers.pynew file mode 100644index 000000000..eab35b6b9--- /dev/null+++ b/tests/test_loop_writers.py@@ -0,0 +1,258 @@+"""The controller-loop SQLite writer census (#2710).++The census replaces a hand-kept list of loop-side writers that review kept finding+incomplete, so what it must get right is which transactions it counts: every write+transaction the loop runs, under the code location that ran it, and nothing else.+"""++from __future__ import annotations++import logging+import sqlite3+import sys+import threading++import pytest+from sqlalchemy import text++from sqlalchemy.exc import IntegrityError, OperationalError++from core.session_turns import _try_reserve_write_lock+from storage import loop_writers+from storage.agent_session_rows import reserve_write_lock+from storage.db import create_sqlite_engine+++def census(monkeypatch, tmp_path):+ monkeypatch.setattr(loop_writers, "_totals", {})+ monkeypatch.setattr(loop_writers, "_changed_since_summary", False)+ monkeypatch.setattr(loop_writers, "_last_summary_at", float("inf"))+ monkeypatch.setattr(loop_writers, "_loop_thread", None)+ loop_writers.bind_loop_thread()+ engine = create_sqlite_engine(tmp_path / "state.sqlite")+ with engine.begin() as conn:+ conn.execute(text("CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT)"))+ loop_writers._totals.clear()+ yield engine+ engine.dispose()+++def _sites() -> dict[str, list[float]]:+ return {+ frames[0][2]: totals+ for frames, totals in loop_writers._totals.items()+ }+++def _explicit_write(engine) -> None:+ with engine.begin() as conn:+ conn.execute(text("INSERT INTO notes (body) VALUES ('a')"))+++def _autobegun_write(engine) -> None:+ with engine.connect() as conn:+ conn.execute(text("UPDATE notes SET body = 'b'"))+ conn.commit()+++def _read(engine) -> None:+ with engine.connect() as conn:+ conn.execute(text("SELECT count(*) FROM notes")).scalar()+++def _rolled_back_write(engine) -> None:+ with engine.connect() as conn:+ conn.execute(text("DELETE FROM notes"))+ conn.rollback()+++def _never_waiting_write(engine) -> None:+ with engine.begin() as conn:+ assert _try_reserve_write_lock(conn)+ conn.execute(text("INSERT INTO notes (body) VALUES ('c')"))+++def _timed_out_writer_wait(engine, db_path) -> None:+ """``reserve_write_lock`` while another connection holds the writer: the wait+ times out and leaves no transaction, but it held the loop."""++ holder = sqlite3.connect(db_path)+ holder.execute("BEGIN IMMEDIATE")+ try:+ with engine.connect() as conn:+ conn.exec_driver_sql("PRAGMA busy_timeout = 50")+ try:+ with pytest.raises(OperationalError, match="locked"):+ reserve_write_lock(conn)+ finally:+ conn.exec_driver_sql("PRAGMA busy_timeout = 5000")+ finally:+ holder.rollback()+ holder.close()+++def test_the_census_counts_exactly_the_loops_write_transactions(census) -> None:+ """Every write the loop thread runs is counted under the function that ran it,+ whether it began explicitly or on its first statement, committed or rolled back.+ Reads and writes on other threads are not counted: they do not hold the loop."""++ for _ in range(2):+ _explicit_write(census)+ _autobegun_write(census)+ _rolled_back_write(census)+ _read(census)+ _never_waiting_write(census)+ _timed_out_writer_wait(census, census.url.database)+ off_loop = threading.Thread(target=_explicit_write, args=(census,))+ off_loop.start()+ off_loop.join()+ loop_writers.unbind_loop_thread()+ _autobegun_write(census)++ sites = _sites()+ assert set(sites) == {+ "_explicit_write",+ "_autobegun_write",+ "_rolled_back_write",+ "_never_waiting_write",+ "_timed_out_writer_wait",+ }+ assert sites["_autobegun_write"][0] == 1, "a write after the loop stopped serving was counted"+ assert sites["_explicit_write"][0] == 2, "the off-loop write was counted, or a loop write was missed"+ assert all(totals[1] > 0 and totals[2] > 0 for totals in sites.values())+ never_waits = {name for name, totals in sites.items() if totals[3]}+ assert never_waits == {"_never_waiting_write"}+ assert sites["_timed_out_writer_wait"][1] >= 0.05, "the timed-out wait was not timed"+++def test_a_write_is_timed_to_the_end_of_sqlites_commit(census, monkeypatch) -> None:+ """The loop time of a write includes SQLite's own commit, ``fsync`` and all: the+ census records it only once the commit has finished, not when SQLAlchemy is+ about to issue it."""++ seen: list[bool] = []+ original = loop_writers._count++ def count(site, elapsed, *, never_waits):+ seen.append(bool(holder["conn"].in_transaction))+ original(site, elapsed, never_waits=never_waits)++ holder: dict = {}+ monkeypatch.setattr(loop_writers, "_count", count)+ with census.begin() as conn:+ holder["conn"] = conn.connection.dbapi_connection+ conn.execute(text("INSERT INTO notes (body) VALUES ('d')"))++ assert seen == [False], "the write was counted before its commit finished"+++def test_a_site_names_the_caller_of_a_shared_transaction_helper(census) -> None:+ """A write behind a helper is reported with the helper's caller too, so a helper+ shared by many writers does not hide which of them ran on the loop."""++ def caller_one():+ _explicit_write(census)++ def caller_two():+ _explicit_write(census)++ caller_one()+ caller_two()++ callers = sorted(frames[1][2] for frames in loop_writers._totals)+ assert callers == ["caller_one", "caller_two"]+++def test_the_census_reports_new_sites_once_and_ranks_the_summary(census, caplog) -> None:+ caplog.set_level(logging.INFO, logger=loop_writers.__name__)++ for _ in range(2):+ _explicit_write(census)+ _autobegun_write(census)+ loop_writers.log_summary()+ loop_writers.log_summary()+ loop_writers.log_summary(final=True)++ messages = [record.getMessage() for record in caplog.records]+ first_seen = [message for message in messages if message.startswith("Controller-loop SQLite write site:")]+ assert len(first_seen) == 2, "a site was reported more than once, or not at all"+ summaries = [message for message in messages if message.startswith("Controller-loop SQLite writers")]+ assert len(summaries) == 2, "an unchanged periodic summary was logged, or the final one was skipped"+ assert "3 writes at 2 sites" in summaries[0]+ assert summaries[1].startswith("Controller-loop SQLite writers at shutdown")+ assert "tests/test_loop_writers.py:" in summaries[0]+++def test_a_commit_that_fails_counts_its_transaction_once(census) -> None:+ """A commit can fail with the transaction still open, as a deferred constraint+ does. The rollback that follows ends that transaction, and it is counted once."""++ with census.begin() as conn:+ conn.execute(text("CREATE TABLE parents (id INTEGER PRIMARY KEY)"))+ conn.execute(+ text(+ "CREATE TABLE children (id INTEGER PRIMARY KEY, parent_id INTEGER "+ "REFERENCES parents (id) DEFERRABLE INITIALLY DEFERRED)"+ )+ )+ loop_writers._totals.clear()++ def _deferred_violation():+ with pytest.raises(IntegrityError):+ with census.begin() as conn:+ conn.execute(text("INSERT INTO children (parent_id) VALUES (42)"))++ _deferred_violation()++ assert {name: int(totals[0]) for name, totals in _sites().items()} == {"_deferred_violation": 1}+++def test_the_shutdown_summary_lists_every_site(census, caplog, monkeypatch) -> None:+ """A periodic summary is capped; the final one keeps every site, so a rotated log+ cannot lose a rare writer from the inventory."""++ caplog.set_level(logging.INFO, logger=loop_writers.__name__)+ monkeypatch.setattr(loop_writers, "SUMMARY_SITES", 2)+ for name in ("writer_a", "writer_b", "writer_c"):+ namespace: dict = {"_explicit_write": _explicit_write}+ exec(f"def {name}(engine):\n _explicit_write(engine)\n", namespace)+ namespace[name](census)++ loop_writers.log_summary()+ loop_writers.log_summary(final=True)++ periodic, final = [record.getMessage() for record in caplog.records if "SQLite writers" in record.getMessage()]+ assert "(top 2)" in periodic and len(periodic.splitlines()) == 3+ assert "(top" not in final and len(final.splitlines()) == 4+++def test_the_final_summary_reports_a_run_without_writes(census, caplog) -> None:+ """A run that wrote nothing on the loop still ends with a census, so it cannot be+ mistaken for a census that never ran."""++ caplog.set_level(logging.INFO, logger=loop_writers.__name__)++ loop_writers.log_summary(final=True)++ assert [record.getMessage() for record in caplog.records] == [+ "Controller-loop SQLite writers at shutdown: 0 writes at 0 sites, by total loop time"+ ]+++def test_the_site_walk_is_bounded(census) -> None:+ """The walk inspects at most ``_MAX_FRAMES`` frames per write, so a deep stack+ inside the internal modules costs the same and reports an unknown site."""++ def internal_depth(depth: int):+ if depth:+ return internal_depth(depth - 1)+ return loop_writers._site(sys._getframe())++ original = loop_writers._INTERNAL_PREFIXES+ loop_writers._INTERNAL_PREFIXES = original + (__file__,)+ try:+ assert internal_depth(3 * loop_writers._MAX_FRAMES) == ()+ finally:+ loop_writers._INTERNAL_PREFIXES = original+ assert loop_writers._format_site(()) == "unknown"