Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(persistence): let a failed reconcile statement stop being permanent (#1509)

Raghav Chamadiya528296acdc51
diff --git a/packages/core/src/repowise/core/persistence/database.py b/packages/core/src/repowise/core/persistence/database.py
index a18509f98..d47b1e7cc 100644
--- a/packages/core/src/repowise/core/persistence/database.py
+++ b/packages/core/src/repowise/core/persistence/database.py
@@ -12,10 +12,11 @@ Call init_db() once at startup to create all tables and the FTS index.
from __future__ import annotations
import os
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Callable
from contextlib import asynccontextmanager
from pathlib import Path
+import structlog
from sqlalchemy import event, inspect
from sqlalchemy.ext.asyncio import (
AsyncEngine,
@@ -29,6 +30,8 @@ from sqlalchemy.sql import text
from .models import Base
+log = structlog.get_logger(__name__)
+
# SQLite tuning. WAL allows concurrent readers while a writer is active and
# turns the "database is locked" failure mode into a polite block, while the
# busy_timeout gives that block a bounded retry window before giving up.
@@ -319,13 +322,53 @@ def _reconcile_schema(connection: object) -> None:
* Postgres extensions / functions (e.g. pgvector) are NOT created
here — those still belong in Alembic migrations.
- The function is invoked inside the same transactional block as
- ``create_all`` and uses a sync connection (via ``run_sync``) so the
- SQLAlchemy DDL compilers work directly.
+ Uses a sync connection (via ``run_sync``) so the SQLAlchemy DDL compilers
+ work directly.
+
+ **Not atomic on SQLite, deliberately convergent instead.** ``init_db``
+ opens a transactional block, but pysqlite does not begin a transaction for
+ DDL, so every ``ALTER TABLE`` and ``CREATE INDEX`` here autocommits as it
+ runs. A statement that fails therefore cannot be rolled back, and aborting
+ the loop would strand every table ordered after the victim in
+ ``Base.metadata`` on that call and on every later one: a permanently
+ half-migrated store. So a failed statement is recorded and the walk
+ continues, and the first failure is re-raised once the walk is done. Two
+ properties follow, both wanted:
+
+ * calling again makes progress, because the tables after the victim are
+ reached on the first call rather than never;
+ * the caller still sees the error, so a write path fails as loudly as it
+ did before. The read paths pair this with
+ ``reconcile_schema_best_effort``, which swallows it on purpose.
+
+ On a backend with transactional DDL (PostgreSQL) the first failure poisons
+ the transaction, so continuing is pointless and the error is raised at
+ once, which keeps that backend's existing all-or-nothing behaviour.
"""
inspector = inspect(connection)
db_tables = set(inspector.get_table_names())
dialect = connection.dialect # type: ignore[attr-defined]
+ # Only SQLite autocommits DDL, which is what makes partial progress real
+ # rather than something a rollback undoes on the way out.
+ continue_past_failure = dialect.name == "sqlite" # type: ignore[attr-defined]
+ failures: list[tuple[str, Exception]] = []
+
+ def _run(what: str, build: Callable[[], object]) -> None:
+ # ``build`` renders the statement as well as running it, because
+ # compiling a column's type can fail on its own and that failure has
+ # to strand no more than compiling it successfully and failing to
+ # execute it would.
+ try:
+ connection.execute(build()) # type: ignore[attr-defined]
+ except Exception as exc: # re-raised below, once the walk is done
+ if not continue_past_failure:
+ raise
+ log.warning(
+ "schema_reconcile_statement_failed",
+ statement=what,
+ error=str(exc),
+ )
+ failures.append((what, exc))
for table in Base.metadata.tables.values():
if table.name not in db_tables:
@@ -344,9 +387,12 @@ def _reconcile_schema(connection: object) -> None:
for column in table.columns:
if column.name in db_cols:
continue
- column_ddl = _add_column_ddl(column, dialect)
- connection.execute( # type: ignore[attr-defined]
- text(f'ALTER TABLE "{table.name}" ADD COLUMN {column_ddl}')
+ _run(
+ f"{table.name}.{column.name}",
+ lambda table=table, column=column: text(
+ f'ALTER TABLE "{table.name}" ADD COLUMN '
+ f"{_add_column_ddl(column, dialect)}"
+ ),
)
# --- Indexes ---------------------------------------------------
@@ -358,7 +404,19 @@ def _reconcile_schema(connection: object) -> None:
for index in table.indexes:
if index.name in db_indexes:
continue
- connection.execute(CreateIndex(index)) # type: ignore[attr-defined]
+ _run(
+ f"{table.name}:{index.name}",
+ lambda index=index: CreateIndex(index),
+ )
+
+ if failures:
+ what = ", ".join(name for name, _ in failures)
+ log.warning(
+ "schema_reconcile_incomplete",
+ failed=len(failures),
+ statements=what,
+ )
+ raise failures[0][1]
async def init_db(engine: AsyncEngine) -> None:
diff --git a/packages/server/src/repowise/server/mcp_server/_failure_shield.py b/packages/server/src/repowise/server/mcp_server/_failure_shield.py
index 7b7cb1207..73e00948d 100644
--- a/packages/server/src/repowise/server/mcp_server/_failure_shield.py
+++ b/packages/server/src/repowise/server/mcp_server/_failure_shield.py
@@ -97,11 +97,20 @@ def _shape_stale_index(exc: Exception) -> dict[str, Any]:
"""Success-shaped response for a store whose schema predates this repowise.
``init_db`` back-fills *additive* drift on every read path, so reaching
- here means the drift is the kind ``_reconcile_schema`` deliberately does
- not touch (a removed or renamed column, a changed type — see
- ``database.py``). The generic internal-error shape is the wrong answer for
- it: it tells the caller to stop using the tool for the session, when one
- ``repowise update`` fixes it.
+ here has two causes, not one:
+
+ * the drift is the kind ``_reconcile_schema`` deliberately does not touch
+ (a removed or renamed column, a changed type: see ``database.py``);
+ * or the reconcile ran and one of its statements failed, leaving this
+ column still missing. The reconciler continues past a failure and
+ converges over repeated calls, so this cause can clear on its own.
+
+ Which one it is cannot be told apart from here, and that is why the remedy
+ leads with a retry rather than with ``repowise update``: telling a caller
+ to re-index a store whose reconcile is failing sends them somewhere that
+ fails the same way. The generic internal-error shape is still the wrong
+ answer for either cause, because it tells the caller to stop using the tool
+ for the whole session when the store is very likely repairable.
"""
return {
# The driver's own message: SQLAlchemy appends the whole compiled
@@ -112,14 +121,19 @@ def _shape_stale_index(exc: Exception) -> dict[str, Any]:
f"repowise: {_driver_message(exc).splitlines()[0].strip()}"
),
"remedy": (
- "The user can rebuild it by running 'repowise update' in the repo "
- "root. Re-indexing is the user's decision — suggest it once, do "
- "not run it yourself."
+ "Retry this call once: the schema repair runs on every read and "
+ "makes progress each time, so a partial repair can finish on the "
+ "next call. If it returns this notice again, the user can rebuild "
+ "the index with 'repowise update' in the repo root, or "
+ "'repowise init --force' if update reports the same error. "
+ "Re-indexing is the user's decision: suggest it once, do not run "
+ "it yourself."
),
"guidance": (
- "Until the index is rebuilt, every repowise tool will return this "
- "notice. Answer questions about this repo with your built-in "
- "tools (Read/Grep/Glob) for the rest of the session."
+ "If the retry returns this notice too, every repowise tool will "
+ "keep returning it until the index is rebuilt. Answer questions "
+ "about this repo with your built-in tools (Read/Grep/Glob) for the "
+ "rest of the session."
),
}
diff --git a/tests/unit/persistence/test_schema_reconciliation.py b/tests/unit/persistence/test_schema_reconciliation.py
index fee779b5e..9b8dc8052 100644
--- a/tests/unit/persistence/test_schema_reconciliation.py
+++ b/tests/unit/persistence/test_schema_reconciliation.py
@@ -250,6 +250,157 @@ async def test_init_db_is_idempotent(tmp_path: Path) -> None:
assert len(rows) == len(cols), "duplicate column names after re-running init_db"
+def _backfillable_column(table: Any) -> Any:
+ """The first non-PK column on *table* the reconciler can safely back-fill."""
+ for column in table.columns:
+ if column.primary_key:
+ continue
+ if (
+ column.nullable
+ or column.server_default is not None
+ or (
+ column.default is not None
+ and getattr(column.default, "arg", None) is not None
+ and not callable(getattr(column.default, "arg", None))
+ )
+ ):
+ return column
+ return None
+
+
+def _two_drop_targets() -> tuple[str, str, str, str]:
+ """Two (table, column) pairs, the second ordered after the first.
+
+ ``Base.metadata.tables`` preserves model-definition order, which is the
+ order ``_reconcile_schema`` walks. Picking a victim and a target after it
+ is what makes "a failure on the victim did not strand the rest" testable.
+ """
+ found: list[tuple[str, str]] = []
+ for table in Base.metadata.tables.values():
+ column = _backfillable_column(table)
+ if column is not None:
+ found.append((table.name, column.name))
+ if len(found) == 2:
+ break
+ assert len(found) == 2, "metadata has fewer than two back-fillable tables"
+ return found[0][0], found[0][1], found[1][0], found[1][1]
+
+
+async def test_failed_statement_does_not_strand_later_tables(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """A statement that fails mid-loop must not abort the whole reconcile.
+
+ SQLite DDL autocommits statement by statement, so an abort leaves the
+ store half-migrated permanently: every table ordered after the victim is
+ unreachable on that call and on every later one. The reconciler therefore
+ continues past a failure and reports it at the end, which is what lets a
+ repeated call converge.
+ """
+ from repowise.core.persistence import database as db_module
+
+ victim_table, victim_column, later_table, later_column = _two_drop_targets()
+
+ db_path = tmp_path / "wiki.db"
+ engine = create_engine(f"sqlite+aiosqlite:///{db_path}")
+ try:
+ await init_db(engine)
+ finally:
+ await engine.dispose()
+
+ try:
+ _execute(db_path, f'ALTER TABLE "{victim_table}" DROP COLUMN "{victim_column}"')
+ _execute(db_path, f'ALTER TABLE "{later_table}" DROP COLUMN "{later_column}"')
+ except sqlite3.OperationalError as exc:
+ pytest.skip(f"SQLite build doesn't support DROP COLUMN: {exc}")
+
+ real_add_column_ddl = db_module._add_column_ddl
+
+ def _poisoned(column: Any, dialect: Any) -> str:
+ if column.name == victim_column and column.table.name == victim_table:
+ # Deliberately unparseable, so the ALTER fails the way a real
+ # unsupported type or a locked table would.
+ return f'"{column.name}" NOT_A_REAL_TYPE DEFAULT'
+ return real_add_column_ddl(column, dialect)
+
+ monkeypatch.setattr(db_module, "_add_column_ddl", _poisoned)
+
+ # The failure is still reported: write paths must not silently succeed.
+ engine = create_engine(f"sqlite+aiosqlite:///{db_path}")
+ with pytest.raises(Exception): # noqa: B017 - the driver's own error type
+ try:
+ await init_db(engine)
+ finally:
+ await engine.dispose()
+
+ assert victim_column not in _table_columns(db_path, victim_table)
+ assert later_column in _table_columns(db_path, later_table), (
+ "a failure on an earlier table stranded every table after it"
+ )
+
+
+async def test_second_reconcile_finishes_what_the_first_could_not(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Repeated calls converge, even while the failing statement keeps failing.
+
+ The failure that strands a reconcile is usually not transient, so "call it
+ again" has to make progress on its own. Only once the cause clears does the
+ victim itself get picked up.
+ """
+ from repowise.core.persistence import database as db_module
+
+ victim_table, victim_column, later_table, later_column = _two_drop_targets()
+
+ db_path = tmp_path / "wiki.db"
+ engine = create_engine(f"sqlite+aiosqlite:///{db_path}")
+ try:
+ await init_db(engine)
+ finally:
+ await engine.dispose()
+
+ try:
+ _execute(db_path, f'ALTER TABLE "{victim_table}" DROP COLUMN "{victim_column}"')
+ _execute(db_path, f'ALTER TABLE "{later_table}" DROP COLUMN "{later_column}"')
+ except sqlite3.OperationalError as exc:
+ pytest.skip(f"SQLite build doesn't support DROP COLUMN: {exc}")
+
+ real_add_column_ddl = db_module._add_column_ddl
+
+ def _poisoned(column: Any, dialect: Any) -> str:
+ if column.name == victim_column and column.table.name == victim_table:
+ return f'"{column.name}" NOT_A_REAL_TYPE DEFAULT'
+ return real_add_column_ddl(column, dialect)
+
+ monkeypatch.setattr(db_module, "_add_column_ddl", _poisoned)
+ for _ in range(2):
+ engine = create_engine(f"sqlite+aiosqlite:///{db_path}")
+ with pytest.raises(Exception): # noqa: B017
+ try:
+ await init_db(engine)
+ finally:
+ await engine.dispose()
+
+ # This is the D7 measurement verbatim: "after a SECOND reconcile, later
+ # table fixed?" used to be False forever.
+ assert later_column in _table_columns(db_path, later_table)
+ assert victim_column not in _table_columns(db_path, victim_table)
+
+ # Whatever made the statement fail is gone (a lock released, a driver
+ # upgraded); the next call must pick the victim back up.
+ monkeypatch.setattr(db_module, "_add_column_ddl", real_add_column_ddl)
+ engine = create_engine(f"sqlite+aiosqlite:///{db_path}")
+ try:
+ await init_db(engine)
+ finally:
+ await engine.dispose()
+
+ assert victim_column in _table_columns(db_path, victim_table)
+ assert later_column in _table_columns(db_path, later_table)
+
+
@pytest.mark.asyncio
async def test_reconciler_handles_arbitrary_new_column(tmp_path: Path) -> None:
"""Forward-compat contract: simulate a *future* migration by dropping any