Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(search): index a page's summary and target path (#1189)

Swati Ahuja4f067f3ce8bd
diff --git a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
index 8ce9a8bf4..d6850d25c 100644
--- a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
+++ b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
@@ -479,7 +479,13 @@ def _run_repo_checks(
select(Page).where(Page.id.in_(list(missing_from_fts)))
)
for page in rows.scalars().all():
- await fts.index(page.id, page.title, page.content)
+ await fts.index(
+ page.id,
+ page.title,
+ page.content,
+ summary=page.summary,
+ target_path=page.target_path,
+ )
repaired += 1
for pid in orphaned_fts:
await fts.delete(pid)
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
index 5568fc1ce..a3d05d93f 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
@@ -81,11 +81,23 @@ async def _index_preserved_pages(sf: Any, fts: Any, preserved_page_ids: set[str]
async with get_session(sf) as session:
rows = (
await session.execute(
- select(Page.id, Page.title, Page.content).where(Page.id.in_(batch))
+ select(
+ Page.id,
+ Page.title,
+ Page.content,
+ Page.summary,
+ Page.target_path,
+ ).where(Page.id.in_(batch))
)
).all()
- for page_id, title, content in rows:
- await fts.index(page_id, title or "", content or "")
+ for page_id, title, content, summary, target_path in rows:
+ await fts.index(
+ page_id,
+ title or "",
+ content or "",
+ summary=summary or "",
+ target_path=target_path or "",
+ )
except Exception as exc: # pragma: no cover - defensive
logger.debug("persist.preserved_fts_backfill_failed", error=str(exc))
@@ -207,7 +219,13 @@ async def persist_result(result: Any, repo_path: Path) -> None:
await fts.delete_many(swept_page_ids)
if fts is not None and result.generated_pages:
for page in result.generated_pages:
- await fts.index(page.page_id, page.title, page.content)
+ await fts.index(
+ page.page_id,
+ page.title,
+ page.content,
+ summary=page.summary,
+ target_path=page.target_path,
+ )
await _index_preserved_pages(sf, fts, getattr(result, "preserved_page_ids", None))
# Stamp the analysis (+ generation) phases in the resume ledger now that
diff --git a/packages/cli/src/repowise/cli/commands/restyle_cmd.py b/packages/cli/src/repowise/cli/commands/restyle_cmd.py
index ee70a4912..63f090e55 100644
--- a/packages/cli/src/repowise/cli/commands/restyle_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/restyle_cmd.py
@@ -132,7 +132,13 @@ async def _run_restyle(
fts = FullTextSearch(engine)
await fts.ensure_index()
for page in generated_pages:
- await fts.index(page.page_id, page.title, page.content)
+ await fts.index(
+ page.page_id,
+ page.title,
+ page.content,
+ summary=page.summary,
+ target_path=page.target_path,
+ )
except Exception:
pass # FTS indexing is best-effort
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
index 14677eab8..39f4d651c 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
@@ -310,7 +310,13 @@ async def _persist_async(
fts = FullTextSearch(engine)
await fts.ensure_index()
for page in generated_pages:
- await fts.index(page.page_id, page.title, page.content)
+ await fts.index(
+ page.page_id,
+ page.title,
+ page.content,
+ summary=page.summary,
+ target_path=page.target_path,
+ )
except Exception as exc:
degraded.append(f"Full-text index: {exc}")
finally:
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
index 200b3b6d1..27e449762 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
@@ -743,7 +743,13 @@ async def _persist_full_update_async(
fts = FullTextSearch(engine)
await fts.ensure_index()
for page in generated_pages:
- await fts.index(page.page_id, page.title, page.content)
+ await fts.index(
+ page.page_id,
+ page.title,
+ page.content,
+ summary=page.summary,
+ target_path=page.target_path,
+ )
except Exception as exc:
_skip("Full-text search indexing", exc)
return total_pages
diff --git a/packages/cli/src/repowise/cli/commands/upgrade_flow.py b/packages/cli/src/repowise/cli/commands/upgrade_flow.py
index 86578a1d2..52b0a4136 100644
--- a/packages/cli/src/repowise/cli/commands/upgrade_flow.py
+++ b/packages/cli/src/repowise/cli/commands/upgrade_flow.py
@@ -321,7 +321,13 @@ async def _run_upgrade(
fts = FullTextSearch(engine)
await fts.ensure_index()
for page in generated_pages:
- await fts.index(page.page_id, page.title, page.content)
+ await fts.index(
+ page.page_id,
+ page.title,
+ page.content,
+ summary=page.summary,
+ target_path=page.target_path,
+ )
except Exception:
pass # FTS indexing is best-effort
diff --git a/packages/cli/src/repowise/cli/commands/workspace_cmd.py b/packages/cli/src/repowise/cli/commands/workspace_cmd.py
index caab728a7..2ce5096f1 100644
--- a/packages/cli/src/repowise/cli/commands/workspace_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/workspace_cmd.py
@@ -726,7 +726,13 @@ def _generate_docs_for_added_repo(
fts = FullTextSearch(engine)
await fts.ensure_index()
for p in pages:
- await fts.index(p.page_id, p.title, p.content)
+ await fts.index(
+ p.page_id,
+ p.title,
+ p.content,
+ summary=p.summary,
+ target_path=p.target_path,
+ )
await engine.dispose()
return len(pages)
diff --git a/packages/core/alembic/versions/0044_fts_summary_and_target_path.py b/packages/core/alembic/versions/0044_fts_summary_and_target_path.py
new file mode 100644
index 000000000..2fe68ce33
--- /dev/null
+++ b/packages/core/alembic/versions/0044_fts_summary_and_target_path.py
@@ -0,0 +1,85 @@
+"""Full-text search covers a page's summary and target path.
+
+Two fields already decide whether a page is the right answer and could not
+cause it to be found. ``summary`` is a fresh LLM paraphrase written for every
+page — never a prefix of the prose it paraphrases — and the answer tool's
+coverage re-ranker reads it when ordering results, so it reorders candidates it
+cannot produce. ``target_path`` is the file the page documents; most page
+titles are that path verbatim, so a question that names a directory or a file
+matched only whatever the generated prose happened to mention.
+
+PostgreSQL: the GIN index is on an expression, so widening the searchable text
+means dropping the index and rebuilding it over the wider expression. The rows
+themselves do not move. ``search.py`` builds the identical expression at query
+time — if the two ever drift the planner silently stops using the index and
+falls back to a sequential scan, which still returns the right rows and is why
+the expression is defined in one place and read from there.
+
+SQLite: FTS5 has no ALTER TABLE, so the virtual table is dropped and refilled
+from ``wiki_pages``, which is the system of record for all four indexed fields.
+The old index could not supply the new columns — it never held them.
+
+``FullTextSearch.ensure_index`` performs the same upgrade at runtime, because
+most local stores are created by ``init_db`` and never see Alembic at all. Both
+paths are idempotent and converge on the same shape.
+
+Revision ID: 0044
+Revises: 0043
+Create Date: 2026-07-30
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+from repowise.core.persistence.search import PAGE_FTS_DDL, PG_FTS_EXPRESSION
+
+# revision identifiers
+revision: str = "0044"
+down_revision: str | None = "0043"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_OLD_PG_EXPRESSION = "to_tsvector('english', COALESCE(title,'') || ' ' || COALESCE(content,''))"
+
+_REFILL_SQL = (
+ "INSERT INTO page_fts(page_id, title, content, summary, target_path) "
+ "SELECT id, COALESCE(title,''), COALESCE(content,''), "
+ " COALESCE(summary,''), COALESCE(target_path,'') "
+ "FROM wiki_pages"
+)
+
+_OLD_REFILL_SQL = (
+ "INSERT INTO page_fts(page_id, title, content) "
+ "SELECT id, COALESCE(title,''), COALESCE(content,'') FROM wiki_pages"
+)
+
+_OLD_SQLITE_DDL = (
+ "CREATE VIRTUAL TABLE IF NOT EXISTS page_fts USING fts5(page_id UNINDEXED, title, content)"
+)
+
+
+def upgrade() -> None:
+ dialect = op.get_bind().dialect.name
+
+ if dialect == "postgresql":
+ op.execute("DROP INDEX IF EXISTS idx_wiki_pages_fts")
+ op.execute(f"CREATE INDEX idx_wiki_pages_fts ON wiki_pages USING GIN({PG_FTS_EXPRESSION})")
+ elif dialect == "sqlite":
+ op.execute("DROP TABLE IF EXISTS page_fts")
+ op.execute(PAGE_FTS_DDL)
+ op.execute(_REFILL_SQL)
+
+
+def downgrade() -> None:
+ dialect = op.get_bind().dialect.name
+
+ if dialect == "postgresql":
+ op.execute("DROP INDEX IF EXISTS idx_wiki_pages_fts")
+ op.execute(f"CREATE INDEX idx_wiki_pages_fts ON wiki_pages USING GIN({_OLD_PG_EXPRESSION})")
+ elif dialect == "sqlite":
+ op.execute("DROP TABLE IF EXISTS page_fts")
+ op.execute(_OLD_SQLITE_DDL)
+ op.execute(_OLD_REFILL_SQL)
diff --git a/packages/core/src/repowise/core/persistence/database.py b/packages/core/src/repowise/core/persistence/database.py
index 309ab40fe..a18509f98 100644
--- a/packages/core/src/repowise/core/persistence/database.py
+++ b/packages/core/src/repowise/core/persistence/database.py
@@ -377,10 +377,12 @@ async def init_db(engine: AsyncEngine) -> None:
# SQLite-only: create FTS5 virtual table for full-text search.
# PostgreSQL uses a GIN index added by the Alembic migration.
+ #
+ # The statement comes from ``search.py`` rather than being written out
+ # again here. A second copy of the DDL is how a widened index ends up
+ # only half applied: fresh stores would be born on the old column set
+ # and then be rebuilt by ``FullTextSearch.ensure_index`` on first use.
if engine.dialect.name == "sqlite":
- await conn.execute(
- text(
- "CREATE VIRTUAL TABLE IF NOT EXISTS page_fts "
- "USING fts5(page_id UNINDEXED, title, content)"
- )
- )
+ from repowise.core.persistence.search import PAGE_FTS_DDL
+
+ await conn.execute(text(PAGE_FTS_DDL))
diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.py
index 0029ab60b..de2035c56 100644
--- a/packages/core/src/repowise/core/persistence/search.py
+++ b/packages/core/src/repowise/core/persistence/search.py
@@ -10,7 +10,7 @@ Usage::
fts = FullTextSearch(engine)
await fts.ensure_index() # idempotent — safe to call at startup
- await fts.index("id", "Title", "content …")
+ await fts.index("id", "Title", "content …", summary="…", target_path="a/b.py")
results = await fts.search("decorator pattern", limit=5)
"""
@@ -42,6 +42,34 @@ class SearchResult:
_SNIPPET_LEN = 200
+# Every column of ``page_fts``, in table order. ``page_id`` is stored but not
+# indexed — it is the join key, never a thing to match on.
+#
+# ``summary`` and ``target_path`` are indexed because both decide whether a
+# page is the right answer while being unable to make it a candidate. The
+# summary is a fresh LLM paraphrase, written for every page and never a prefix
+# of the prose it paraphrases, and the answer tool's coverage re-ranker already
+# reads it — so it reorders results it cannot produce. The target path is the
+# file the page documents; most page titles are that path verbatim, which
+# leaves a question naming a directory matching only whatever the generated
+# prose happens to mention.
+PAGE_FTS_COLUMNS = ("page_id", "title", "content", "summary", "target_path")
+
+PAGE_FTS_DDL = (
+ "CREATE VIRTUAL TABLE IF NOT EXISTS page_fts "
+ "USING fts5(page_id UNINDEXED, title, content, summary, target_path)"
+)
+
+# The text PostgreSQL indexes and searches. The GIN index built by the Alembic
+# migration uses this exact expression: any drift between the two silently
+# drops the index from the query plan, leaving a sequential scan that still
+# returns the right rows, so both read it from here.
+PG_FTS_EXPRESSION = (
+ "to_tsvector('english', "
+ "COALESCE(title,'') || ' ' || COALESCE(content,'') || ' ' "
+ "|| COALESCE(summary,'') || ' ' || COALESCE(target_path,''))"
+)
+
# Shortest term that still gets a prefix wildcard. See :func:`_match_term`.
_PREFIX_MIN_CHARS = 4
@@ -246,35 +274,125 @@ class FullTextSearch:
self._dialect = engine.dialect.name # "sqlite" or "postgresql"
# term → how many pages it matches. "" is the corpus size.
self._df_cache: OrderedDict[str, int] = OrderedDict()
+ self._warned_missing_fields = False
async def ensure_index(self) -> None:
"""Create the FTS index if it does not exist (idempotent).
- For SQLite the FTS5 table is created here.
+ For SQLite the FTS5 table is created here, and an index left on an
+ older column set by a previous repowise is rebuilt — see
+ :meth:`_upgrade_sqlite_schema`.
For PostgreSQL the GIN index is created by the Alembic migration; this
- method is a no-op in that case.
+ recreates it when absent, which is what a store built before the
+ migration existed needs.
"""
if self._dialect == "sqlite":
async with self._engine.begin() as conn:
- await conn.execute(
- text(
- "CREATE VIRTUAL TABLE IF NOT EXISTS page_fts "
- "USING fts5(page_id UNINDEXED, title, content)"
- )
- )
+ await conn.execute(text(PAGE_FTS_DDL))
+ await self._upgrade_sqlite_schema()
elif self._dialect == "postgresql":
async with self._engine.begin() as conn:
await conn.execute(
text(
"CREATE INDEX IF NOT EXISTS idx_wiki_pages_fts "
- "ON wiki_pages USING GIN("
- " to_tsvector('english', COALESCE(title, '') || ' ' || COALESCE(content, ''))"
- ")"
+ f"ON wiki_pages USING GIN({PG_FTS_EXPRESSION})"
)
)
- async def index(self, page_id: str, title: str, content: str) -> None:
- """Add or replace a page in the FTS index."""
+ async def _upgrade_sqlite_schema(self) -> None:
+ """Rebuild ``page_fts`` when it predates a column, refilling from SQL.
+
+ FTS5 has no ``ALTER TABLE``, so widening the index means dropping it
+ and writing every row again. The rows themselves cannot supply the new
+ columns — the old index never held them — so the refill reads
+ ``wiki_pages``, which is the system of record for all four indexed
+ fields.
+
+ The table is created with ``IF NOT EXISTS`` by three separate callers,
+ so without this an upgraded install would keep its old shape
+ indefinitely and quietly search fewer fields than it writes.
+ """
+ async with self._engine.connect() as conn:
+ info = await conn.execute(text("PRAGMA table_info(page_fts)"))
+ columns = [r[1] for r in info.fetchall()]
+ if list(columns) == list(PAGE_FTS_COLUMNS):
+ return
+ missing = [c for c in PAGE_FTS_COLUMNS if c not in columns]
+ if not missing:
+ # Extra or reordered columns are somebody else's table. Refuse
+ # rather than drop data this class did not write.
+ raise RuntimeError(
+ f"page_fts has unexpected columns {columns!r}; "
+ f"expected {list(PAGE_FTS_COLUMNS)!r}"
+ )
+
+ indexed_rows = await conn.execute(text("SELECT count(*) FROM page_fts"))
+ indexed_count = int(indexed_rows.scalar() or 0)
+ page_rows = await conn.execute(text("SELECT count(*) FROM wiki_pages"))
+ page_count = int(page_rows.scalar() or 0)
+
+ # The rebuild is a delete followed by a refill from a different table.
+ # If that table cannot account for what is already indexed, the two
+ # halves of the store have drifted and refilling would delete
+ # searchable pages outright. An index on the old shape still answers
+ # queries, so leaving it alone is strictly the safer failure.
+ if indexed_count > page_count:
+ raise RuntimeError(
+ f"Refusing to rebuild page_fts: {indexed_count} indexed rows but only "
+ f"{page_count} rows in wiki_pages to refill them from. The full-text "
+ f"index and the page table have drifted apart; repair the store "
+ f"(repowise doctor --repair) before upgrading it."
+ )
+
+ _log.info(
+ "Rebuilding page_fts to add %s (%d pages to reindex)",
+ ", ".join(missing),
+ page_count,
+ )
+ async with self._engine.begin() as conn:
+ await conn.execute(text("DROP TABLE page_fts"))
+ await conn.execute(text(PAGE_FTS_DDL))
+ await conn.execute(
+ text(
+ "INSERT INTO page_fts(page_id, title, content, summary, target_path) "
+ "SELECT id, COALESCE(title,''), COALESCE(content,''), "
+ " COALESCE(summary,''), COALESCE(target_path,'') "
+ "FROM wiki_pages"
+ )
+ )
+ written = await conn.execute(text("SELECT count(*) FROM page_fts"))
+ refilled = int(written.scalar() or 0)
+
+ if refilled != page_count:
+ # Nothing in the statement above should be able to drop a row, so
+ # a mismatch means the page table moved underneath the rebuild.
+ _log.warning("page_fts rebuild wrote %d rows for %d pages", refilled, page_count)
+
+ async def index(
+ self,
+ page_id: str,
+ title: str,
+ content: str,
+ summary: str | None = None,
+ target_path: str | None = None,
+ ) -> None:
+ """Add or replace a page in the FTS index.
+
+ *summary* and *target_path* are optional so a caller written against
+ the older three-column index keeps working, but omitting them is
+ logged: the row it writes looks healthy and can never be found by
+ either field, which is a ranking regression with no symptom. Nine
+ places inside repowise write this index and every one of them passes
+ both — an omission is a call site that was missed, not a choice.
+
+ An empty *string* is a different thing and is not warned about: some
+ page types genuinely have no summary.
+ """
+ if summary is None or target_path is None:
+ self._warn_missing_index_fields(page_id, summary, target_path)
+ summary = summary if summary is not None else ""
+ target_path = target_path if target_path is not None else ""
+
if self._dialect == "sqlite":
async with self._engine.begin() as conn:
# FTS5 does not support UPDATE; use DELETE + INSERT
@@ -284,14 +402,43 @@ class FullTextSearch:
)
await conn.execute(
text(
- "INSERT INTO page_fts(page_id, title, content) "
- "VALUES (:pid, :title, :content)"
+ "INSERT INTO page_fts(page_id, title, content, summary, target_path) "
+ "VALUES (:pid, :title, :content, :summary, :target_path)"
),
- {"pid": page_id, "title": title, "content": content},
+ {
+ "pid": page_id,
+ "title": title,
+ "content": content,
+ "summary": summary,
+ "target_path": target_path,
+ },
)
# PostgreSQL: the GIN index on wiki_pages is maintained automatically
# by the database as rows are inserted/updated via the CRUD layer.
+ def _warn_missing_index_fields(
+ self, page_id: str, summary: str | None, target_path: str | None
+ ) -> None:
+ """Log the first caller that indexes a page without the newer fields.
+
+ Once per instance, because a caller inside a loop would otherwise
+ write one line per page in the corpus.
+ """
+ if self._warned_missing_fields:
+ return
+ self._warned_missing_fields = True
+ omitted = [
+ name
+ for name, value in (("summary", summary), ("target_path", target_path))
+ if value is None
+ ]
+ _log.warning(
+ "Indexing %r without %s — those pages will not be findable by the "
+ "omitted field. This is a call site that needs updating.",
+ page_id,
+ " or ".join(omitted),
+ )
+
async def delete(self, page_id: str) -> None:
"""Remove a page from the FTS index."""
if self._dialect == "sqlite":
@@ -504,8 +651,8 @@ class FullTextSearch:
if term:
sql = (
- "SELECT count(*) FROM wiki_pages WHERE to_tsvector('english', "
- "COALESCE(title,'') || ' ' || COALESCE(content,'')) @@ to_tsquery('english', :q)"
+ f"SELECT count(*) FROM wiki_pages "
+ f"WHERE {PG_FTS_EXPRESSION} @@ to_tsquery('english', :q)"
)
params = {"q": _pg_term(term)}
else:
@@ -563,16 +710,12 @@ class FullTextSearch:
return []
rows = await conn.execute(
text(
- "SELECT id, title, content, page_type, target_path, "
- " ts_rank("
- " to_tsvector('english', COALESCE(title,'') || ' ' || COALESCE(content,'')), "
- " to_tsquery('english', :q)"
- " ) AS rank "
- "FROM wiki_pages "
- "WHERE to_tsvector('english', COALESCE(title,'') || ' ' || COALESCE(content,'')) "
- " @@ to_tsquery('english', :q) "
- "ORDER BY rank DESC "
- "LIMIT :lim",
+ f"SELECT id, title, content, page_type, target_path, "
+ f" ts_rank({PG_FTS_EXPRESSION}, to_tsquery('english', :q)) AS rank "
+ f"FROM wiki_pages "
+ f"WHERE {PG_FTS_EXPRESSION} @@ to_tsquery('english', :q) "
+ f"ORDER BY rank DESC "
+ f"LIMIT :lim",
),
{"q": ts_query, "lim": limit},
)
diff --git a/packages/core/src/repowise/core/pipeline/scoped_generation.py b/packages/core/src/repowise/core/pipeline/scoped_generation.py
index 232fe859b..726cca089 100644
--- a/packages/core/src/repowise/core/pipeline/scoped_generation.py
+++ b/packages/core/src/repowise/core/pipeline/scoped_generation.py
@@ -295,7 +295,13 @@ async def execute_scoped_generation(
if swept_page_ids:
await fts.delete_many(swept_page_ids)
for page in generated_pages:
- await fts.index(page.page_id, page.title, page.content)
+ await fts.index(
+ page.page_id,
+ page.title,
+ page.content,
+ summary=page.summary,
+ target_path=page.target_path,
+ )
except Exception as exc:
logger.debug("fts_index_skipped", error=str(exc))
diff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.py
index da3b6eabd..0cee4a74c 100644
--- a/packages/server/src/repowise/server/job_executor.py
+++ b/packages/server/src/repowise/server/job_executor.py
@@ -534,7 +534,13 @@ async def execute_job(
await fts.delete_many(swept_page_ids)
if fts is not None and all_pages:
for page in all_pages:
- await fts.index(page.page_id, page.title, page.content)
+ await fts.index(
+ page.page_id,
+ page.title,
+ page.content,
+ summary=page.summary,
+ target_path=page.target_path,
+ )
# ---- Mark completed ------------------------------------------------
# Stop progress updates before writing final status to prevent a
diff --git a/tests/unit/persistence/test_search_fts_columns.py b/tests/unit/persistence/test_search_fts_columns.py
new file mode 100644
index 000000000..e9057e8d0
--- /dev/null
+++ b/tests/unit/persistence/test_search_fts_columns.py
@@ -0,0 +1,211 @@
+"""``page_fts`` indexes a page's summary and target path, not only its prose.
+
+Two fields decide whether a page is the right answer and could not cause it to
+be found. ``summary`` is a fresh LLM paraphrase written for every page, and it
+is already part of what the answer tool's coverage re-ranker reads — a field
+that reorders results but cannot produce one. ``target_path`` is the page's
+file, and most page titles are that path verbatim, so a question that names a
+directory or a file had almost nothing to match against.
+
+These tests cover the upgrade as much as the behaviour: an FTS5 table cannot
+be altered, so an index built by an older repowise has to be rebuilt and
+refilled from ``wiki_pages`` rather than left on the old shape.
+"""
+
+from __future__ import annotations
+
+import pytest
+from sqlalchemy.sql import text
+
+from repowise.core.persistence.crud import upsert_page
+from repowise.core.persistence.search import FullTextSearch
+from tests.unit.persistence.helpers import insert_repo
+
+_OLD_SCHEMA_DDL = "CREATE VIRTUAL TABLE page_fts USING fts5(page_id UNINDEXED, title, content)"
+
+
+async def _fts_columns(engine) -> list[str]:
+ async with engine.connect() as conn:
+ rows = await conn.execute(text("PRAGMA table_info(page_fts)"))
+ return [r[1] for r in rows.fetchall()]
+
+
+async def _downgrade_to_old_schema(engine) -> None:
+ """Put the store back on the three-column table an older repowise wrote."""
+ async with engine.begin() as conn:
+ await conn.execute(text("DROP TABLE IF EXISTS page_fts"))
+ await conn.execute(text(_OLD_SCHEMA_DDL))
+
+
+async def _seed_page(session, repo_id: str, **overrides) -> None:
+ kwargs = {
+ "page_id": "file_page:src/main.py",
+ "repository_id": repo_id,
+ "page_type": "file_page",
+ "title": "File: src/main.py",
+ "content": "# Overview\n\nEntry point for the application.",
+ "summary": "Boots the application and wires the request router.",
+ "target_path": "src/main.py",
+ "source_hash": "abc123",
+ "model_name": "mock",
+ "provider_name": "mock",
+ }
+ kwargs.update(overrides)
+ await upsert_page(session, **kwargs)
+ await session.commit()
+
+
+async def repo(async_session):
+ return await insert_repo(async_session)
+
+
+async def test_ensure_index_upgrades_old_schema_and_backfills(async_engine, async_session, repo):
+ """An index written by an older repowise gains both columns, populated.
+
+ The rebuild reads ``wiki_pages``, which is the only place the two new
+ fields exist — the old FTS rows never carried them. A silent no-op here
+ would leave the store on the old shape forever, because the table is
+ created with ``IF NOT EXISTS``.
+ """
+ await _seed_page(async_session, repo.id)
+ await _downgrade_to_old_schema(async_engine)
+ async with async_engine.begin() as conn:
+ await conn.execute(
+ text("INSERT INTO page_fts(page_id, title, content) VALUES (:p, :t, :c)"),
+ {"p": "file_page:src/main.py", "t": "File: src/main.py", "c": "# Overview"},
+ )
+
+ await FullTextSearch(async_engine).ensure_index()
+
+ assert await _fts_columns(async_engine) == [
+ "page_id",
+ "title",
+ "content",
+ "summary",
+ "target_path",
+ ]
+ async with async_engine.connect() as conn:
+ rows = await conn.execute(text("SELECT page_id, summary, target_path FROM page_fts"))
+ indexed = rows.fetchall()
+ assert len(indexed) == 1
+ assert indexed[0][1] == "Boots the application and wires the request router."
+ assert indexed[0][2] == "src/main.py"
+
+
+async def test_summary_only_match_returns_the_page(async_engine, async_session, repo):
+ """A word that appears in the summary and nowhere else still finds the page."""
+ await _seed_page(
+ async_session,
+ repo.id,
+ summary="Reconciles the changelog against the release manifest.",
+ )
+ fts = FullTextSearch(async_engine)
+ await fts.ensure_index()
+ await fts.index(
+ "file_page:src/main.py",
+ "File: src/main.py",
+ "# Overview\n\nEntry point for the application.",
+ summary="Reconciles the changelog against the release manifest.",
+ target_path="src/main.py",
+ )
+
+ results = await fts.search("release manifest")
+
+ assert [r.page_id for r in results] == ["file_page:src/main.py"]
+
+
+async def test_path_shaped_query_matches_on_target_path_alone(async_engine, async_session, repo):
+ """A question naming a directory hits, even when the prose never says it.
+
+ Titles are the same path in most of the corpus, so without this column a
+ path-shaped question depends on whichever directory names the generated
+ prose happens to mention.
+ """
+ await _seed_page(
+ async_session,
+ repo.id,
+ page_id="file_page:packages/telemetry/collector.py",
+ title="Collector",
+ content="# Overview\n\nGathers counters and flushes them on a timer.",
+ summary="Gathers counters and flushes them.",
+ target_path="packages/telemetry/collector.py",
+ )
+ fts = FullTextSearch(async_engine)
+ await fts.ensure_index()
+ await fts.index(
+ "file_page:packages/telemetry/collector.py",
+ "Collector",
+ "# Overview\n\nGathers counters and flushes them on a timer.",
+ summary="Gathers counters and flushes them.",
+ target_path="packages/telemetry/collector.py",
+ )
+
+ results = await fts.search("telemetry collector")
+
+ assert [r.page_id for r in results] == ["file_page:packages/telemetry/collector.py"]
+
+
+async def test_index_without_the_new_fields_still_writes_a_row(async_engine):
+ """The two new arguments are optional, so an old caller keeps working."""
+ fts = FullTextSearch(async_engine)
+ await fts.ensure_index()
+ await fts.index("p1", "Title", "searchable body text")
+
+ results = await fts.search("searchable body")
+
+ assert [r.page_id for r in results] == ["p1"]
+
+
+async def test_index_without_the_new_fields_warns_once(async_engine, caplog):
+ """A missed call site writes a healthy-looking, half-searchable row.
+
+ Nine places in repowise write this index. One left on the old signature
+ produces pages that cannot be found by their summary or their path, with
+ nothing failing anywhere, so the omission is logged. Once per instance —
+ every one of those call sites writes in a loop over the whole corpus.
+ """
+ fts = FullTextSearch(async_engine)
+ await fts.ensure_index()
+
+ with caplog.at_level("WARNING", logger="repowise.core.persistence.search"):
+ await fts.index("p1", "Title", "body")
+ await fts.index("p2", "Title", "body")
+
+ warnings = [r for r in caplog.records if r.levelname == "WARNING"]
+ assert len(warnings) == 1
+ assert "summary or target_path" in warnings[0].getMessage()
+
+
+async def test_index_with_an_empty_summary_does_not_warn(async_engine):
+ """An empty string is a real value — some page types have no summary."""
+ fts = FullTextSearch(async_engine)
+ await fts.ensure_index()
+ await fts.index("p1", "Title", "body", summary="", target_path="")
+
+ assert fts._warned_missing_fields is False
+
+
+async def test_rebuild_refuses_to_shrink_the_index(async_engine, async_session, repo):
+ """The upgrade drops the table, so it must not run against a thinner source.
+
+ The rebuild refills from ``wiki_pages``. If that table cannot account for
+ the rows already indexed — a store whose two halves have drifted apart, or
+ an engine pointed at the wrong file — dropping and refilling silently
+ deletes searchable pages. Raise instead, and leave the old index in place.
+ """
+ await _downgrade_to_old_schema(async_engine)
+ async with async_engine.begin() as conn:
+ for i in range(3):
+ await conn.execute(
+ text("INSERT INTO page_fts(page_id, title, content) VALUES (:p, :t, :c)"),
+ {"p": f"orphan-{i}", "t": "Orphan", "c": "no wiki_pages row backs this"},
+ )
+
+ with pytest.raises(RuntimeError, match="page_fts"):
+ await FullTextSearch(async_engine).ensure_index()
+
+ async with async_engine.connect() as conn:
+ rows = await conn.execute(text("SELECT count(*) FROM page_fts"))
+ assert rows.scalar() == 3
+ assert await _fts_columns(async_engine) == ["page_id", "title", "content"]