Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(search): index the file vocabulary for full-text search (#2718)

Raghav Chamadiya6942f17fd9a2
diff --git a/packages/core/alembic/versions/0044_fts_summary_and_target_path.py b/packages/core/alembic/versions/0044_fts_summary_and_target_path.py
index 2fe68ce33..5a7777807 100644
--- a/packages/core/alembic/versions/0044_fts_summary_and_target_path.py
+++ b/packages/core/alembic/versions/0044_fts_summary_and_target_path.py
@@ -34,7 +34,7 @@ from collections.abc import Sequence
from alembic import op
-from repowise.core.persistence.search import PAGE_FTS_DDL, PG_FTS_EXPRESSION
+from repowise.core.persistence.search import PG_FTS_EXPRESSION
# revision identifiers
revision: str = "0044"
@@ -56,6 +56,13 @@ _OLD_REFILL_SQL = (
"SELECT id, COALESCE(title,''), COALESCE(content,'') FROM wiki_pages"
)
+# Frozen at this revision's shape. ``ensure_index`` widens it to the current
+# columns, refilling any that were added later.
+PAGE_FTS_DDL = (
+ "CREATE VIRTUAL TABLE IF NOT EXISTS page_fts "
+ "USING fts5(page_id UNINDEXED, title, content, summary, target_path)"
+)
+
_OLD_SQLITE_DDL = (
"CREATE VIRTUAL TABLE IF NOT EXISTS page_fts USING fts5(page_id UNINDEXED, title, content)"
)
diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.py
index 4d9a7bd8c..4b644d4c3 100644
--- a/packages/core/src/repowise/core/persistence/search.py
+++ b/packages/core/src/repowise/core/persistence/search.py
@@ -61,16 +61,33 @@ _SNIPPET_LEN = 200
# file the page documents; most page titles are that path verbatim, which
# leaves a question naming a directory matching only whatever the generated
# prose happens to mention.
-PAGE_FTS_COLUMNS = ("page_id", "title", "content", "summary", "target_path")
+#
+# ``vocabulary`` is a file page's own identifiers and comment prose. It is kept
+# off the rendered page (it reads as noise), so without its own column a
+# keyless store cannot match a page on any of those words.
+PAGE_FTS_COLUMNS = ("page_id", "title", "content", "summary", "target_path", "vocabulary")
PAGE_FTS_DDL = (
"CREATE VIRTUAL TABLE IF NOT EXISTS page_fts "
- "USING fts5(page_id UNINDEXED, title, content, summary, target_path)"
+ "USING fts5(page_id UNINDEXED, title, content, summary, target_path, vocabulary)"
)
_PAGE_FTS_INSERT_SQL = (
- "INSERT INTO page_fts(page_id, title, content, summary, target_path) "
- "VALUES (:pid, :title, :content, :summary, :target_path)"
+ "INSERT INTO page_fts(page_id, title, content, summary, target_path, vocabulary) "
+ "VALUES (:pid, :title, :content, :summary, :target_path, :vocabulary)"
+)
+
+# Page-metadata key for a file page's vocabulary (field names, string
+# literals, comment prose). Indexed for search, never rendered on the page.
+FILE_VOCABULARY_KEY = "file_vocabulary"
+
+# The vocabulary as stored on a ``wiki_pages`` row. Every writer indexes after
+# the page row commits, so reading it here keeps all of them in agreement
+# without each passing it. Invalid JSON yields "" rather than failing a write.
+_VOCABULARY_SQL = (
+ "CASE WHEN json_valid(metadata_json) "
+ f"THEN COALESCE(json_extract(metadata_json, '$.{FILE_VOCABULARY_KEY}'), '') "
+ "ELSE '' END"
)
# SQLite allows 999 host parameters per statement by default.
@@ -93,6 +110,23 @@ async def _delete_page_ids(conn: Any, page_ids: Sequence[str]) -> None:
)
+async def _stored_vocabulary(conn: Any, page_ids: Sequence[str]) -> dict[str, str]:
+ """``page_id -> vocabulary`` from ``wiki_pages``; ``{}`` without that table."""
+ if not await FullTextSearch._page_table_exists(conn):
+ return {}
+ found: dict[str, str] = {}
+ for start in range(0, len(page_ids), _ID_CHUNK):
+ chunk = page_ids[start : start + _ID_CHUNK]
+ placeholders = ", ".join(f":p{i}" for i in range(len(chunk)))
+ params = {f"p{i}": pid for i, pid in enumerate(chunk)}
+ rows = await conn.execute(
+ text(f"SELECT id, {_VOCABULARY_SQL} FROM wiki_pages WHERE id IN ({placeholders})"),
+ params,
+ )
+ found.update({r[0]: r[1] or "" for r in rows.fetchall()})
+ return found
+
+
# An indexed row whose page is gone from ``wiki_pages``. Counting and deleting
# share the predicate so the number reported is exactly the number removed.
# ``NOT EXISTS`` rather than ``NOT IN``: the subquery's column is a primary key
@@ -108,6 +142,10 @@ _ORPHAN_DELETE_SQL = f"DELETE FROM page_fts WHERE {_ORPHAN_PREDICATE}"
# migration uses this exact expression: any drift between the two silently
# drops the index from the query plan, leaving a sequential scan that still
# returns the right rows, so both read it from here.
+#
+# It does not cover the file vocabulary yet: that needs a migration rebuilding
+# the GIN index over ``metadata_json``, which a text column cast to JSON makes
+# fail on any unparseable row.
PG_FTS_EXPRESSION = (
"to_tsvector('english', "
"COALESCE(title,'') || ' ' || COALESCE(content,'') || ' ' "
@@ -128,8 +166,9 @@ _SCORE_EPSILON = 1e-6
# it. ``page_id`` is UNINDEXED and contributes nothing, but bm25() takes one
# weight per column, so it still needs one. bm25() ignores a weight past the
# last column and defaults a missing one to 1.0, so the arity is checked by
-# test_search_fts_columns rather than by anything raising here.
-_BM25_COLUMN_WEIGHTS = (0.0, 4.0, 1.0, 1.0, 3.0)
+# test_search_fts_columns rather than by anything raising here. The vocabulary
+# is a bag of words, so it counts for less than prose written about the file.
+_BM25_COLUMN_WEIGHTS = (0.0, 4.0, 1.0, 1.0, 3.0, 0.5)
_BM25_SCORE = "bm25(page_fts, " + ", ".join(str(w) for w in _BM25_COLUMN_WEIGHTS) + ")"
_log = logging.getLogger(__name__)
@@ -227,8 +266,8 @@ class FullTextSearch:
FTS5 has no ``ALTER TABLE``, so widening the index means dropping it
and writing every row again. The rows themselves cannot supply the new
columns — the old index never held them — so the refill reads
- ``wiki_pages``, which is the system of record for all four indexed
- fields.
+ ``wiki_pages``, which is the system of record for every indexed
+ field (the vocabulary from ``metadata_json``).
The table is created with ``IF NOT EXISTS`` by three separate callers,
so without this an upgraded install would keep its old shape
@@ -307,9 +346,11 @@ class FullTextSearch:
await conn.execute(text(PAGE_FTS_DDL))
await conn.execute(
text(
- "INSERT INTO page_fts(page_id, title, content, summary, target_path) "
+ "INSERT INTO page_fts"
+ "(page_id, title, content, summary, target_path, vocabulary) "
"SELECT id, COALESCE(title,''), COALESCE(content,''), "
- " COALESCE(summary,''), COALESCE(target_path,'') "
+ " COALESCE(summary,''), COALESCE(target_path,''), "
+ f" {_VOCABULARY_SQL} "
"FROM wiki_pages"
)
)
@@ -451,6 +492,7 @@ class FullTextSearch:
"content": content,
"summary": summary or "",
"target_path": target_path or "",
+ "vocabulary": "",
}
else:
self._count_skipped_below_floor(page_id, content)
@@ -464,6 +506,8 @@ class FullTextSearch:
async with self._engine.begin() as conn:
await _delete_page_ids(conn, list(deletions))
if insertions:
+ for pid, vocabulary in (await _stored_vocabulary(conn, list(insertions))).items():
+ insertions[pid]["vocabulary"] = vocabulary
await conn.execute(text(_PAGE_FTS_INSERT_SQL), list(insertions.values()))
def _count_skipped_below_floor(self, page_id: str, content: str) -> None:
diff --git a/packages/core/src/repowise/core/persistence/vector_store/_base.py b/packages/core/src/repowise/core/persistence/vector_store/_base.py
index bbce123c8..4359d1d55 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/_base.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/_base.py
@@ -17,7 +17,7 @@ from dataclasses import dataclass
from typing import Literal
from ..information_floor import count_page_denied_a_vector, meets_information_floor
-from ..search import SearchResult
+from ..search import FILE_VOCABULARY_KEY, SearchResult
__all__ = [
"EMBED_BATCH_MAX_ITEMS",
@@ -97,11 +97,6 @@ EMBED_TEXT_MAX_CHARS = 16_000
# and on the paths that passed no content at all, an empty string.
STORED_SNIPPET_CHARS = 2_000
-# Page-metadata key for a file page's own vocabulary (field names, string
-# literals, comment prose). It is embedded with the page but not rendered on
-# it: a bag of words helps retrieval and reads to a person as noise.
-FILE_VOCABULARY_KEY = "file_vocabulary"
-
def cap_embed_text(page_id: str, text: str) -> str:
"""Return text bounded for one embedding input and report any loss.
diff --git a/tests/unit/cli/test_doctor_fts_drift.py b/tests/unit/cli/test_doctor_fts_drift.py
index 3ac92e683..423eec98e 100644
--- a/tests/unit/cli/test_doctor_fts_drift.py
+++ b/tests/unit/cli/test_doctor_fts_drift.py
@@ -20,6 +20,7 @@ import sqlite3
from pathlib import Path
from repowise.cli.commands.doctor_cmd import repo_checks
+from repowise.core.persistence.search import PAGE_FTS_COLUMNS
LIVE_PAGES = 24
ORPHANS = 89
@@ -128,13 +129,7 @@ def test_repair_leaves_the_index_on_the_current_column_set(tmp_path: Path) -> No
repo_checks._run_repo_checks(repo_path, repair=True)
- assert _fts_columns(repo_path) == [
- "page_id",
- "title",
- "content",
- "summary",
- "target_path",
- ]
+ assert _fts_columns(repo_path) == list(PAGE_FTS_COLUMNS)
def test_a_plain_run_heals_the_drift_on_its_own(tmp_path: Path) -> None:
@@ -155,4 +150,4 @@ def test_a_plain_run_heals_the_drift_on_its_own(tmp_path: Path) -> None:
asyncio.run(_open())
assert _indexed_ids(repo_path) == {f"file_page:src/mod_{i}.py" for i in range(LIVE_PAGES)}
- assert _fts_columns(repo_path)[-1] == "target_path"
+ assert _fts_columns(repo_path) == list(PAGE_FTS_COLUMNS)
diff --git a/tests/unit/persistence/test_search_fts_columns.py b/tests/unit/persistence/test_search_fts_columns.py
index 02791e1d0..ce74407be 100644
--- a/tests/unit/persistence/test_search_fts_columns.py
+++ b/tests/unit/persistence/test_search_fts_columns.py
@@ -20,6 +20,7 @@ from sqlalchemy.sql import text
from repowise.core.persistence.crud import upsert_page
from repowise.core.persistence.search import (
_BM25_COLUMN_WEIGHTS,
+ FILE_VOCABULARY_KEY,
PAGE_FTS_COLUMNS,
FullTextSearch,
)
@@ -82,13 +83,7 @@ async def test_ensure_index_upgrades_old_schema_and_backfills(async_engine, asyn
await FullTextSearch(async_engine).ensure_index()
- assert await _fts_columns(async_engine) == [
- "page_id",
- "title",
- "content",
- "summary",
- "target_path",
- ]
+ assert await _fts_columns(async_engine) == list(PAGE_FTS_COLUMNS)
async with async_engine.connect() as conn:
rows = await conn.execute(text("SELECT page_id, summary, target_path FROM page_fts"))
indexed = rows.fetchall()
@@ -326,3 +321,72 @@ def test_one_bm25_weight_per_indexed_column():
changes ranking without raising anywhere.
"""
assert len(_BM25_COLUMN_WEIGHTS) == len(PAGE_FTS_COLUMNS)
+
+
+_VOCABULARY_PAGE = {
+ "content": "# Overview\n\nEntry point for the application.",
+ "metadata": {FILE_VOCABULARY_KEY: "localeError customError xylophone"},
+}
+
+
+async def test_a_vocabulary_only_term_finds_the_page_without_an_embedder(
+ async_engine, async_session, repo
+):
+ """The vocabulary is kept off the page, so full-text is its only reach.
+
+ A keyless store has no vectors. When the vocabulary left the rendered page
+ it left the index with it, and a question whose only match was a field
+ name or comment returned nothing.
+ """
+ await _seed_page(async_session, repo.id, **_VOCABULARY_PAGE)
+ fts = FullTextSearch(async_engine)
+ await fts.ensure_index()
+ await fts.index(
+ "file_page:src/main.py",
+ "File: src/main.py",
+ _VOCABULARY_PAGE["content"],
+ summary="",
+ target_path="src/main.py",
+ )
+
+ results = await fts.search("xylophone")
+
+ assert [r.page_id for r in results] == ["file_page:src/main.py"]
+ assert "xylophone" not in results[0].snippet
+
+
+async def test_upgrade_refills_the_vocabulary_from_page_metadata(
+ async_engine, async_session, repo
+):
+ """A store indexed before the column existed gains it, populated."""
+ await _seed_page(async_session, repo.id, **_VOCABULARY_PAGE)
+ async with async_engine.begin() as conn:
+ await conn.execute(text("DROP TABLE IF EXISTS page_fts"))
+ await conn.execute(
+ text(
+ "CREATE VIRTUAL TABLE page_fts USING "
+ "fts5(page_id UNINDEXED, title, content, summary, target_path)"
+ )
+ )
+
+ fts = FullTextSearch(async_engine)
+ await fts.ensure_index()
+
+ assert await _fts_columns(async_engine) == list(PAGE_FTS_COLUMNS)
+ assert [r.page_id for r in await fts.search("xylophone")] == ["file_page:src/main.py"]
+
+
+async def test_unreadable_page_metadata_indexes_without_a_vocabulary(
+ async_engine, async_session, repo
+):
+ """Bad JSON on one row must not fail the write for the batch."""
+ await _seed_page(async_session, repo.id)
+ async with async_engine.begin() as conn:
+ await conn.execute(text("UPDATE wiki_pages SET metadata_json = '{not json'"))
+ fts = FullTextSearch(async_engine)
+ await fts.ensure_index()
+ await fts.index(
+ "file_page:src/main.py", "File: src/main.py", "Entry point body.", summary="", target_path=""
+ )
+
+ assert [r.page_id for r in await fts.search("entry point")] == ["file_page:src/main.py"]