Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(search): index the file vocabulary for full-text search (#2718)
Raghav Chamadiya6942f17fd9a2
diff --git a/packages/core/alembic/versions/0044_fts_summary_and_target_path.py b/packages/core/alembic/versions/0044_fts_summary_and_target_path.pyindex 2fe68ce33..5a7777807 100644--- a/packages/core/alembic/versions/0044_fts_summary_and_target_path.py+++ b/packages/core/alembic/versions/0044_fts_summary_and_target_path.py@@ -34,7 +34,7 @@ from collections.abc import Sequencefrom alembic import op-from repowise.core.persistence.search import PAGE_FTS_DDL, PG_FTS_EXPRESSION+from repowise.core.persistence.search import PG_FTS_EXPRESSION# revision identifiersrevision: str = "0044"@@ -56,6 +56,13 @@ _OLD_REFILL_SQL = ("SELECT id, COALESCE(title,''), COALESCE(content,'') FROM wiki_pages")+# Frozen at this revision's shape. ``ensure_index`` widens it to the current+# columns, refilling any that were added later.+PAGE_FTS_DDL = (+ "CREATE VIRTUAL TABLE IF NOT EXISTS page_fts "+ "USING fts5(page_id UNINDEXED, title, content, summary, target_path)"+)+_OLD_SQLITE_DDL = ("CREATE VIRTUAL TABLE IF NOT EXISTS page_fts USING fts5(page_id UNINDEXED, title, content)")diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.pyindex 4d9a7bd8c..4b644d4c3 100644--- a/packages/core/src/repowise/core/persistence/search.py+++ b/packages/core/src/repowise/core/persistence/search.py@@ -61,16 +61,33 @@ _SNIPPET_LEN = 200# file the page documents; most page titles are that path verbatim, which# leaves a question naming a directory matching only whatever the generated# prose happens to mention.-PAGE_FTS_COLUMNS = ("page_id", "title", "content", "summary", "target_path")+#+# ``vocabulary`` is a file page's own identifiers and comment prose. It is kept+# off the rendered page (it reads as noise), so without its own column a+# keyless store cannot match a page on any of those words.+PAGE_FTS_COLUMNS = ("page_id", "title", "content", "summary", "target_path", "vocabulary")PAGE_FTS_DDL = ("CREATE VIRTUAL TABLE IF NOT EXISTS page_fts "- "USING fts5(page_id UNINDEXED, title, content, summary, target_path)"+ "USING fts5(page_id UNINDEXED, title, content, summary, target_path, vocabulary)")_PAGE_FTS_INSERT_SQL = (- "INSERT INTO page_fts(page_id, title, content, summary, target_path) "- "VALUES (:pid, :title, :content, :summary, :target_path)"+ "INSERT INTO page_fts(page_id, title, content, summary, target_path, vocabulary) "+ "VALUES (:pid, :title, :content, :summary, :target_path, :vocabulary)"+)++# Page-metadata key for a file page's vocabulary (field names, string+# literals, comment prose). Indexed for search, never rendered on the page.+FILE_VOCABULARY_KEY = "file_vocabulary"++# The vocabulary as stored on a ``wiki_pages`` row. Every writer indexes after+# the page row commits, so reading it here keeps all of them in agreement+# without each passing it. Invalid JSON yields "" rather than failing a write.+_VOCABULARY_SQL = (+ "CASE WHEN json_valid(metadata_json) "+ f"THEN COALESCE(json_extract(metadata_json, '$.{FILE_VOCABULARY_KEY}'), '') "+ "ELSE '' END")# SQLite allows 999 host parameters per statement by default.@@ -93,6 +110,23 @@ async def _delete_page_ids(conn: Any, page_ids: Sequence[str]) -> None:)+async def _stored_vocabulary(conn: Any, page_ids: Sequence[str]) -> dict[str, str]:+ """``page_id -> vocabulary`` from ``wiki_pages``; ``{}`` without that table."""+ if not await FullTextSearch._page_table_exists(conn):+ return {}+ found: dict[str, str] = {}+ for start in range(0, len(page_ids), _ID_CHUNK):+ chunk = page_ids[start : start + _ID_CHUNK]+ placeholders = ", ".join(f":p{i}" for i in range(len(chunk)))+ params = {f"p{i}": pid for i, pid in enumerate(chunk)}+ rows = await conn.execute(+ text(f"SELECT id, {_VOCABULARY_SQL} FROM wiki_pages WHERE id IN ({placeholders})"),+ params,+ )+ found.update({r[0]: r[1] or "" for r in rows.fetchall()})+ return found++# An indexed row whose page is gone from ``wiki_pages``. Counting and deleting# share the predicate so the number reported is exactly the number removed.# ``NOT EXISTS`` rather than ``NOT IN``: the subquery's column is a primary key@@ -108,6 +142,10 @@ _ORPHAN_DELETE_SQL = f"DELETE FROM page_fts WHERE {_ORPHAN_PREDICATE}"# migration uses this exact expression: any drift between the two silently# drops the index from the query plan, leaving a sequential scan that still# returns the right rows, so both read it from here.+#+# It does not cover the file vocabulary yet: that needs a migration rebuilding+# the GIN index over ``metadata_json``, which a text column cast to JSON makes+# fail on any unparseable row.PG_FTS_EXPRESSION = ("to_tsvector('english', ""COALESCE(title,'') || ' ' || COALESCE(content,'') || ' ' "@@ -128,8 +166,9 @@ _SCORE_EPSILON = 1e-6# it. ``page_id`` is UNINDEXED and contributes nothing, but bm25() takes one# weight per column, so it still needs one. bm25() ignores a weight past the# last column and defaults a missing one to 1.0, so the arity is checked by-# test_search_fts_columns rather than by anything raising here.-_BM25_COLUMN_WEIGHTS = (0.0, 4.0, 1.0, 1.0, 3.0)+# test_search_fts_columns rather than by anything raising here. The vocabulary+# is a bag of words, so it counts for less than prose written about the file.+_BM25_COLUMN_WEIGHTS = (0.0, 4.0, 1.0, 1.0, 3.0, 0.5)_BM25_SCORE = "bm25(page_fts, " + ", ".join(str(w) for w in _BM25_COLUMN_WEIGHTS) + ")"_log = logging.getLogger(__name__)@@ -227,8 +266,8 @@ class FullTextSearch:FTS5 has no ``ALTER TABLE``, so widening the index means dropping itand writing every row again. The rows themselves cannot supply the newcolumns — the old index never held them — so the refill reads- ``wiki_pages``, which is the system of record for all four indexed- fields.+ ``wiki_pages``, which is the system of record for every indexed+ field (the vocabulary from ``metadata_json``).The table is created with ``IF NOT EXISTS`` by three separate callers,so without this an upgraded install would keep its old shape@@ -307,9 +346,11 @@ class FullTextSearch:await conn.execute(text(PAGE_FTS_DDL))await conn.execute(text(- "INSERT INTO page_fts(page_id, title, content, summary, target_path) "+ "INSERT INTO page_fts"+ "(page_id, title, content, summary, target_path, vocabulary) ""SELECT id, COALESCE(title,''), COALESCE(content,''), "- " COALESCE(summary,''), COALESCE(target_path,'') "+ " COALESCE(summary,''), COALESCE(target_path,''), "+ f" {_VOCABULARY_SQL} ""FROM wiki_pages"))@@ -451,6 +492,7 @@ class FullTextSearch:"content": content,"summary": summary or "","target_path": target_path or "",+ "vocabulary": "",}else:self._count_skipped_below_floor(page_id, content)@@ -464,6 +506,8 @@ class FullTextSearch:async with self._engine.begin() as conn:await _delete_page_ids(conn, list(deletions))if insertions:+ for pid, vocabulary in (await _stored_vocabulary(conn, list(insertions))).items():+ insertions[pid]["vocabulary"] = vocabularyawait conn.execute(text(_PAGE_FTS_INSERT_SQL), list(insertions.values()))def _count_skipped_below_floor(self, page_id: str, content: str) -> None:diff --git a/packages/core/src/repowise/core/persistence/vector_store/_base.py b/packages/core/src/repowise/core/persistence/vector_store/_base.pyindex bbce123c8..4359d1d55 100644--- a/packages/core/src/repowise/core/persistence/vector_store/_base.py+++ b/packages/core/src/repowise/core/persistence/vector_store/_base.py@@ -17,7 +17,7 @@ from dataclasses import dataclassfrom typing import Literalfrom ..information_floor import count_page_denied_a_vector, meets_information_floor-from ..search import SearchResult+from ..search import FILE_VOCABULARY_KEY, SearchResult__all__ = ["EMBED_BATCH_MAX_ITEMS",@@ -97,11 +97,6 @@ EMBED_TEXT_MAX_CHARS = 16_000# and on the paths that passed no content at all, an empty string.STORED_SNIPPET_CHARS = 2_000-# Page-metadata key for a file page's own vocabulary (field names, string-# literals, comment prose). It is embedded with the page but not rendered on-# it: a bag of words helps retrieval and reads to a person as noise.-FILE_VOCABULARY_KEY = "file_vocabulary"-def cap_embed_text(page_id: str, text: str) -> str:"""Return text bounded for one embedding input and report any loss.diff --git a/tests/unit/cli/test_doctor_fts_drift.py b/tests/unit/cli/test_doctor_fts_drift.pyindex 3ac92e683..423eec98e 100644--- a/tests/unit/cli/test_doctor_fts_drift.py+++ b/tests/unit/cli/test_doctor_fts_drift.py@@ -20,6 +20,7 @@ import sqlite3from pathlib import Pathfrom repowise.cli.commands.doctor_cmd import repo_checks+from repowise.core.persistence.search import PAGE_FTS_COLUMNSLIVE_PAGES = 24ORPHANS = 89@@ -128,13 +129,7 @@ def test_repair_leaves_the_index_on_the_current_column_set(tmp_path: Path) -> Norepo_checks._run_repo_checks(repo_path, repair=True)- assert _fts_columns(repo_path) == [- "page_id",- "title",- "content",- "summary",- "target_path",- ]+ assert _fts_columns(repo_path) == list(PAGE_FTS_COLUMNS)def test_a_plain_run_heals_the_drift_on_its_own(tmp_path: Path) -> None:@@ -155,4 +150,4 @@ def test_a_plain_run_heals_the_drift_on_its_own(tmp_path: Path) -> None:asyncio.run(_open())assert _indexed_ids(repo_path) == {f"file_page:src/mod_{i}.py" for i in range(LIVE_PAGES)}- assert _fts_columns(repo_path)[-1] == "target_path"+ assert _fts_columns(repo_path) == list(PAGE_FTS_COLUMNS)diff --git a/tests/unit/persistence/test_search_fts_columns.py b/tests/unit/persistence/test_search_fts_columns.pyindex 02791e1d0..ce74407be 100644--- a/tests/unit/persistence/test_search_fts_columns.py+++ b/tests/unit/persistence/test_search_fts_columns.py@@ -20,6 +20,7 @@ from sqlalchemy.sql import textfrom repowise.core.persistence.crud import upsert_pagefrom repowise.core.persistence.search import (_BM25_COLUMN_WEIGHTS,+ FILE_VOCABULARY_KEY,PAGE_FTS_COLUMNS,FullTextSearch,)@@ -82,13 +83,7 @@ async def test_ensure_index_upgrades_old_schema_and_backfills(async_engine, asynawait FullTextSearch(async_engine).ensure_index()- assert await _fts_columns(async_engine) == [- "page_id",- "title",- "content",- "summary",- "target_path",- ]+ assert await _fts_columns(async_engine) == list(PAGE_FTS_COLUMNS)async with async_engine.connect() as conn:rows = await conn.execute(text("SELECT page_id, summary, target_path FROM page_fts"))indexed = rows.fetchall()@@ -326,3 +321,72 @@ def test_one_bm25_weight_per_indexed_column():changes ranking without raising anywhere."""assert len(_BM25_COLUMN_WEIGHTS) == len(PAGE_FTS_COLUMNS)+++_VOCABULARY_PAGE = {+ "content": "# Overview\n\nEntry point for the application.",+ "metadata": {FILE_VOCABULARY_KEY: "localeError customError xylophone"},+}+++async def test_a_vocabulary_only_term_finds_the_page_without_an_embedder(+ async_engine, async_session, repo+):+ """The vocabulary is kept off the page, so full-text is its only reach.++ A keyless store has no vectors. When the vocabulary left the rendered page+ it left the index with it, and a question whose only match was a field+ name or comment returned nothing.+ """+ await _seed_page(async_session, repo.id, **_VOCABULARY_PAGE)+ fts = FullTextSearch(async_engine)+ await fts.ensure_index()+ await fts.index(+ "file_page:src/main.py",+ "File: src/main.py",+ _VOCABULARY_PAGE["content"],+ summary="",+ target_path="src/main.py",+ )++ results = await fts.search("xylophone")++ assert [r.page_id for r in results] == ["file_page:src/main.py"]+ assert "xylophone" not in results[0].snippet+++async def test_upgrade_refills_the_vocabulary_from_page_metadata(+ async_engine, async_session, repo+):+ """A store indexed before the column existed gains it, populated."""+ await _seed_page(async_session, repo.id, **_VOCABULARY_PAGE)+ async with async_engine.begin() as conn:+ await conn.execute(text("DROP TABLE IF EXISTS page_fts"))+ await conn.execute(+ text(+ "CREATE VIRTUAL TABLE page_fts USING "+ "fts5(page_id UNINDEXED, title, content, summary, target_path)"+ )+ )++ fts = FullTextSearch(async_engine)+ await fts.ensure_index()++ assert await _fts_columns(async_engine) == list(PAGE_FTS_COLUMNS)+ assert [r.page_id for r in await fts.search("xylophone")] == ["file_page:src/main.py"]+++async def test_unreadable_page_metadata_indexes_without_a_vocabulary(+ async_engine, async_session, repo+):+ """Bad JSON on one row must not fail the write for the batch."""+ await _seed_page(async_session, repo.id)+ async with async_engine.begin() as conn:+ await conn.execute(text("UPDATE wiki_pages SET metadata_json = '{not json'"))+ fts = FullTextSearch(async_engine)+ await fts.ensure_index()+ await fts.index(+ "file_page:src/main.py", "File: src/main.py", "Entry point body.", summary="", target_path=""+ )++ assert [r.page_id for r in await fts.search("entry point")] == ["file_page:src/main.py"]