Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(search): stop asking full-text search for most of the corpus (#1188)

Swati Ahuja957ddc7dadaa
diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.py
index 0f35358c3..0029ab60b 100644
--- a/packages/core/src/repowise/core/persistence/search.py
+++ b/packages/core/src/repowise/core/persistence/search.py
@@ -16,6 +16,10 @@ Usage::
from __future__ import annotations
+import logging
+import os
+from collections import OrderedDict
+from collections.abc import Callable
from dataclasses import dataclass
from typing import Literal
@@ -38,6 +42,32 @@ class SearchResult:
_SNIPPET_LEN = 200
+# Shortest term that still gets a prefix wildcard. See :func:`_match_term`.
+_PREFIX_MIN_CHARS = 4
+
+# A term matching more than this share of the corpus is dropped from the query
+# expression: it cannot discriminate between pages, and every one of them drags
+# thousands of candidates in for BM25 to sort out afterwards. Env-tunable
+# because the right ceiling is a property of a corpus's vocabulary, and the only
+# honest way to set it is to measure recall on the corpus in question.
+_DF_CEILING = float(os.environ.get("REPOWISE_FTS_DF_CEILING", "0.20"))
+
+# Never let the ceiling empty a query. A question written entirely out of common
+# words keeps its rarest terms — a thin match beats no match, especially since
+# vector retrieval is answering the same question alongside this.
+_MIN_KEPT_TERMS = 3
+
+# How many distinct terms' document frequencies one FullTextSearch keeps. Term
+# frequency changes only when the corpus is rewritten, and questions reuse
+# vocabulary heavily, so a small cache removes almost all of the counting.
+_DF_CACHE_MAX = 2048
+
+# Keeps scores strictly decreasing when two MATCH expressions' results are
+# concatenated. Small enough not to disturb any threshold downstream.
+_SCORE_EPSILON = 1e-6
+
+_log = logging.getLogger(__name__)
+
# Common English stop words to strip from FTS queries
_STOP_WORDS = frozenset(
{
@@ -128,28 +158,80 @@ _STOP_WORDS = frozenset(
)
-def _build_fts5_query(query: str) -> str:
- """Build an FTS5 MATCH expression from a natural-language query.
-
- Strips stop words, then joins remaining terms with OR so that pages
- containing *any* keyword match. Each term gets a ``*`` suffix for
- prefix matching (e.g. "pay*" matches "payment", "payload", etc.).
- Falls back to the raw (quoted) query when all tokens are stop words.
- """
+def _meaningful_terms(query: str) -> list[str]:
+ """Alphanumeric tokens of *query*, minus stop words and single characters."""
import re
- # Keep only alphanumeric tokens
tokens = re.findall(r"[a-zA-Z0-9_]+", query.lower())
- meaningful = [t for t in tokens if t not in _STOP_WORDS and len(t) > 1]
+ return [t for t in tokens if t not in _STOP_WORDS and len(t) > 1]
+
+
+def _match_term(term: str) -> str:
+ """One FTS5 term, prefix-matched only when it is long enough to be specific.
+
+ A prefix wildcard is what earns morphological recall — ``invalidat*`` finds
+ "invalidates" and "invalidation" — and it is also what turns a short token
+ into a corpus-wide match: ``set*`` hits settings, setup, setter, setdefault.
+ Below the cut the term is matched exactly, which costs the odd plural and
+ buys back a candidate set that BM25 was being asked to sort out afterwards.
+ """
+ return f'"{term}"*' if len(term) >= _PREFIX_MIN_CHARS else f'"{term}"'
+
+
+def _pg_term(term: str) -> str:
+ """One ``to_tsquery`` lexeme, prefix-matched on the same rule as FTS5.
+
+ Terms reach here already reduced to ``[a-zA-Z0-9_]+`` by
+ :func:`_meaningful_terms`, so there is no quoting to get wrong — anything
+ that could change the expression's shape has been dropped before this.
+ """
+ return f"{term}:*" if len(term) >= _PREFIX_MIN_CHARS else term
+
+
+def _build_fts5_query(query: str, document_frequency: Callable[[str], int] | None = None) -> str:
+ """Build an FTS5 MATCH expression from a natural-language query.
+
+ Terms are OR-ed, because a developer question is a paraphrase of the page
+ that answers it: its rarest words are frequently the asker's vocabulary
+ rather than the page's, so requiring all of them (AND) excludes the right
+ page far more often than it narrows usefully. Measured on a 3,678-page
+ corpus, AND over the meaningful terms returned nothing at all for 65 of 99
+ questions, and held the expected page for 25 of 99 against 99 of 99 for OR.
+
+ What OR needs is not a different operator but fewer junk terms. A question
+ carries words that are not stop words yet still match a large share of any
+ code corpus ("file", "does", "page", "index"), and each one drags in
+ thousands of pages that only BM25 tie-breaking then has to sort out. Given
+ a *document_frequency* callable, terms matching more than
+ :data:`_DF_CEILING` of the corpus are dropped from the expression; the same
+ measurement puts the median candidate set at 20% of the corpus rather than
+ 65%. Without the callable the expression keeps every term, which is the
+ prior behaviour.
+
+ At least :data:`_MIN_KEPT_TERMS` terms always survive — the rarest ones —
+ so a question written entirely from common words still matches something.
+ """
+ meaningful = _meaningful_terms(query)
if not meaningful:
# All stop words — fall back to exact phrase
safe = query.replace('"', '""')
return f'"{safe}"'
- # FTS5: OR between prefix-match terms gives broad recall;
- # FTS5 rank (BM25) naturally boosts pages matching more terms.
- return " OR ".join(f'"{t}"*' for t in meaningful)
+ kept = meaningful
+ if document_frequency is not None:
+ total = document_frequency("") # corpus size, by convention
+ if total > 0:
+ ceiling = _DF_CEILING * total
+ selective = [t for t in meaningful if document_frequency(t) <= ceiling]
+ if len(selective) < _MIN_KEPT_TERMS:
+ # Every term is common. Keep the rarest few rather than none:
+ # a thin match beats an empty one, and the vector arm is still
+ # answering alongside this.
+ selective = sorted(meaningful, key=document_frequency)[:_MIN_KEPT_TERMS]
+ kept = selective
+
+ return " OR ".join(_match_term(t) for t in kept)
def _snippet(content: str) -> str:
@@ -162,6 +244,8 @@ class FullTextSearch:
def __init__(self, engine: AsyncEngine) -> None:
self._engine = engine
self._dialect = engine.dialect.name # "sqlite" or "postgresql"
+ # term → how many pages it matches. "" is the corpus size.
+ self._df_cache: OrderedDict[str, int] = OrderedDict()
async def ensure_index(self) -> None:
"""Create the FTS index if it does not exist (idempotent).
@@ -268,22 +352,98 @@ class FullTextSearch:
return await self._search_sqlite(query, limit)
return await self._search_postgresql(query, limit)
+ async def _document_frequency(self, conn, term: str) -> int:
+ """How many indexed pages *term* matches. ``""`` is the corpus size.
+
+ Cached per instance: a term's frequency only moves when the corpus is
+ rewritten, and questions reuse vocabulary heavily, so the count query
+ is paid roughly once per distinct word this process ever sees.
+ """
+ if term in self._df_cache:
+ self._df_cache.move_to_end(term)
+ return self._df_cache[term]
+
+ if term:
+ sql = "SELECT count(*) FROM page_fts WHERE page_fts MATCH :q"
+ params = {"q": _match_term(term)}
+ else:
+ sql = "SELECT count(*) FROM page_fts"
+ params = {}
+ try:
+ row = await conn.execute(text(sql), params)
+ count = int(row.scalar() or 0)
+ except Exception:
+ # A term FTS5 refuses to parse must not fail the search: treat it as
+ # rare so it stays in the expression, where it either matches or is
+ # rejected there instead.
+ _log.warning("Could not count pages matching %r", term, exc_info=True)
+ count = 0
+
+ self._df_cache[term] = count
+ while len(self._df_cache) > _DF_CACHE_MAX:
+ self._df_cache.popitem(last=False)
+ return count
+
+ async def _build_selective_query(self, query: str, df) -> str:
+ """``_build_fts5_query`` with this corpus's term frequencies resolved.
+
+ The builder stays a pure synchronous function — it is the piece worth
+ testing on its own — so the counts it needs are awaited here and handed
+ over as a plain lookup.
+ """
+ terms = _meaningful_terms(query)
+ if not terms:
+ return _build_fts5_query(query)
+ counts = {"": await df("")}
+ for term in terms:
+ if term not in counts:
+ counts[term] = await df(term)
+ return _build_fts5_query(query, counts.__getitem__)
+
+ async def _matching_rows(self, conn, fts_query: str, limit: int) -> list:
+ rows = await conn.execute(
+ text(
+ "SELECT f.page_id, f.title, f.content, f.rank "
+ "FROM page_fts f "
+ "WHERE page_fts MATCH :q "
+ "ORDER BY rank "
+ "LIMIT :lim"
+ ),
+ {"q": fts_query, "lim": limit},
+ )
+ return list(rows.fetchall())
+
async def _search_sqlite(self, query: str, limit: int) -> list[SearchResult]:
"""FTS5 search. ``rank`` is negative; we negate it to get a positive score."""
- fts_query = _build_fts5_query(query)
-
async with self._engine.connect() as conn:
- rows = await conn.execute(
- text(
- "SELECT f.page_id, f.title, f.content, f.rank "
- "FROM page_fts f "
- "WHERE page_fts MATCH :q "
- "ORDER BY rank "
- "LIMIT :lim"
- ),
- {"q": fts_query, "lim": limit},
- )
- raw = rows.fetchall()
+
+ async def df(term: str) -> int:
+ return await self._document_frequency(conn, term)
+
+ fts_query = await self._build_selective_query(query, df)
+ raw = await self._matching_rows(conn, fts_query, limit)
+
+ # The frequency ceiling can cut a question down to terms that
+ # nothing carries together. Retrying with every term is the prior
+ # behaviour, so the narrowing can lose ranking precision but never
+ # the only hits there were.
+ if len(raw) < limit:
+ widened = _build_fts5_query(query)
+ if widened != fts_query:
+ seen = {r[0] for r in raw}
+ extra = [
+ r
+ for r in await self._matching_rows(conn, widened, limit)
+ if r[0] not in seen
+ ]
+ if extra:
+ _log.debug(
+ "FTS widened from %d to %d hits after the frequency ceiling "
+ "left too few",
+ len(raw),
+ len(raw) + len(extra),
+ )
+ raw = (raw + extra)[:limit]
# We need page_type and target_path from wiki_pages
if not raw:
@@ -307,9 +467,18 @@ class FullTextSearch:
meta = {r[0]: (r[1], r[2]) for r in page_rows.fetchall()}
results = []
+ # Rows can come from two MATCH expressions, whose BM25 scales are not
+ # comparable. Order is authoritative — the narrow pass ranks ahead of
+ # the widened one — so scores are clamped to be non-increasing along it,
+ # keeping a caller that sorts by score in agreement with one that reads
+ # the list in order.
+ ceiling: float | None = None
for pid in page_ids:
page_type, target_path = meta.get(pid, ("", ""))
score = -(rank_by_id[pid] or 0.0) # FTS5 rank is negative
+ if ceiling is not None and score > ceiling:
+ score = ceiling
+ ceiling = score - _SCORE_EPSILON
results.append(
SearchResult(
page_id=pid,
@@ -323,23 +492,89 @@ class FullTextSearch:
)
return results
+ async def _pg_document_frequency(self, conn, term: str) -> int:
+ """Pages matching *term* in PostgreSQL. ``""`` is the corpus size.
+
+ Shares :attr:`_df_cache` with the SQLite path; an instance only ever
+ talks to one backend.
+ """
+ if term in self._df_cache:
+ self._df_cache.move_to_end(term)
+ return self._df_cache[term]
+
+ if term:
+ sql = (
+ "SELECT count(*) FROM wiki_pages WHERE to_tsvector('english', "
+ "COALESCE(title,'') || ' ' || COALESCE(content,'')) @@ to_tsquery('english', :q)"
+ )
+ params = {"q": _pg_term(term)}
+ else:
+ sql = "SELECT count(*) FROM wiki_pages"
+ params = {}
+ try:
+ row = await conn.execute(text(sql), params)
+ count = int(row.scalar() or 0)
+ except Exception:
+ _log.warning("Could not count pages matching %r", term, exc_info=True)
+ count = 0
+
+ self._df_cache[term] = count
+ while len(self._df_cache) > _DF_CACHE_MAX:
+ self._df_cache.popitem(last=False)
+ return count
+
+ async def _build_ts_query(self, conn, query: str) -> str:
+ """OR-joined ``to_tsquery`` expression over this query's kept terms."""
+ terms = _meaningful_terms(query)
+ if not terms:
+ return ""
+ total = await self._pg_document_frequency(conn, "")
+ kept = terms
+ if total > 0:
+ selective = []
+ for term in terms:
+ if await self._pg_document_frequency(conn, term) <= _DF_CEILING * total:
+ selective.append(term)
+ if len(selective) < _MIN_KEPT_TERMS:
+ counts = {t: await self._pg_document_frequency(conn, t) for t in terms}
+ selective = sorted(terms, key=counts.__getitem__)[:_MIN_KEPT_TERMS]
+ kept = selective
+ return " | ".join(_pg_term(t) for t in kept)
+
async def _search_postgresql(self, query: str, limit: int) -> list[SearchResult]:
- """PostgreSQL tsvector search with ts_rank scoring."""
+ """PostgreSQL tsvector search with ts_rank scoring.
+
+ The query used to be handed to ``plainto_tsquery`` whole, which strips
+ nothing the caller intended and — more consequentially — joins every
+ remaining lexeme with AND. A developer question is a paraphrase, so
+ requiring all of its words matches almost nothing: on a 3,678-page
+ corpus, AND over the meaningful terms of 99 questions returned no rows
+ at all for 65 of them. This backend was therefore the strict mirror
+ image of SQLite's, which matched two thirds of the corpus, and neither
+ was doing retrieval.
+
+ Both now build the same expression from the same terms with the same
+ frequency ceiling, so a hosted index and a local one rank a question the
+ same way.
+ """
async with self._engine.connect() as conn:
+ ts_query = await self._build_ts_query(conn, query)
+ if not ts_query:
+ return []
rows = await conn.execute(
text(
"SELECT id, title, content, page_type, target_path, "
" ts_rank("
" to_tsvector('english', COALESCE(title,'') || ' ' || COALESCE(content,'')), "
- " plainto_tsquery('english', :q)"
+ " to_tsquery('english', :q)"
" ) AS rank "
"FROM wiki_pages "
"WHERE to_tsvector('english', COALESCE(title,'') || ' ' || COALESCE(content,'')) "
- " @@ plainto_tsquery('english', :q) "
+ " @@ to_tsquery('english', :q) "
"ORDER BY rank DESC "
"LIMIT :lim",
),
- {"q": query, "lim": limit},
+ {"q": ts_query, "lim": limit},
)
raw = rows.fetchall()
diff --git a/tests/unit/persistence/test_search_query_builder.py b/tests/unit/persistence/test_search_query_builder.py
new file mode 100644
index 000000000..4790db04e
--- /dev/null
+++ b/tests/unit/persistence/test_search_query_builder.py
@@ -0,0 +1,229 @@
+"""What the full-text query asks for, and how much of the corpus that matches.
+
+Both backends built an expression that could not retrieve. SQLite OR-ed every
+token with a prefix wildcard, so a question matched two thirds of the corpus and
+BM25 was left to sort out the rest -- a full scan with tie-breaking. PostgreSQL
+handed the raw question to ``plainto_tsquery``, which joins every lexeme with
+AND, so a paraphrased question matched almost nothing. Opposite failures, same
+cause: nothing looked at how many pages a term could actually distinguish.
+
+The fix is a document-frequency ceiling. A term matching more of the corpus than
+the ceiling is dropped from the expression, because it cannot tell two pages
+apart. Measured on a 3,678-page corpus, that moves the median candidate set from
+65% of the corpus to 20% with recall@5 unchanged.
+
+AND-first is deliberately *not* what this does, and the tests below pin why: a
+question is a paraphrase of the page that answers it, so requiring all of its
+terms excludes the right page far more often than it narrows usefully.
+"""
+
+from __future__ import annotations
+
+import pytest
+from sqlalchemy.sql import text
+
+from repowise.core.persistence.search import FullTextSearch, _build_fts5_query
+
+
+def _df(counts: dict[str, int], total: int):
+ """A document-frequency lookup over fixed counts. ``""`` is the corpus size."""
+
+ def lookup(term: str) -> int:
+ return total if term == "" else counts.get(term, 0)
+
+ return lookup
+
+
+# --- term selection --------------------------------------------------------
+
+
+class TestMeaningfulTerms:
+ def test_stop_words_and_single_characters_go(self):
+ from repowise.core.persistence.search import _meaningful_terms
+
+ assert _meaningful_terms("How does the a b walker work") == ["walker", "work"]
+
+ def test_punctuation_is_not_a_term(self):
+ from repowise.core.persistence.search import _meaningful_terms
+
+ assert _meaningful_terms("walker, nested-git (checkouts)!") == [
+ "walker",
+ "nested",
+ "git",
+ "checkouts",
+ ]
+
+
+class TestPrefixWildcard:
+ """The wildcard buys morphology on long terms and floods on short ones."""
+
+ def test_a_long_term_keeps_its_wildcard(self):
+ assert '"invalidation"*' in _build_fts5_query("invalidation")
+
+ def test_a_short_term_is_matched_exactly(self):
+ # "set*" matches settings, setup, setter, setdefault: a corpus-wide hit
+ # from a three-letter word.
+ assert _build_fts5_query("set") == '"set"'
+
+ def test_the_cut_is_where_the_constant_says(self):
+ from repowise.core.persistence.search import _PREFIX_MIN_CHARS
+
+ short = "x" * (_PREFIX_MIN_CHARS - 1)
+ long = "x" * _PREFIX_MIN_CHARS
+ assert _build_fts5_query(short) == f'"{short}"'
+ assert _build_fts5_query(long) == f'"{long}"*'
+
+
+# --- the frequency ceiling -------------------------------------------------
+
+
+class TestFrequencyCeiling:
+ def test_without_a_lookup_every_term_is_kept(self):
+ """The prior behaviour, which every caller without counts still gets."""
+ built = _build_fts5_query("walker nested checkouts")
+ assert built.count(" OR ") == 2
+
+ def test_a_term_matching_most_of_the_corpus_is_dropped(self):
+ counts = {"walker": 30, "file": 900, "checkouts": 40, "nested": 25, "descending": 12}
+ built = _build_fts5_query(
+ "walker file checkouts nested descending", _df(counts, total=1000)
+ )
+ assert "walker" in built
+ assert "checkouts" in built
+ assert "file" not in built
+
+ def test_the_floor_outranks_the_ceiling_on_a_short_question(self):
+ """A question with too few selective terms keeps its rarest few even when
+ the ceiling would reject them.
+
+ This is the measured configuration, not an accident of ordering: the
+ eval's questions run 8 to 15 meaningful terms, where the ceiling decides,
+ and the floor only takes over on questions too short to narrow safely.
+ """
+ counts = {"walker": 30, "file": 900, "checkouts": 40}
+ built = _build_fts5_query("walker file checkouts", _df(counts, total=1000))
+ assert "file" in built, "with only three terms the floor keeps all of them"
+
+ def test_a_term_just_under_the_ceiling_survives(self):
+ from repowise.core.persistence.search import _DF_CEILING
+
+ total = 1000
+ counts = {"walker": 10, "index": int(_DF_CEILING * total) - 1, "nested": 10}
+ built = _build_fts5_query("walker index nested", _df(counts, total))
+ assert "index" in built
+
+ def test_the_ceiling_never_empties_the_query(self):
+ """A question written entirely from common words still has to match
+ something: a thin match beats none, and vector retrieval is answering
+ alongside this."""
+ from repowise.core.persistence.search import _MIN_KEPT_TERMS
+
+ counts = {"file": 900, "page": 950, "index": 800, "does": 990}
+ built = _build_fts5_query("file page index does", _df(counts, total=1000))
+ kept = [t for t in ("file", "page", "index", "does") if t in built]
+ assert len(kept) == _MIN_KEPT_TERMS
+ # The rarest ones, not an arbitrary three.
+ assert set(kept) == {"index", "file", "page"}
+
+ def test_terms_are_or_ed_never_and_ed(self):
+ """AND is the tempting fix and the wrong one.
+
+ On a 3,678-page corpus, AND over the meaningful terms of 99 developer
+ questions returned no rows at all for 65 of them, and held the expected
+ page for 25 of 99 against 99 of 99 for OR. A question is a paraphrase;
+ its rarest words are often the asker's, not the page's.
+ """
+ built = _build_fts5_query("walker nested checkouts", _df({}, total=0))
+ assert " AND " not in built
+ assert built.count(" OR ") == 2
+
+ def test_an_empty_corpus_keeps_every_term(self):
+ """Nothing is measurable against zero pages, so nothing is dropped."""
+ built = _build_fts5_query("walker file checkouts", _df({}, total=0))
+ assert built.count(" OR ") == 2
+
+ def test_a_question_of_only_stop_words_falls_back_to_a_phrase(self):
+ assert _build_fts5_query("how does it") == '"how does it"'
+
+
+class TestPostgresTerms:
+ """The same terms, in the other backend's syntax."""
+
+ def test_a_long_term_is_a_prefix_lexeme(self):
+ from repowise.core.persistence.search import _pg_term
+
+ assert _pg_term("invalidation") == "invalidation:*"
+
+ def test_a_short_term_is_exact(self):
+ from repowise.core.persistence.search import _pg_term
+
+ assert _pg_term("set") == "set"
+
+
+# --- against a real index --------------------------------------------------
+
+
+async def fts(async_engine):
+ fs = FullTextSearch(async_engine)
+ await fs.ensure_index()
+ return fs
+
+
+async def _index_corpus(fts, common_pages: int) -> None:
+ """One page that answers a question, plus *common_pages* that only share its
+ filler words -- the shape that makes a whole-corpus match possible."""
+ await fts.index(
+ "gold",
+ "File: walk.py",
+ "The walker skips any nested checkout it finds while descending.",
+ )
+ for i in range(common_pages):
+ await fts.index(f"noise{i}", f"File: other{i}.py", "This file documents a page index.")
+
+
+class TestCandidateSetSize:
+ async def test_a_common_term_no_longer_drags_the_corpus_in(self, fts, async_engine):
+ """The regression this exists to catch: one filler word matching every
+ page, and BM25 asked to rank the whole corpus afterwards."""
+ await _index_corpus(fts, common_pages=40)
+
+ hits = await fts.search("which file documents how the walker skips a nested checkout", 5)
+
+ assert hits, "the answering page must still be found"
+ assert hits[0].page_id == "gold"
+
+ question = "which file documents how the walker skips a nested checkout"
+ async with async_engine.connect() as conn:
+
+ async def df(term: str) -> int:
+ return await fts._document_frequency(conn, term)
+
+ expression = await fts._build_selective_query(question, df)
+ matched = await conn.execute(
+ text("SELECT count(*) FROM page_fts WHERE page_fts MATCH :q"),
+ {"q": expression},
+ )
+ candidates = matched.scalar()
+
+ assert candidates < 41, (
+ f"{candidates} of 41 pages matched by {expression!r}: the filler terms "
+ "are still in the query"
+ )
+
+ async def test_results_stay_ordered_by_score(self, fts):
+ """Two MATCH expressions' BM25 scales are not comparable, so a widened
+ retry must not out-score the narrow hits it was appended to."""
+ await _index_corpus(fts, common_pages=8)
+
+ hits = await fts.search("walker nested checkout descending page index file", 10)
+
+ scores = [h.score for h in hits]
+ assert scores == sorted(scores, reverse=True), scores
+
+ async def test_a_question_whose_terms_are_all_common_still_retrieves(self, fts):
+ await _index_corpus(fts, common_pages=30)
+
+ hits = await fts.search("this file documents a page index", 5)
+
+ assert hits