Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(search): stop asking full-text search for most of the corpus (#1188)
Swati Ahuja957ddc7dadaa
diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.pyindex 0f35358c3..0029ab60b 100644--- a/packages/core/src/repowise/core/persistence/search.py+++ b/packages/core/src/repowise/core/persistence/search.py@@ -16,6 +16,10 @@ Usage::from __future__ import annotations+import logging+import os+from collections import OrderedDict+from collections.abc import Callablefrom dataclasses import dataclassfrom typing import Literal@@ -38,6 +42,32 @@ class SearchResult:_SNIPPET_LEN = 200+# Shortest term that still gets a prefix wildcard. See :func:`_match_term`.+_PREFIX_MIN_CHARS = 4++# A term matching more than this share of the corpus is dropped from the query+# expression: it cannot discriminate between pages, and every one of them drags+# thousands of candidates in for BM25 to sort out afterwards. Env-tunable+# because the right ceiling is a property of a corpus's vocabulary, and the only+# honest way to set it is to measure recall on the corpus in question.+_DF_CEILING = float(os.environ.get("REPOWISE_FTS_DF_CEILING", "0.20"))++# Never let the ceiling empty a query. A question written entirely out of common+# words keeps its rarest terms — a thin match beats no match, especially since+# vector retrieval is answering the same question alongside this.+_MIN_KEPT_TERMS = 3++# How many distinct terms' document frequencies one FullTextSearch keeps. Term+# frequency changes only when the corpus is rewritten, and questions reuse+# vocabulary heavily, so a small cache removes almost all of the counting.+_DF_CACHE_MAX = 2048++# Keeps scores strictly decreasing when two MATCH expressions' results are+# concatenated. Small enough not to disturb any threshold downstream.+_SCORE_EPSILON = 1e-6++_log = logging.getLogger(__name__)+# Common English stop words to strip from FTS queries_STOP_WORDS = frozenset({@@ -128,28 +158,80 @@ _STOP_WORDS = frozenset()-def _build_fts5_query(query: str) -> str:- """Build an FTS5 MATCH expression from a natural-language query.-- Strips stop words, then joins remaining terms with OR so that pages- containing *any* keyword match. Each term gets a ``*`` suffix for- prefix matching (e.g. "pay*" matches "payment", "payload", etc.).- Falls back to the raw (quoted) query when all tokens are stop words.- """+def _meaningful_terms(query: str) -> list[str]:+ """Alphanumeric tokens of *query*, minus stop words and single characters."""import re- # Keep only alphanumeric tokenstokens = re.findall(r"[a-zA-Z0-9_]+", query.lower())- meaningful = [t for t in tokens if t not in _STOP_WORDS and len(t) > 1]+ return [t for t in tokens if t not in _STOP_WORDS and len(t) > 1]+++def _match_term(term: str) -> str:+ """One FTS5 term, prefix-matched only when it is long enough to be specific.++ A prefix wildcard is what earns morphological recall — ``invalidat*`` finds+ "invalidates" and "invalidation" — and it is also what turns a short token+ into a corpus-wide match: ``set*`` hits settings, setup, setter, setdefault.+ Below the cut the term is matched exactly, which costs the odd plural and+ buys back a candidate set that BM25 was being asked to sort out afterwards.+ """+ return f'"{term}"*' if len(term) >= _PREFIX_MIN_CHARS else f'"{term}"'+++def _pg_term(term: str) -> str:+ """One ``to_tsquery`` lexeme, prefix-matched on the same rule as FTS5.++ Terms reach here already reduced to ``[a-zA-Z0-9_]+`` by+ :func:`_meaningful_terms`, so there is no quoting to get wrong — anything+ that could change the expression's shape has been dropped before this.+ """+ return f"{term}:*" if len(term) >= _PREFIX_MIN_CHARS else term+++def _build_fts5_query(query: str, document_frequency: Callable[[str], int] | None = None) -> str:+ """Build an FTS5 MATCH expression from a natural-language query.++ Terms are OR-ed, because a developer question is a paraphrase of the page+ that answers it: its rarest words are frequently the asker's vocabulary+ rather than the page's, so requiring all of them (AND) excludes the right+ page far more often than it narrows usefully. Measured on a 3,678-page+ corpus, AND over the meaningful terms returned nothing at all for 65 of 99+ questions, and held the expected page for 25 of 99 against 99 of 99 for OR.++ What OR needs is not a different operator but fewer junk terms. A question+ carries words that are not stop words yet still match a large share of any+ code corpus ("file", "does", "page", "index"), and each one drags in+ thousands of pages that only BM25 tie-breaking then has to sort out. Given+ a *document_frequency* callable, terms matching more than+ :data:`_DF_CEILING` of the corpus are dropped from the expression; the same+ measurement puts the median candidate set at 20% of the corpus rather than+ 65%. Without the callable the expression keeps every term, which is the+ prior behaviour.++ At least :data:`_MIN_KEPT_TERMS` terms always survive — the rarest ones —+ so a question written entirely from common words still matches something.+ """+ meaningful = _meaningful_terms(query)if not meaningful:# All stop words — fall back to exact phrasesafe = query.replace('"', '""')return f'"{safe}"'- # FTS5: OR between prefix-match terms gives broad recall;- # FTS5 rank (BM25) naturally boosts pages matching more terms.- return " OR ".join(f'"{t}"*' for t in meaningful)+ kept = meaningful+ if document_frequency is not None:+ total = document_frequency("") # corpus size, by convention+ if total > 0:+ ceiling = _DF_CEILING * total+ selective = [t for t in meaningful if document_frequency(t) <= ceiling]+ if len(selective) < _MIN_KEPT_TERMS:+ # Every term is common. Keep the rarest few rather than none:+ # a thin match beats an empty one, and the vector arm is still+ # answering alongside this.+ selective = sorted(meaningful, key=document_frequency)[:_MIN_KEPT_TERMS]+ kept = selective++ return " OR ".join(_match_term(t) for t in kept)def _snippet(content: str) -> str:@@ -162,6 +244,8 @@ class FullTextSearch:def __init__(self, engine: AsyncEngine) -> None:self._engine = engineself._dialect = engine.dialect.name # "sqlite" or "postgresql"+ # term → how many pages it matches. "" is the corpus size.+ self._df_cache: OrderedDict[str, int] = OrderedDict()async def ensure_index(self) -> None:"""Create the FTS index if it does not exist (idempotent).@@ -268,22 +352,98 @@ class FullTextSearch:return await self._search_sqlite(query, limit)return await self._search_postgresql(query, limit)+ async def _document_frequency(self, conn, term: str) -> int:+ """How many indexed pages *term* matches. ``""`` is the corpus size.++ Cached per instance: a term's frequency only moves when the corpus is+ rewritten, and questions reuse vocabulary heavily, so the count query+ is paid roughly once per distinct word this process ever sees.+ """+ if term in self._df_cache:+ self._df_cache.move_to_end(term)+ return self._df_cache[term]++ if term:+ sql = "SELECT count(*) FROM page_fts WHERE page_fts MATCH :q"+ params = {"q": _match_term(term)}+ else:+ sql = "SELECT count(*) FROM page_fts"+ params = {}+ try:+ row = await conn.execute(text(sql), params)+ count = int(row.scalar() or 0)+ except Exception:+ # A term FTS5 refuses to parse must not fail the search: treat it as+ # rare so it stays in the expression, where it either matches or is+ # rejected there instead.+ _log.warning("Could not count pages matching %r", term, exc_info=True)+ count = 0++ self._df_cache[term] = count+ while len(self._df_cache) > _DF_CACHE_MAX:+ self._df_cache.popitem(last=False)+ return count++ async def _build_selective_query(self, query: str, df) -> str:+ """``_build_fts5_query`` with this corpus's term frequencies resolved.++ The builder stays a pure synchronous function — it is the piece worth+ testing on its own — so the counts it needs are awaited here and handed+ over as a plain lookup.+ """+ terms = _meaningful_terms(query)+ if not terms:+ return _build_fts5_query(query)+ counts = {"": await df("")}+ for term in terms:+ if term not in counts:+ counts[term] = await df(term)+ return _build_fts5_query(query, counts.__getitem__)++ async def _matching_rows(self, conn, fts_query: str, limit: int) -> list:+ rows = await conn.execute(+ text(+ "SELECT f.page_id, f.title, f.content, f.rank "+ "FROM page_fts f "+ "WHERE page_fts MATCH :q "+ "ORDER BY rank "+ "LIMIT :lim"+ ),+ {"q": fts_query, "lim": limit},+ )+ return list(rows.fetchall())+async def _search_sqlite(self, query: str, limit: int) -> list[SearchResult]:"""FTS5 search. ``rank`` is negative; we negate it to get a positive score."""- fts_query = _build_fts5_query(query)-async with self._engine.connect() as conn:- rows = await conn.execute(- text(- "SELECT f.page_id, f.title, f.content, f.rank "- "FROM page_fts f "- "WHERE page_fts MATCH :q "- "ORDER BY rank "- "LIMIT :lim"- ),- {"q": fts_query, "lim": limit},- )- raw = rows.fetchall()++ async def df(term: str) -> int:+ return await self._document_frequency(conn, term)++ fts_query = await self._build_selective_query(query, df)+ raw = await self._matching_rows(conn, fts_query, limit)++ # The frequency ceiling can cut a question down to terms that+ # nothing carries together. Retrying with every term is the prior+ # behaviour, so the narrowing can lose ranking precision but never+ # the only hits there were.+ if len(raw) < limit:+ widened = _build_fts5_query(query)+ if widened != fts_query:+ seen = {r[0] for r in raw}+ extra = [+ r+ for r in await self._matching_rows(conn, widened, limit)+ if r[0] not in seen+ ]+ if extra:+ _log.debug(+ "FTS widened from %d to %d hits after the frequency ceiling "+ "left too few",+ len(raw),+ len(raw) + len(extra),+ )+ raw = (raw + extra)[:limit]# We need page_type and target_path from wiki_pagesif not raw:@@ -307,9 +467,18 @@ class FullTextSearch:meta = {r[0]: (r[1], r[2]) for r in page_rows.fetchall()}results = []+ # Rows can come from two MATCH expressions, whose BM25 scales are not+ # comparable. Order is authoritative — the narrow pass ranks ahead of+ # the widened one — so scores are clamped to be non-increasing along it,+ # keeping a caller that sorts by score in agreement with one that reads+ # the list in order.+ ceiling: float | None = Nonefor pid in page_ids:page_type, target_path = meta.get(pid, ("", ""))score = -(rank_by_id[pid] or 0.0) # FTS5 rank is negative+ if ceiling is not None and score > ceiling:+ score = ceiling+ ceiling = score - _SCORE_EPSILONresults.append(SearchResult(page_id=pid,@@ -323,23 +492,89 @@ class FullTextSearch:)return results+ async def _pg_document_frequency(self, conn, term: str) -> int:+ """Pages matching *term* in PostgreSQL. ``""`` is the corpus size.++ Shares :attr:`_df_cache` with the SQLite path; an instance only ever+ talks to one backend.+ """+ if term in self._df_cache:+ self._df_cache.move_to_end(term)+ return self._df_cache[term]++ if term:+ sql = (+ "SELECT count(*) FROM wiki_pages WHERE to_tsvector('english', "+ "COALESCE(title,'') || ' ' || COALESCE(content,'')) @@ to_tsquery('english', :q)"+ )+ params = {"q": _pg_term(term)}+ else:+ sql = "SELECT count(*) FROM wiki_pages"+ params = {}+ try:+ row = await conn.execute(text(sql), params)+ count = int(row.scalar() or 0)+ except Exception:+ _log.warning("Could not count pages matching %r", term, exc_info=True)+ count = 0++ self._df_cache[term] = count+ while len(self._df_cache) > _DF_CACHE_MAX:+ self._df_cache.popitem(last=False)+ return count++ async def _build_ts_query(self, conn, query: str) -> str:+ """OR-joined ``to_tsquery`` expression over this query's kept terms."""+ terms = _meaningful_terms(query)+ if not terms:+ return ""+ total = await self._pg_document_frequency(conn, "")+ kept = terms+ if total > 0:+ selective = []+ for term in terms:+ if await self._pg_document_frequency(conn, term) <= _DF_CEILING * total:+ selective.append(term)+ if len(selective) < _MIN_KEPT_TERMS:+ counts = {t: await self._pg_document_frequency(conn, t) for t in terms}+ selective = sorted(terms, key=counts.__getitem__)[:_MIN_KEPT_TERMS]+ kept = selective+ return " | ".join(_pg_term(t) for t in kept)+async def _search_postgresql(self, query: str, limit: int) -> list[SearchResult]:- """PostgreSQL tsvector search with ts_rank scoring."""+ """PostgreSQL tsvector search with ts_rank scoring.++ The query used to be handed to ``plainto_tsquery`` whole, which strips+ nothing the caller intended and — more consequentially — joins every+ remaining lexeme with AND. A developer question is a paraphrase, so+ requiring all of its words matches almost nothing: on a 3,678-page+ corpus, AND over the meaningful terms of 99 questions returned no rows+ at all for 65 of them. This backend was therefore the strict mirror+ image of SQLite's, which matched two thirds of the corpus, and neither+ was doing retrieval.++ Both now build the same expression from the same terms with the same+ frequency ceiling, so a hosted index and a local one rank a question the+ same way.+ """async with self._engine.connect() as conn:+ ts_query = await self._build_ts_query(conn, query)+ if not ts_query:+ return []rows = await conn.execute(text("SELECT id, title, content, page_type, target_path, "" ts_rank("" to_tsvector('english', COALESCE(title,'') || ' ' || COALESCE(content,'')), "- " plainto_tsquery('english', :q)"+ " to_tsquery('english', :q)"" ) AS rank ""FROM wiki_pages ""WHERE to_tsvector('english', COALESCE(title,'') || ' ' || COALESCE(content,'')) "- " @@ plainto_tsquery('english', :q) "+ " @@ to_tsquery('english', :q) ""ORDER BY rank DESC ""LIMIT :lim",),- {"q": query, "lim": limit},+ {"q": ts_query, "lim": limit},)raw = rows.fetchall()diff --git a/tests/unit/persistence/test_search_query_builder.py b/tests/unit/persistence/test_search_query_builder.pynew file mode 100644index 000000000..4790db04e--- /dev/null+++ b/tests/unit/persistence/test_search_query_builder.py@@ -0,0 +1,229 @@+"""What the full-text query asks for, and how much of the corpus that matches.++Both backends built an expression that could not retrieve. SQLite OR-ed every+token with a prefix wildcard, so a question matched two thirds of the corpus and+BM25 was left to sort out the rest -- a full scan with tie-breaking. PostgreSQL+handed the raw question to ``plainto_tsquery``, which joins every lexeme with+AND, so a paraphrased question matched almost nothing. Opposite failures, same+cause: nothing looked at how many pages a term could actually distinguish.++The fix is a document-frequency ceiling. A term matching more of the corpus than+the ceiling is dropped from the expression, because it cannot tell two pages+apart. Measured on a 3,678-page corpus, that moves the median candidate set from+65% of the corpus to 20% with recall@5 unchanged.++AND-first is deliberately *not* what this does, and the tests below pin why: a+question is a paraphrase of the page that answers it, so requiring all of its+terms excludes the right page far more often than it narrows usefully.+"""++from __future__ import annotations++import pytest+from sqlalchemy.sql import text++from repowise.core.persistence.search import FullTextSearch, _build_fts5_query+++def _df(counts: dict[str, int], total: int):+ """A document-frequency lookup over fixed counts. ``""`` is the corpus size."""++ def lookup(term: str) -> int:+ return total if term == "" else counts.get(term, 0)++ return lookup+++# --- term selection --------------------------------------------------------+++class TestMeaningfulTerms:+ def test_stop_words_and_single_characters_go(self):+ from repowise.core.persistence.search import _meaningful_terms++ assert _meaningful_terms("How does the a b walker work") == ["walker", "work"]++ def test_punctuation_is_not_a_term(self):+ from repowise.core.persistence.search import _meaningful_terms++ assert _meaningful_terms("walker, nested-git (checkouts)!") == [+ "walker",+ "nested",+ "git",+ "checkouts",+ ]+++class TestPrefixWildcard:+ """The wildcard buys morphology on long terms and floods on short ones."""++ def test_a_long_term_keeps_its_wildcard(self):+ assert '"invalidation"*' in _build_fts5_query("invalidation")++ def test_a_short_term_is_matched_exactly(self):+ # "set*" matches settings, setup, setter, setdefault: a corpus-wide hit+ # from a three-letter word.+ assert _build_fts5_query("set") == '"set"'++ def test_the_cut_is_where_the_constant_says(self):+ from repowise.core.persistence.search import _PREFIX_MIN_CHARS++ short = "x" * (_PREFIX_MIN_CHARS - 1)+ long = "x" * _PREFIX_MIN_CHARS+ assert _build_fts5_query(short) == f'"{short}"'+ assert _build_fts5_query(long) == f'"{long}"*'+++# --- the frequency ceiling -------------------------------------------------+++class TestFrequencyCeiling:+ def test_without_a_lookup_every_term_is_kept(self):+ """The prior behaviour, which every caller without counts still gets."""+ built = _build_fts5_query("walker nested checkouts")+ assert built.count(" OR ") == 2++ def test_a_term_matching_most_of_the_corpus_is_dropped(self):+ counts = {"walker": 30, "file": 900, "checkouts": 40, "nested": 25, "descending": 12}+ built = _build_fts5_query(+ "walker file checkouts nested descending", _df(counts, total=1000)+ )+ assert "walker" in built+ assert "checkouts" in built+ assert "file" not in built++ def test_the_floor_outranks_the_ceiling_on_a_short_question(self):+ """A question with too few selective terms keeps its rarest few even when+ the ceiling would reject them.++ This is the measured configuration, not an accident of ordering: the+ eval's questions run 8 to 15 meaningful terms, where the ceiling decides,+ and the floor only takes over on questions too short to narrow safely.+ """+ counts = {"walker": 30, "file": 900, "checkouts": 40}+ built = _build_fts5_query("walker file checkouts", _df(counts, total=1000))+ assert "file" in built, "with only three terms the floor keeps all of them"++ def test_a_term_just_under_the_ceiling_survives(self):+ from repowise.core.persistence.search import _DF_CEILING++ total = 1000+ counts = {"walker": 10, "index": int(_DF_CEILING * total) - 1, "nested": 10}+ built = _build_fts5_query("walker index nested", _df(counts, total))+ assert "index" in built++ def test_the_ceiling_never_empties_the_query(self):+ """A question written entirely from common words still has to match+ something: a thin match beats none, and vector retrieval is answering+ alongside this."""+ from repowise.core.persistence.search import _MIN_KEPT_TERMS++ counts = {"file": 900, "page": 950, "index": 800, "does": 990}+ built = _build_fts5_query("file page index does", _df(counts, total=1000))+ kept = [t for t in ("file", "page", "index", "does") if t in built]+ assert len(kept) == _MIN_KEPT_TERMS+ # The rarest ones, not an arbitrary three.+ assert set(kept) == {"index", "file", "page"}++ def test_terms_are_or_ed_never_and_ed(self):+ """AND is the tempting fix and the wrong one.++ On a 3,678-page corpus, AND over the meaningful terms of 99 developer+ questions returned no rows at all for 65 of them, and held the expected+ page for 25 of 99 against 99 of 99 for OR. A question is a paraphrase;+ its rarest words are often the asker's, not the page's.+ """+ built = _build_fts5_query("walker nested checkouts", _df({}, total=0))+ assert " AND " not in built+ assert built.count(" OR ") == 2++ def test_an_empty_corpus_keeps_every_term(self):+ """Nothing is measurable against zero pages, so nothing is dropped."""+ built = _build_fts5_query("walker file checkouts", _df({}, total=0))+ assert built.count(" OR ") == 2++ def test_a_question_of_only_stop_words_falls_back_to_a_phrase(self):+ assert _build_fts5_query("how does it") == '"how does it"'+++class TestPostgresTerms:+ """The same terms, in the other backend's syntax."""++ def test_a_long_term_is_a_prefix_lexeme(self):+ from repowise.core.persistence.search import _pg_term++ assert _pg_term("invalidation") == "invalidation:*"++ def test_a_short_term_is_exact(self):+ from repowise.core.persistence.search import _pg_term++ assert _pg_term("set") == "set"+++# --- against a real index --------------------------------------------------+++async def fts(async_engine):+ fs = FullTextSearch(async_engine)+ await fs.ensure_index()+ return fs+++async def _index_corpus(fts, common_pages: int) -> None:+ """One page that answers a question, plus *common_pages* that only share its+ filler words -- the shape that makes a whole-corpus match possible."""+ await fts.index(+ "gold",+ "File: walk.py",+ "The walker skips any nested checkout it finds while descending.",+ )+ for i in range(common_pages):+ await fts.index(f"noise{i}", f"File: other{i}.py", "This file documents a page index.")+++class TestCandidateSetSize:+ async def test_a_common_term_no_longer_drags_the_corpus_in(self, fts, async_engine):+ """The regression this exists to catch: one filler word matching every+ page, and BM25 asked to rank the whole corpus afterwards."""+ await _index_corpus(fts, common_pages=40)++ hits = await fts.search("which file documents how the walker skips a nested checkout", 5)++ assert hits, "the answering page must still be found"+ assert hits[0].page_id == "gold"++ question = "which file documents how the walker skips a nested checkout"+ async with async_engine.connect() as conn:++ async def df(term: str) -> int:+ return await fts._document_frequency(conn, term)++ expression = await fts._build_selective_query(question, df)+ matched = await conn.execute(+ text("SELECT count(*) FROM page_fts WHERE page_fts MATCH :q"),+ {"q": expression},+ )+ candidates = matched.scalar()++ assert candidates < 41, (+ f"{candidates} of 41 pages matched by {expression!r}: the filler terms "+ "are still in the query"+ )++ async def test_results_stay_ordered_by_score(self, fts):+ """Two MATCH expressions' BM25 scales are not comparable, so a widened+ retry must not out-score the narrow hits it was appended to."""+ await _index_corpus(fts, common_pages=8)++ hits = await fts.search("walker nested checkout descending page index file", 10)++ scores = [h.score for h in hits]+ assert scores == sorted(scores, reverse=True), scores++ async def test_a_question_whose_terms_are_all_common_still_retrieves(self, fts):+ await _index_corpus(fts, common_pages=30)++ hits = await fts.search("this file documents a page index", 5)++ assert hits