Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
"""Unit tests for FullTextSearch.
All tests use an in-memory SQLite engine with the FTS5 virtual table
created by init_db().
"""
from __future__ import annotations
import pytest
from repowise.core.persistence.search import FullTextSearch, SearchResult
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
async def fts(async_engine):
"""FullTextSearch instance tied to the in-memory engine."""
fs = FullTextSearch(async_engine)
await fs.ensure_index()
return fs
# ---------------------------------------------------------------------------
# FTS5 table creation
# ---------------------------------------------------------------------------
async def test_fts5_table_created_after_ensure_index(async_engine):
"""The page_fts virtual table must exist after ensure_index()."""
from sqlalchemy.sql import text
fts = FullTextSearch(async_engine)
await fts.ensure_index()
async with async_engine.connect() as conn:
result = await conn.execute(
text("SELECT name FROM sqlite_master WHERE type='table' AND name='page_fts'")
)
row = result.fetchone()
assert row is not None, "page_fts virtual table was not created"
async def test_ensure_index_is_idempotent(async_engine):
"""Calling ensure_index() twice must not raise."""
fts = FullTextSearch(async_engine)
await fts.ensure_index()
await fts.ensure_index() # should not raise
# ---------------------------------------------------------------------------
# Search behaviour
# ---------------------------------------------------------------------------
async def test_full_text_search_exact_title_match(fts):
await fts.index("p1", "Python Decorator Pattern", "This module explains decorators.")
results = await fts.search("Decorator Pattern")
assert len(results) >= 1
assert results[0].page_id == "p1"
async def test_full_text_search_content_body_match(fts):
await fts.index("p2", "Graph Builder", "The GraphBuilder class builds dependency graphs.")
results = await fts.search("dependency graphs")
assert len(results) >= 1
assert results[0].page_id == "p2"
async def test_full_text_search_no_match_returns_empty(fts):
await fts.index("p3", "File Traverser", "Handles gitignore-aware file traversal.")
results = await fts.search("blockchain cryptocurrency")
assert results == []
async def test_full_text_search_limit_respected(fts):
for i in range(5):
await fts.index(f"page{i}", f"repowise Module {i}", "Module documentation content here.")
results = await fts.search("repowise Module", limit=3)
assert len(results) <= 3
async def test_full_text_search_empty_query_returns_empty(fts):
await fts.index("p", "Title", "Content")
results = await fts.search("")
assert results == []
async def test_full_text_search_score_is_positive(fts):
"""FTS5 rank is negative; FullTextSearch must negate it for SearchResult.score."""
await fts.index("p", "Python async await", "Coroutines and event loops explained.")
results = await fts.search("Python async")
assert len(results) >= 1
assert results[0].score > 0.0
async def test_full_text_search_snippet_truncated_to_200(fts):
long_content = "word " * 200
await fts.index("p", "Title", long_content)
results = await fts.search("word")
assert len(results) >= 1
assert len(results[0].snippet) <= 200
async def test_full_text_search_returns_search_result_type(fts):
await fts.index("p", "Symbol Spotlight", "High-pagerank symbol documentation.")
results = await fts.search("Symbol Spotlight")
assert len(results) >= 1
assert isinstance(results[0], SearchResult)
assert results[0].search_type == "fulltext"
async def test_full_text_search_index_update_reflects_new_content(fts):
"""Re-indexing a page with new content should make new terms searchable."""
await fts.index("p1", "OldTitle", "nothing useful here")
results_before = await fts.search("async decorators")
count_before = len(results_before)
await fts.index("p1", "NewTitle", "async decorators in Python are powerful")
results_after = await fts.search("async decorators")
assert len(results_after) > count_before
async def test_full_text_search_delete_removes_from_index(fts):
await fts.index("to-remove", "Removable Page", "removable unique xylophone content")
results_before = await fts.search("xylophone")
assert len(results_before) >= 1
await fts.delete("to-remove")
results_after = await fts.search("xylophone")
assert results_after == []
async def test_full_text_search_multiple_pages_relevance_ordered(fts):
"""Page with title match should rank higher than page with only content match."""
await fts.index("exact", "Python Asyncio", "asyncio event loop management")
await fts.index(
"content-only", "Event Loop Guide", "Python asyncio is great for concurrent tasks"
)
results = await fts.search("Python Asyncio")
# Both should appear; 'exact' should rank first (title match)
assert len(results) >= 1
assert results[0].page_id == "exact"
# ---------------------------------------------------------------------------
# Batch indexing
# ---------------------------------------------------------------------------
async def _indexed_ids(fts) -> set[str]:
return await fts.list_indexed_ids()
async def test_index_many_writes_the_same_rows_as_indexing_one_at_a_time(async_engine):
"""The batch path is the single-page path, so the two must not diverge."""
pages = [
(f"p{i}", f"Title {i}", f"content about widgets number {i}", f"sum {i}", f"a/b{i}.py", "")
for i in range(20)
]
one_at_a_time = FullTextSearch(async_engine)
await one_at_a_time.ensure_index()
for page in pages:
await one_at_a_time.index(page[0], page[1], page[2], summary=page[3], target_path=page[4])
sequential = sorted(
(r.page_id, r.title) for r in await one_at_a_time.search("widgets", limit=50)
)
await one_at_a_time.delete_many([p[0] for p in pages])
await one_at_a_time.index_many(pages)
batched = sorted((r.page_id, r.title) for r in await one_at_a_time.search("widgets", limit=50))
assert batched == sequential
assert len(batched) == 20
async def test_index_many_spans_more_ids_than_one_statement_can_bind(fts):
"""The delete half chunks; 1,200 ids is past both chunk boundaries.
A wiki is thousands of pages, so the first real corpus would have been the
first test of this path.
"""
pages = [(f"p{i}", f"Title {i}", "a page about chunking", "", f"{i}.py", "") for i in range(1200)]
await fts.index_many(pages)
assert len(await _indexed_ids(fts)) == 1200
# Re-indexing the same ids must replace rather than duplicate them.
await fts.index_many(pages)
assert len(await _indexed_ids(fts)) == 1200
async def test_a_page_id_repeated_in_one_batch_keeps_its_last_entry(fts):
"""Every id is deleted before any is inserted, so a duplicate would double."""
await fts.index_many(
[
("p1", "First", "a page about alpacas", "", "a.py", ""),
("p1", "Second", "a page about zebras", "", "a.py", ""),
]
)
assert await _indexed_ids(fts) == {"p1"}
assert [r.page_id for r in await fts.search("zebras")] == ["p1"]
assert await fts.search("alpacas") == []
async def test_index_many_of_nothing_is_a_no_op(fts):
await fts.index_many([])
assert await _indexed_ids(fts) == set()
async def test_path_only_match_outranks_content_only_match(fts):
"""A page the query names by path beats one that only mentions the word.
Both pages carry the term once, and the path-matching one has the longer
body, so bm25's length normalisation ranks it second under uniform column
weights. Only weighting target_path above content flips it.
The fillers keep the corpus off the degenerate case where every page
matches and bm25 clamps a non-positive idf to a floor, leaving every score
at ~1e-6.
"""
sentence = "This module handles records for the wider system and explains its use. "
await fts.index(
"path-match",
"Collector",
sentence * 6,
summary="A collector module.",
target_path="packages/core/telemetry/collector.py",
)
await fts.index(
"content-match",
"Emitter",
sentence * 2 + " It forwards each record to telemetry.",
summary="An emitter module.",
target_path="packages/core/reporting/emitter.py",
)
await fts.index_many(
[
(
f"filler{i}",
f"Filler {i}",
"An unrelated module about parsing and graphs. " * 3,
"Filler.",
f"packages/core/other/mod{i}.py",
"",
)
for i in range(10)
]
)
results = await fts.search("telemetry")
assert [r.page_id for r in results] == ["path-match", "content-match"]
async def test_full_text_search_repository_id_filter(async_engine, async_session):
"""When repository_id is given, only pages belonging to that repository are returned."""
from repowise.core.persistence import crud
repo_a = await crud.upsert_repository(
async_session, name="repo-a", local_path="/tmp/repo-a", repo_id="repo-a-id"
)
repo_b = await crud.upsert_repository(
async_session, name="repo-b", local_path="/tmp/repo-b", repo_id="repo-b-id"
)
await crud.upsert_page(
async_session,
page_id="p-a",
repository_id=repo_a.id,
page_type="file_page",
title="Worker Service",
content="Handles background queue jobs in worker service.",
summary="Worker queue",
target_path="worker.py",
source_hash="ha",
model_name="m",
provider_name="p",
)
await crud.upsert_page(
async_session,
page_id="p-b",
repository_id=repo_b.id,
page_type="file_page",
title="Queue Handler",
content="Handles background queue jobs in queue handler.",
summary="Queue handler",
target_path="queue.py",
source_hash="hb",
model_name="m",
provider_name="p",
)
fts = FullTextSearch(async_engine)
await fts.ensure_index()
await fts.index("p-a", "Worker Service", "Handles background queue jobs in worker service.")
await fts.index("p-b", "Queue Handler", "Handles background queue jobs in queue handler.")
# Unscoped returns both
all_hits = await fts.search("queue")
assert {r.page_id for r in all_hits} == {"p-a", "p-b"}
# Scoped to repo-a returns only p-a
a_hits = await fts.search("queue", repository_id="repo-a-id")
assert [r.page_id for r in a_hits] == ["p-a"]
# Scoped to repo-b returns only p-b
b_hits = await fts.search("queue", repository_id="repo-b-id")
assert [r.page_id for r in b_hits] == ["p-b"]
# ---------------------------------------------------------------------------
# Batched page writes
# ---------------------------------------------------------------------------
async def test_index_pages_scans_the_index_once_per_id_chunk(async_engine, fts):
"""A delete by the unindexed ``page_id`` scans the whole index, so a wiki
written page by page is quadratic. ``index_pages`` must issue one delete
per id chunk, not one per page, and leave the same rows the loop did."""
from types import SimpleNamespace
from sqlalchemy import event
from sqlalchemy.sql import text
pages = [
SimpleNamespace(
page_id=f"file_page:m{i}.py",
title=f"File: m{i}.py",
content=f"Module {i} documents widget number {i}.",
summary=f"summary {i}",
target_path=f"m{i}.py",
digest="",
)
for i in range(1100)
]
deletes: list[str] = []
def _count(conn, cursor, statement, *_):
if statement.startswith("DELETE FROM page_fts"):
deletes.append(statement)
event.listen(async_engine.sync_engine, "before_cursor_execute", _count)
try:
await fts.index_pages(pages)
finally:
event.remove(async_engine.sync_engine, "before_cursor_execute", _count)
assert len(deletes) == 3 # 1100 ids in chunks of 500
async with async_engine.connect() as conn:
rows = (
await conn.execute(text("SELECT page_id, title, summary, target_path FROM page_fts"))
).fetchall()
assert sorted(rows) == sorted((p.page_id, p.title, p.summary, p.target_path) for p in pages)