Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(search): drop a tombstoned page from the full-text index (#1190)

Swati Ahuja9704991ede18
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
index 27e449762..f043e7067 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
@@ -448,6 +448,9 @@ async def _persist_full_update_async(
url = get_db_url_for_repo(repo_path)
engine = create_engine(url)
+ # Filled by the tombstone step; read by the full-text block after the
+ # session closes, so it has to survive a step that was skipped.
+ tombstoned_page_ids: list[str] = []
try:
await init_db(engine)
sf = create_session_factory(engine)
@@ -470,7 +473,9 @@ async def _persist_full_update_async(
tombstone_candidates,
)
- await mark_tombstone_pages(session, repo_id, tombstone_candidates(file_diffs))
+ tombstoned_page_ids = await mark_tombstone_pages(
+ session, repo_id, tombstone_candidates(file_diffs)
+ )
except Exception as exc:
_skip("Tombstone marking", exc)
@@ -750,6 +755,12 @@ async def _persist_full_update_async(
summary=page.summary,
target_path=page.target_path,
)
+ # A tombstone can never be an answer — hydration drops it — but
+ # retrieval fetches a fixed number of rows before that check runs,
+ # so every tombstone left in the index costs a real candidate its
+ # slot.
+ if tombstoned_page_ids:
+ await fts.delete_many(tombstoned_page_ids)
except Exception as exc:
_skip("Full-text search indexing", exc)
return total_pages
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.py
index 7f1da02f6..d3cf54f1d 100644
--- a/packages/core/src/repowise/core/pipeline/incremental.py
+++ b/packages/core/src/repowise/core/pipeline/incremental.py
@@ -934,6 +934,9 @@ async def persist_incremental_index(
url = resolve_db_url(repo_path)
engine = create_engine(url)
+ # Filled by the tombstone step; read after the session closes, so it has to
+ # survive a step that was skipped.
+ tombstoned_page_ids: list[str] = []
try:
await init_db(engine)
sf = create_session_factory(engine)
@@ -960,7 +963,9 @@ async def persist_incremental_index(
tombstone_candidates,
)
- await mark_tombstone_pages(session, repo_id, tombstone_candidates(file_diffs))
+ tombstoned_page_ids = await mark_tombstone_pages(
+ session, repo_id, tombstone_candidates(file_diffs)
+ )
except Exception as exc:
_skip("Tombstone marking", exc)
@@ -1102,5 +1107,22 @@ async def persist_incremental_index(
await purge_proposed_decisions_by_source(session, repo_id, "code_comment")
except Exception as exc:
_skip("Decision purge", exc)
+
+ # After the session closes: on SQLite the full-text index shares the
+ # database file, so writing to it while the session holds a write lock
+ # raises "database is locked".
+ #
+ # A tombstone can never be an answer — hydration drops it — but
+ # retrieval fetches a fixed number of rows before that check runs, so
+ # every tombstone left in the index costs a real candidate its slot.
+ if tombstoned_page_ids:
+ try:
+ from repowise.core.persistence.search import FullTextSearch
+
+ fts = FullTextSearch(engine)
+ await fts.ensure_index()
+ await fts.delete_many(tombstoned_page_ids)
+ except Exception as exc:
+ _skip("Tombstone full-text removal", exc)
finally:
await engine.dispose()
diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.py
index d1d9c203d..2f7ed14f5 100644
--- a/packages/core/src/repowise/core/pipeline/persist.py
+++ b/packages/core/src/repowise/core/pipeline/persist.py
@@ -43,7 +43,7 @@ def tombstone_candidates(file_diffs: list[Any]) -> list[tuple[str, list[str]]]:
async def mark_tombstone_pages(
session: Any, repo_id: str, candidates: list[tuple[str, list[str]]]
-) -> int:
+) -> list[str]:
"""Mark file pages for deleted/renamed files as tombstones.
A ``freshness_status="fresh"`` page for a file that no longer exists is
@@ -52,10 +52,16 @@ async def mark_tombstone_pages(
(the prose may still orient a reader) but carry status=tombstone and
``successor_paths`` in metadata so serving layers can skip or redirect.
- Returns the number of pages marked.
+ Returns the page ids marked, so the caller can drop them from the
+ full-text index once its session has closed. Every serving layer already
+ discards a tombstone, but retrieval fetches a fixed number of rows before
+ any of that runs, so a tombstone still takes one of those slots and pushes
+ a real candidate out of the fetch. The row has to go, and only the caller
+ knows when it is safe to write to the index — on SQLite it shares a file
+ with this session.
"""
if not candidates:
- return 0
+ return []
from sqlalchemy import select
from repowise.core.persistence.models import Page
@@ -64,7 +70,7 @@ async def mark_tombstone_pages(
res = await session.execute(
select(Page).where(Page.repository_id == repo_id, Page.id.in_(page_ids))
)
- marked = 0
+ marked: list[str] = []
for page in res.scalars().all():
_, successors = page_ids[page.id]
page.freshness_status = "tombstone"
@@ -74,9 +80,9 @@ async def mark_tombstone_pages(
meta = {}
meta["successor_paths"] = successors
page.metadata_json = json.dumps(meta)
- marked += 1
+ marked.append(page.id)
if marked:
- logger.info("pages_tombstoned", repo_id=repo_id, count=marked)
+ logger.info("pages_tombstoned", repo_id=repo_id, count=len(marked))
elif candidates:
# Candidates existed but no page matched — the id scheme drifted from
# ``file_page:{path}`` or the paths don't line up. Silent success here
diff --git a/tests/unit/pipeline/test_tombstone_fts_removal.py b/tests/unit/pipeline/test_tombstone_fts_removal.py
new file mode 100644
index 000000000..8e012c183
--- /dev/null
+++ b/tests/unit/pipeline/test_tombstone_fts_removal.py
@@ -0,0 +1,129 @@
+"""A tombstoned page leaves the full-text index instead of holding a slot.
+
+A tombstone documents a file that no longer exists, and every serving layer
+already drops one: hydration in the answer pipeline discards it, and the
+search tools filter it out. But retrieval fetches a fixed number of rows
+*before* any of that runs, so a tombstone still occupies one of those slots
+and pushes a real candidate out of the fetch entirely. The page cannot be an
+answer either way; the cost is the page it displaces.
+
+Deleting the row is the only fix that works before the fetch. Filtering
+afterwards is what already happens, and it is too late.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+from sqlalchemy.pool import StaticPool
+
+from repowise.core.persistence.crud import upsert_page, upsert_repository
+from repowise.core.persistence.database import init_db
+from repowise.core.persistence.search import FullTextSearch
+from repowise.core.pipeline.persist import mark_tombstone_pages, tombstone_candidates
+
+
+async def engine():
+ eng = create_async_engine(
+ "sqlite+aiosqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ await init_db(eng)
+ yield eng
+ await eng.dispose()
+
+
+async def session(engine):
+ factory = async_sessionmaker(engine, expire_on_commit=False)
+ async with factory() as sess:
+ yield sess
+
+
+async def _seed(session, fts: FullTextSearch, *paths: str) -> str:
+ repo = await upsert_repository(session, name="r", local_path="/tmp/r")
+ await session.commit()
+ for path in paths:
+ await upsert_page(
+ session,
+ page_id=f"file_page:{path}",
+ repository_id=repo.id,
+ page_type="file_page",
+ title=f"File: {path}",
+ content=f"# Overview\n\nThe module at {path} handles xylophone tuning.",
+ summary=f"What {path} does.",
+ target_path=path,
+ source_hash="h",
+ model_name="mock",
+ provider_name="mock",
+ )
+ await fts.index(
+ f"file_page:{path}",
+ f"File: {path}",
+ f"# Overview\n\nThe module at {path} handles xylophone tuning.",
+ summary=f"What {path} does.",
+ target_path=path,
+ )
+ await session.commit()
+ return repo.id
+
+
+async def test_marking_returns_the_page_ids_it_marked(session, engine):
+ """The caller cannot delete rows it has not been told about.
+
+ The count this used to return is derivable from the list; the list is not
+ derivable from the count.
+ """
+ fts = FullTextSearch(engine)
+ await fts.ensure_index()
+ repo_id = await _seed(session, fts, "src/gone.py", "src/kept.py")
+
+ marked = await mark_tombstone_pages(session, repo_id, [("src/gone.py", [])])
+
+ assert marked == ["file_page:src/gone.py"]
+
+
+async def test_a_tombstoned_page_is_deleted_from_the_full_text_index(session, engine):
+ fts = FullTextSearch(engine)
+ await fts.ensure_index()
+ repo_id = await _seed(session, fts, "src/gone.py", "src/kept.py")
+ assert len(await fts.search("xylophone", limit=10)) == 2
+
+ marked = await mark_tombstone_pages(session, repo_id, [("src/gone.py", [])])
+ await session.commit()
+ await fts.delete_many(marked)
+
+ assert [r.page_id for r in await fts.search("xylophone", limit=10)] == ["file_page:src/kept.py"]
+
+
+async def test_marking_nothing_returns_an_empty_list(session, engine):
+ """An empty list, never ``None`` — the caller passes it straight to delete."""
+ fts = FullTextSearch(engine)
+ await fts.ensure_index()
+ repo_id = await _seed(session, fts, "src/kept.py")
+
+ assert await mark_tombstone_pages(session, repo_id, []) == []
+ assert await mark_tombstone_pages(session, repo_id, [("src/never-existed.py", [])]) == []
+
+
+async def test_a_renamed_file_is_dropped_under_its_old_path(session, engine):
+ """A rename tombstones the old page, which must leave the index too.
+
+ The new path gets its own page on the next run. Leaving the old one
+ searchable means both are candidates for the same file, and only one of
+ them describes code that exists.
+ """
+ fts = FullTextSearch(engine)
+ await fts.ensure_index()
+ repo_id = await _seed(session, fts, "src/old.py")
+ diffs = [SimpleNamespace(status="renamed", path="src/new.py", old_path="src/old.py")]
+
+ marked = await mark_tombstone_pages(session, repo_id, tombstone_candidates(diffs))
+ await session.commit()
+ await fts.delete_many(marked)
+
+ assert await fts.search("xylophone", limit=10) == []
diff --git a/tests/unit/server/mcp/test_tombstones.py b/tests/unit/server/mcp/test_tombstones.py
index 863e48f01..0deccc26b 100644
--- a/tests/unit/server/mcp/test_tombstones.py
+++ b/tests/unit/server/mcp/test_tombstones.py
@@ -49,7 +49,7 @@ async def test_mark_tombstone_pages_sets_status_and_successors(setup_mcp, sessio
)
await session.commit()
- assert marked == 1
+ assert marked == ["file_page:src/auth/service.py"]
await session.refresh(page)
assert page.freshness_status == "tombstone"
assert json.loads(page.metadata_json)["successor_paths"] == ["src/auth/service_v2.py"]