Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(search): drop a tombstoned page from the full-text index (#1190)
Swati Ahuja9704991ede18
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.pyindex 27e449762..f043e7067 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py@@ -448,6 +448,9 @@ async def _persist_full_update_async(url = get_db_url_for_repo(repo_path)engine = create_engine(url)+ # Filled by the tombstone step; read by the full-text block after the+ # session closes, so it has to survive a step that was skipped.+ tombstoned_page_ids: list[str] = []try:await init_db(engine)sf = create_session_factory(engine)@@ -470,7 +473,9 @@ async def _persist_full_update_async(tombstone_candidates,)- await mark_tombstone_pages(session, repo_id, tombstone_candidates(file_diffs))+ tombstoned_page_ids = await mark_tombstone_pages(+ session, repo_id, tombstone_candidates(file_diffs)+ )except Exception as exc:_skip("Tombstone marking", exc)@@ -750,6 +755,12 @@ async def _persist_full_update_async(summary=page.summary,target_path=page.target_path,)+ # A tombstone can never be an answer — hydration drops it — but+ # retrieval fetches a fixed number of rows before that check runs,+ # so every tombstone left in the index costs a real candidate its+ # slot.+ if tombstoned_page_ids:+ await fts.delete_many(tombstoned_page_ids)except Exception as exc:_skip("Full-text search indexing", exc)return total_pagesdiff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.pyindex 7f1da02f6..d3cf54f1d 100644--- a/packages/core/src/repowise/core/pipeline/incremental.py+++ b/packages/core/src/repowise/core/pipeline/incremental.py@@ -934,6 +934,9 @@ async def persist_incremental_index(url = resolve_db_url(repo_path)engine = create_engine(url)+ # Filled by the tombstone step; read after the session closes, so it has to+ # survive a step that was skipped.+ tombstoned_page_ids: list[str] = []try:await init_db(engine)sf = create_session_factory(engine)@@ -960,7 +963,9 @@ async def persist_incremental_index(tombstone_candidates,)- await mark_tombstone_pages(session, repo_id, tombstone_candidates(file_diffs))+ tombstoned_page_ids = await mark_tombstone_pages(+ session, repo_id, tombstone_candidates(file_diffs)+ )except Exception as exc:_skip("Tombstone marking", exc)@@ -1102,5 +1107,22 @@ async def persist_incremental_index(await purge_proposed_decisions_by_source(session, repo_id, "code_comment")except Exception as exc:_skip("Decision purge", exc)++ # After the session closes: on SQLite the full-text index shares the+ # database file, so writing to it while the session holds a write lock+ # raises "database is locked".+ #+ # A tombstone can never be an answer — hydration drops it — but+ # retrieval fetches a fixed number of rows before that check runs, so+ # every tombstone left in the index costs a real candidate its slot.+ if tombstoned_page_ids:+ try:+ from repowise.core.persistence.search import FullTextSearch++ fts = FullTextSearch(engine)+ await fts.ensure_index()+ await fts.delete_many(tombstoned_page_ids)+ except Exception as exc:+ _skip("Tombstone full-text removal", exc)finally:await engine.dispose()diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex d1d9c203d..2f7ed14f5 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -43,7 +43,7 @@ def tombstone_candidates(file_diffs: list[Any]) -> list[tuple[str, list[str]]]:async def mark_tombstone_pages(session: Any, repo_id: str, candidates: list[tuple[str, list[str]]]-) -> int:+) -> list[str]:"""Mark file pages for deleted/renamed files as tombstones.A ``freshness_status="fresh"`` page for a file that no longer exists is@@ -52,10 +52,16 @@ async def mark_tombstone_pages((the prose may still orient a reader) but carry status=tombstone and``successor_paths`` in metadata so serving layers can skip or redirect.- Returns the number of pages marked.+ Returns the page ids marked, so the caller can drop them from the+ full-text index once its session has closed. Every serving layer already+ discards a tombstone, but retrieval fetches a fixed number of rows before+ any of that runs, so a tombstone still takes one of those slots and pushes+ a real candidate out of the fetch. The row has to go, and only the caller+ knows when it is safe to write to the index — on SQLite it shares a file+ with this session."""if not candidates:- return 0+ return []from sqlalchemy import selectfrom repowise.core.persistence.models import Page@@ -64,7 +70,7 @@ async def mark_tombstone_pages(res = await session.execute(select(Page).where(Page.repository_id == repo_id, Page.id.in_(page_ids)))- marked = 0+ marked: list[str] = []for page in res.scalars().all():_, successors = page_ids[page.id]page.freshness_status = "tombstone"@@ -74,9 +80,9 @@ async def mark_tombstone_pages(meta = {}meta["successor_paths"] = successorspage.metadata_json = json.dumps(meta)- marked += 1+ marked.append(page.id)if marked:- logger.info("pages_tombstoned", repo_id=repo_id, count=marked)+ logger.info("pages_tombstoned", repo_id=repo_id, count=len(marked))elif candidates:# Candidates existed but no page matched — the id scheme drifted from# ``file_page:{path}`` or the paths don't line up. Silent success herediff --git a/tests/unit/pipeline/test_tombstone_fts_removal.py b/tests/unit/pipeline/test_tombstone_fts_removal.pynew file mode 100644index 000000000..8e012c183--- /dev/null+++ b/tests/unit/pipeline/test_tombstone_fts_removal.py@@ -0,0 +1,129 @@+"""A tombstoned page leaves the full-text index instead of holding a slot.++A tombstone documents a file that no longer exists, and every serving layer+already drops one: hydration in the answer pipeline discards it, and the+search tools filter it out. But retrieval fetches a fixed number of rows+*before* any of that runs, so a tombstone still occupies one of those slots+and pushes a real candidate out of the fetch entirely. The page cannot be an+answer either way; the cost is the page it displaces.++Deleting the row is the only fix that works before the fetch. Filtering+afterwards is what already happens, and it is too late.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine+from sqlalchemy.pool import StaticPool++from repowise.core.persistence.crud import upsert_page, upsert_repository+from repowise.core.persistence.database import init_db+from repowise.core.persistence.search import FullTextSearch+from repowise.core.pipeline.persist import mark_tombstone_pages, tombstone_candidates+++async def engine():+ eng = create_async_engine(+ "sqlite+aiosqlite:///:memory:",+ connect_args={"check_same_thread": False},+ poolclass=StaticPool,+ )+ await init_db(eng)+ yield eng+ await eng.dispose()+++async def session(engine):+ factory = async_sessionmaker(engine, expire_on_commit=False)+ async with factory() as sess:+ yield sess+++async def _seed(session, fts: FullTextSearch, *paths: str) -> str:+ repo = await upsert_repository(session, name="r", local_path="/tmp/r")+ await session.commit()+ for path in paths:+ await upsert_page(+ session,+ page_id=f"file_page:{path}",+ repository_id=repo.id,+ page_type="file_page",+ title=f"File: {path}",+ content=f"# Overview\n\nThe module at {path} handles xylophone tuning.",+ summary=f"What {path} does.",+ target_path=path,+ source_hash="h",+ model_name="mock",+ provider_name="mock",+ )+ await fts.index(+ f"file_page:{path}",+ f"File: {path}",+ f"# Overview\n\nThe module at {path} handles xylophone tuning.",+ summary=f"What {path} does.",+ target_path=path,+ )+ await session.commit()+ return repo.id+++async def test_marking_returns_the_page_ids_it_marked(session, engine):+ """The caller cannot delete rows it has not been told about.++ The count this used to return is derivable from the list; the list is not+ derivable from the count.+ """+ fts = FullTextSearch(engine)+ await fts.ensure_index()+ repo_id = await _seed(session, fts, "src/gone.py", "src/kept.py")++ marked = await mark_tombstone_pages(session, repo_id, [("src/gone.py", [])])++ assert marked == ["file_page:src/gone.py"]+++async def test_a_tombstoned_page_is_deleted_from_the_full_text_index(session, engine):+ fts = FullTextSearch(engine)+ await fts.ensure_index()+ repo_id = await _seed(session, fts, "src/gone.py", "src/kept.py")+ assert len(await fts.search("xylophone", limit=10)) == 2++ marked = await mark_tombstone_pages(session, repo_id, [("src/gone.py", [])])+ await session.commit()+ await fts.delete_many(marked)++ assert [r.page_id for r in await fts.search("xylophone", limit=10)] == ["file_page:src/kept.py"]+++async def test_marking_nothing_returns_an_empty_list(session, engine):+ """An empty list, never ``None`` — the caller passes it straight to delete."""+ fts = FullTextSearch(engine)+ await fts.ensure_index()+ repo_id = await _seed(session, fts, "src/kept.py")++ assert await mark_tombstone_pages(session, repo_id, []) == []+ assert await mark_tombstone_pages(session, repo_id, [("src/never-existed.py", [])]) == []+++async def test_a_renamed_file_is_dropped_under_its_old_path(session, engine):+ """A rename tombstones the old page, which must leave the index too.++ The new path gets its own page on the next run. Leaving the old one+ searchable means both are candidates for the same file, and only one of+ them describes code that exists.+ """+ fts = FullTextSearch(engine)+ await fts.ensure_index()+ repo_id = await _seed(session, fts, "src/old.py")+ diffs = [SimpleNamespace(status="renamed", path="src/new.py", old_path="src/old.py")]++ marked = await mark_tombstone_pages(session, repo_id, tombstone_candidates(diffs))+ await session.commit()+ await fts.delete_many(marked)++ assert await fts.search("xylophone", limit=10) == []diff --git a/tests/unit/server/mcp/test_tombstones.py b/tests/unit/server/mcp/test_tombstones.pyindex 863e48f01..0deccc26b 100644--- a/tests/unit/server/mcp/test_tombstones.py+++ b/tests/unit/server/mcp/test_tombstones.py@@ -49,7 +49,7 @@ async def test_mark_tombstone_pages_sets_status_and_successors(setup_mcp, sessio)await session.commit()- assert marked == 1+ assert marked == ["file_page:src/auth/service.py"]await session.refresh(page)assert page.freshness_status == "tombstone"assert json.loads(page.metadata_json)["successor_paths"] == ["src/auth/service_v2.py"]