Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(index): tombstone a page whose file is no longer on disk (#1202)
Swati Ahuja439bbd22d6a5
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.pyindex a3d05d93f..db2851abe 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py@@ -116,6 +116,7 @@ async def persist_result(result: Any, repo_path: Path) -> None:persist_generation,persist_pipeline_result,sweep_stale_generated_pages,+ tombstone_absent_file_pages,)engine, sf, _repo_id = await open_repo_db(repo_path, repo_name=result.repo_name)@@ -177,6 +178,20 @@ async def persist_result(result: Any, repo_path: Path) -> None:else:swept_page_ids = await persist_pipeline_result(result, session, repo.id)+ # Tombstone pages whose file is simply gone. The other tombstone path+ # reads a diff, and an init compares nothing, so until here a page+ # written before its file was deleted stayed ``fresh`` through every+ # later index. Outside the branch above because both branches need it.+ # Its ids join the swept ones so they leave the vector store and the+ # full-text index together, below, after this session closes.+ try:+ swept_page_ids = [+ *swept_page_ids,+ *await tombstone_absent_file_pages(session, repo.id, repo_path),+ ]+ except Exception as exc: # one stale page must not fail a whole index+ logger.warning("tombstone_absent_sweep_failed", error=str(exc))+# Record a completed GenerationJob so the web UI can show# "last synced" / "last re-indexed" timestamps.now = datetime.now(UTC)diff --git a/packages/core/src/repowise/core/pipeline/__init__.py b/packages/core/src/repowise/core/pipeline/__init__.pyindex 50e753f60..9a35e332d 100644--- a/packages/core/src/repowise/core/pipeline/__init__.py+++ b/packages/core/src/repowise/core/pipeline/__init__.py@@ -20,6 +20,7 @@ from .persist import (persist_git,persist_ingestion,persist_pipeline_result,+ tombstone_absent_file_pages,)from .phase_timing import PhaseTimingRecorderfrom .progress import LoggingProgressCallback, ProgressCallback@@ -41,4 +42,5 @@ __all__ = ["run_generation","run_pipeline","sweep_stale_generated_pages",+ "tombstone_absent_file_pages",]diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex 2f7ed14f5..fbc447f28 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -8,7 +8,9 @@ from __future__ import annotationsimport dataclassesimport json+import loggingfrom collections.abc import Iterable+from pathlib import Pathfrom typing import Anyimport structlog@@ -20,6 +22,14 @@ from repowise.core.generation.models import (logger = structlog.get_logger(__name__)+# A second, stdlib logger for the one message here that has to survive+# ``repowise init``. ``configure_cli_logging`` pins both ``repowise.core`` and+# structlog to ERROR for the progress-bar commands, so anything below ERROR is+# discarded in the exact command that writes the index. The refusal below is+# logged through this at ERROR for that reason; the routine counts stay on+# structlog with their siblings.+_log = logging.getLogger(__name__)+# Max page ids per UPDATE ... IN (...) so a large cascade cannot exceed# SQLite's bound-variable limit on the local CLI store._STALE_ID_CHUNK = 500@@ -96,6 +106,89 @@ async def mark_tombstone_pages(return marked+async def tombstone_absent_file_pages(+ session: Any, repo_id: str, repo_path: Path | str+) -> list[str]:+ """Tombstone every file page whose file is no longer on disk.++ :func:`mark_tombstone_pages` works from a diff, so it only ever sees a+ file that was deleted or renamed *between two commits this run compared*.+ ``repowise init`` compares nothing — it indexes a checkout as it stands —+ so it has never tombstoned anything, and a page written before a file was+ deleted keeps ``freshness_status='fresh'`` through every later index.++ That page is the trap the tombstone exists to close: retrieval serves it,+ agents cite it, and the index-age metadata says nothing is wrong.++ The check is a plain existence test against the checkout rather than a+ comparison with what this run parsed. A file can be present but unparsed —+ an unsupported extension, a parse failure, a changed exclude list — and+ such a file is stale, not deleted. Calling it deleted would put a+ ``successor_paths: []`` on a page whose file a reader can still open.++ Returns the page ids marked, so the caller can drop them from the+ full-text index once its session has closed, on the same terms as+ :func:`mark_tombstone_pages`.+ """+ from sqlalchemy import select++ from repowise.core.persistence.models import Page++ root = Path(repo_path)+ res = await session.execute(+ select(Page).where(+ Page.repository_id == repo_id,+ Page.page_type == "file_page",+ Page.freshness_status != "tombstone",+ )+ )+ live = list(res.scalars().all())+ if not live:+ return []++ absent = [p for p in live if not (root / (p.target_path or "")).exists()]+ if not absent:+ return []++ # Every page absent means the paths are not being resolved against the+ # tree they were written from — a wrong root, a bare-repo checkout, a+ # working copy that has not been restored yet. Tombstoning the whole wiki+ # on that reading is worse than leaving it, and unlike a partial mistake+ # it is not recoverable by re-indexing a file. Refuse, loudly, and let the+ # rest of the run proceed.+ if len(absent) == len(live):+ _log.error(+ "tombstone_sweep_refused repo_id=%s file_pages=%d root=%s: every file "+ "page's path is missing from the checkout, which reads as a wrong "+ "root rather than a deleted repository. Nothing was tombstoned.",+ repo_id,+ len(live),+ root,+ )+ return []++ marked: list[str] = []+ for page in absent:+ page.freshness_status = "tombstone"+ try:+ meta = json.loads(page.metadata_json or "{}")+ except (json.JSONDecodeError, TypeError):+ meta = {}+ # Deleted, not renamed: rename detection is the diff-driven sweep's+ # job and it has evidence this one does not.+ meta["successor_paths"] = []+ page.metadata_json = json.dumps(meta)+ marked.append(page.id)++ logger.info(+ "file_pages_tombstoned_absent",+ repo_id=repo_id,+ count=len(marked),+ sample=[p.target_path for p in absent[:3]],+ )+ return marked++async def mark_stale_pages(session: Any, repo_id: str, paths: list[str]) -> int:"""Decay weakly-affected file pages to ``freshness_status='stale'``.@@ -678,9 +771,7 @@ async def sweep_superseded_generated_pages(produced_ids.setdefault(page_type, set()).add(page.page_id)metadata = getattr(page, "metadata", None) or {}members = metadata.get("file_paths") or metadata.get("files") or []- covered.setdefault(page_type, set()).update(- m for m in members if isinstance(m, str)- )+ covered.setdefault(page_type, set()).update(m for m in members if isinstance(m, str))swept: list[str] = []for page_type, current in produced_ids.items():@@ -722,9 +813,7 @@ async def sweep_superseded_generated_pages(swept.extend(stale)if swept:- logger.info(- "superseded_generated_pages_swept", repo_id=repo_id, count=len(swept)- )+ logger.info("superseded_generated_pages_swept", repo_id=repo_id, count=len(swept))return sweptdiff --git a/tests/unit/cli/test_persist_result_tombstones_absent.py b/tests/unit/cli/test_persist_result_tombstones_absent.pynew file mode 100644index 000000000..fb4d5a733--- /dev/null+++ b/tests/unit/cli/test_persist_result_tombstones_absent.py@@ -0,0 +1,136 @@+"""``repowise init`` tombstones a page whose file is gone.++The helper doing the work is unit-tested next door. This drives the CLI+persistence path instead, because the helper existing is not the fix — being+called from both of ``persist_result``'s branches is. The normal single-repo+init persists the index phase during the run and so takes the+``index_persisted_incrementally=True`` branch, which never reached the+diff-driven tombstone path at all.+"""++from __future__ import annotations++from datetime import UTC, datetime+from types import SimpleNamespace++from sqlalchemy import select++from repowise.cli._repo_session import open_repo_db+from repowise.cli.commands.init_cmd.persistence import persist_result+from repowise.core.generation.models import GeneratedPage+from repowise.core.persistence import FullTextSearch, get_session+from repowise.core.persistence.models import Page+++def _generated_page(page_type: str, target: str) -> GeneratedPage:+ now = datetime.now(UTC).isoformat()+ return GeneratedPage(+ page_id=f"{page_type}:{target}",+ page_type=page_type,+ title=target,+ content=f"content for {target}",+ source_hash="x" * 64,+ model_name="mock",+ provider_name="mock",+ input_tokens=1,+ output_tokens=1,+ cached_tokens=0,+ generation_level=1,+ target_path=target,+ created_at=now,+ updated_at=now,+ )+++def _result(repo_name: str, generated_pages: list[GeneratedPage]) -> SimpleNamespace:+ return SimpleNamespace(+ repo_name=repo_name,+ index_persisted_incrementally=True,+ generated_pages=generated_pages,+ tech_stack=None,+ vector_store=None,+ dead_code_report=None,+ health_report=None,+ decision_report=None,+ git_metadata_list=[],+ knowledge_graph_result=None,+ authoritative_page_types=set(),+ preserved_page_ids=set(),+ )+++async def _seed_file_pages(repo_path, *paths: str) -> str:+ """A fresh file page per path, seeded as a previous run would leave it."""+ engine, sf, repo_id = await open_repo_db(repo_path, repo_name="r")+ try:+ now = datetime.now(UTC)+ async with get_session(sf) as session:+ for path in paths:+ session.add(+ Page(+ id=f"file_page:{path}",+ repository_id=repo_id,+ page_type="file_page",+ title=f"File: {path}",+ content=f"# Overview\n\n{path} handles xylophone tuning.",+ target_path=path,+ source_hash="x" * 64,+ model_name="mock",+ provider_name="mock",+ created_at=now,+ updated_at=now,+ )+ )+ finally:+ await engine.dispose()+ return repo_id+++async def _status(repo_path, page_id: str) -> str:+ engine, sf, _ = await open_repo_db(repo_path, repo_name="r")+ try:+ async with get_session(sf) as session:+ res = await session.execute(select(Page).where(Page.id == page_id))+ return res.scalar_one().freshness_status+ finally:+ await engine.dispose()+++async def test_init_tombstones_the_page_of_a_deleted_file(tmp_path):+ """No diff anywhere in this run — just a page whose file is not there."""+ repo_path = tmp_path / "repo"+ repo_path.mkdir()+ (repo_path / "kept.py").write_text("x", encoding="utf-8")+ await _seed_file_pages(repo_path, "kept.py", "gone.py")++ await persist_result(_result("r", [_generated_page("module_page", "community-75")]), repo_path)++ assert await _status(repo_path, "file_page:gone.py") == "tombstone"+ assert await _status(repo_path, "file_page:kept.py") == "fresh"+++async def test_the_tombstoned_page_leaves_the_full_text_index(tmp_path):+ """A tombstone still occupies one of the fixed number of rows retrieval+ fetches before any filter runs, so the row has to go, not just the flag."""+ repo_path = tmp_path / "repo"+ repo_path.mkdir()+ (repo_path / "kept.py").write_text("x", encoding="utf-8")+ await _seed_file_pages(repo_path, "kept.py", "gone.py")++ engine, _sf, _ = await open_repo_db(repo_path, repo_name="r")+ fts = FullTextSearch(engine)+ await fts.ensure_index()+ for path in ("kept.py", "gone.py"):+ await fts.index(f"file_page:{path}", path, f"{path} handles xylophone tuning.")+ await engine.dispose()++ await persist_result(_result("r", [_generated_page("module_page", "community-75")]), repo_path)++ engine, _sf, _ = await open_repo_db(repo_path, repo_name="r")+ fts = FullTextSearch(engine)+ await fts.ensure_index()+ hits = {r.page_id for r in await fts.search("xylophone", limit=10)}+ await engine.dispose()++ assert "file_page:gone.py" not in hits+ assert "file_page:kept.py" in hitsdiff --git a/tests/unit/pipeline/test_tombstone_absent_files.py b/tests/unit/pipeline/test_tombstone_absent_files.pynew file mode 100644index 000000000..b7a236c9f--- /dev/null+++ b/tests/unit/pipeline/test_tombstone_absent_files.py@@ -0,0 +1,172 @@+"""A page for a file that is simply gone becomes a tombstone.++The existing tombstone path reads a diff, so it only ever sees a file that+was deleted or renamed between two commits a run compared. ``repowise init``+compares nothing — it indexes a checkout as it stands — so it has never+tombstoned anything, and a page written before its file was deleted keeps+``freshness_status='fresh'`` through every later index.++That page is the trap the tombstone exists to close: retrieval serves it,+agents cite it, and the index-age metadata says nothing is wrong.+"""++from __future__ import annotations++import json+from pathlib import Path++import pytest+from sqlalchemy import select+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine+from sqlalchemy.pool import StaticPool++from repowise.core.persistence.crud import upsert_page, upsert_repository+from repowise.core.persistence.database import init_db+from repowise.core.persistence.models import Page+from repowise.core.pipeline.persist import tombstone_absent_file_pages+++async def engine():+ eng = create_async_engine(+ "sqlite+aiosqlite:///:memory:",+ connect_args={"check_same_thread": False},+ poolclass=StaticPool,+ )+ await init_db(eng)+ yield eng+ await eng.dispose()+++async def session(engine):+ factory = async_sessionmaker(engine, expire_on_commit=False)+ async with factory() as sess:+ yield sess+++async def _seed(session, repo_root: Path, *paths: str, page_type: str = "file_page") -> str:+ """One page per path. Only the paths written to disk actually exist."""+ repo = await upsert_repository(session, name="r", local_path=str(repo_root))+ await session.commit()+ for path in paths:+ await upsert_page(+ session,+ page_id=f"{page_type}:{path}",+ repository_id=repo.id,+ page_type=page_type,+ title=f"File: {path}",+ content=f"# Overview\n\nWhat {path} does.",+ summary=f"What {path} does.",+ target_path=path,+ source_hash="h",+ model_name="mock",+ provider_name="mock",+ )+ await session.commit()+ return repo.id+++def _write(repo_root: Path, *paths: str) -> None:+ for path in paths:+ target = repo_root / path+ target.parent.mkdir(parents=True, exist_ok=True)+ target.write_text("x", encoding="utf-8")+++async def _status(session, page_id: str) -> str:+ res = await session.execute(select(Page).where(Page.id == page_id))+ return res.scalar_one().freshness_status+++async def test_a_page_for_a_deleted_file_is_tombstoned(session, tmp_path: Path):+ """The whole point: no diff involved, just a file that is not there."""+ repo_id = await _seed(session, tmp_path, "kept.py", "gone.py")+ _write(tmp_path, "kept.py")++ marked = await tombstone_absent_file_pages(session, repo_id, tmp_path)++ assert marked == ["file_page:gone.py"]+ assert await _status(session, "file_page:gone.py") == "tombstone"+++async def test_a_page_for_a_file_still_on_disk_is_untouched(session, tmp_path: Path):+ repo_id = await _seed(session, tmp_path, "kept.py", "gone.py")+ _write(tmp_path, "kept.py")++ await tombstone_absent_file_pages(session, repo_id, tmp_path)++ assert await _status(session, "file_page:kept.py") == "fresh"+++async def test_the_tombstone_claims_no_successor(session, tmp_path: Path):+ """Deleted, not renamed.++ Rename detection is the diff-driven sweep's job and it has evidence this+ one does not. Guessing a successor here would redirect a reader to a file+ that has nothing to do with the one they asked for.+ """+ repo_id = await _seed(session, tmp_path, "kept.py", "gone.py")+ _write(tmp_path, "kept.py")++ await tombstone_absent_file_pages(session, repo_id, tmp_path)++ res = await session.execute(select(Page).where(Page.id == "file_page:gone.py"))+ assert json.loads(res.scalar_one().metadata_json)["successor_paths"] == []+++async def test_every_page_absent_is_refused_rather_than_obeyed(+ session, tmp_path: Path, caplog: pytest.LogCaptureFixture+):+ """A wrong root reads exactly like a repository that lost every file.++ One of those readings wipes the wiki, and unlike a partial mistake it is+ not recoverable by re-indexing a file, so the sweep declines to act on+ it — at ERROR, which is the only level that survives ``repowise init``.+ """+ repo_id = await _seed(session, tmp_path, "a.py", "b.py", "c.py")+ # Nothing written to disk: as far as the sweep can see, all three are gone.++ with caplog.at_level("ERROR"):+ marked = await tombstone_absent_file_pages(session, repo_id, tmp_path)++ assert marked == []+ assert await _status(session, "file_page:a.py") == "fresh"+ assert "tombstone_sweep_refused" in caplog.text+++async def test_only_file_pages_are_swept(session, tmp_path: Path):+ """A module page's target_path is a grouping, not a file that must exist.++ Several page types carry a target_path that never named a file — a+ clustering ordinal, a layer's curated id — and testing those against the+ filesystem would tombstone every one of them.+ """+ repo_id = await _seed(session, tmp_path, "community-7", page_type="module_page")+ await _seed(session, tmp_path, "kept.py", "gone.py")+ _write(tmp_path, "kept.py")++ marked = await tombstone_absent_file_pages(session, repo_id, tmp_path)++ assert marked == ["file_page:gone.py"]+ assert await _status(session, "module_page:community-7") == "fresh"+++async def test_an_already_tombstoned_page_is_not_marked_again(session, tmp_path: Path):+ """The returned ids drive a full-text delete, so re-reporting one every+ run means re-deleting a row that left the index the first time."""+ repo_id = await _seed(session, tmp_path, "kept.py", "gone.py")+ _write(tmp_path, "kept.py")++ first = await tombstone_absent_file_pages(session, repo_id, tmp_path)+ second = await tombstone_absent_file_pages(session, repo_id, tmp_path)++ assert first == ["file_page:gone.py"]+ assert second == []+++async def test_a_repository_with_no_file_pages_does_nothing(session, tmp_path: Path):+ repo = await upsert_repository(session, name="r", local_path=str(tmp_path))+ await session.commit()++ assert await tombstone_absent_file_pages(session, repo.id, tmp_path) == []