Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(index): tombstone a page whose file is no longer on disk (#1202)

Swati Ahuja439bbd22d6a5
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
index a3d05d93f..db2851abe 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
@@ -116,6 +116,7 @@ async def persist_result(result: Any, repo_path: Path) -> None:
persist_generation,
persist_pipeline_result,
sweep_stale_generated_pages,
+ tombstone_absent_file_pages,
)
engine, sf, _repo_id = await open_repo_db(repo_path, repo_name=result.repo_name)
@@ -177,6 +178,20 @@ async def persist_result(result: Any, repo_path: Path) -> None:
else:
swept_page_ids = await persist_pipeline_result(result, session, repo.id)
+ # Tombstone pages whose file is simply gone. The other tombstone path
+ # reads a diff, and an init compares nothing, so until here a page
+ # written before its file was deleted stayed ``fresh`` through every
+ # later index. Outside the branch above because both branches need it.
+ # Its ids join the swept ones so they leave the vector store and the
+ # full-text index together, below, after this session closes.
+ try:
+ swept_page_ids = [
+ *swept_page_ids,
+ *await tombstone_absent_file_pages(session, repo.id, repo_path),
+ ]
+ except Exception as exc: # one stale page must not fail a whole index
+ logger.warning("tombstone_absent_sweep_failed", error=str(exc))
+
# Record a completed GenerationJob so the web UI can show
# "last synced" / "last re-indexed" timestamps.
now = datetime.now(UTC)
diff --git a/packages/core/src/repowise/core/pipeline/__init__.py b/packages/core/src/repowise/core/pipeline/__init__.py
index 50e753f60..9a35e332d 100644
--- a/packages/core/src/repowise/core/pipeline/__init__.py
+++ b/packages/core/src/repowise/core/pipeline/__init__.py
@@ -20,6 +20,7 @@ from .persist import (
persist_git,
persist_ingestion,
persist_pipeline_result,
+ tombstone_absent_file_pages,
)
from .phase_timing import PhaseTimingRecorder
from .progress import LoggingProgressCallback, ProgressCallback
@@ -41,4 +42,5 @@ __all__ = [
"run_generation",
"run_pipeline",
"sweep_stale_generated_pages",
+ "tombstone_absent_file_pages",
]
diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.py
index 2f7ed14f5..fbc447f28 100644
--- a/packages/core/src/repowise/core/pipeline/persist.py
+++ b/packages/core/src/repowise/core/pipeline/persist.py
@@ -8,7 +8,9 @@ from __future__ import annotations
import dataclasses
import json
+import logging
from collections.abc import Iterable
+from pathlib import Path
from typing import Any
import structlog
@@ -20,6 +22,14 @@ from repowise.core.generation.models import (
logger = structlog.get_logger(__name__)
+# A second, stdlib logger for the one message here that has to survive
+# ``repowise init``. ``configure_cli_logging`` pins both ``repowise.core`` and
+# structlog to ERROR for the progress-bar commands, so anything below ERROR is
+# discarded in the exact command that writes the index. The refusal below is
+# logged through this at ERROR for that reason; the routine counts stay on
+# structlog with their siblings.
+_log = logging.getLogger(__name__)
+
# Max page ids per UPDATE ... IN (...) so a large cascade cannot exceed
# SQLite's bound-variable limit on the local CLI store.
_STALE_ID_CHUNK = 500
@@ -96,6 +106,89 @@ async def mark_tombstone_pages(
return marked
+async def tombstone_absent_file_pages(
+ session: Any, repo_id: str, repo_path: Path | str
+) -> list[str]:
+ """Tombstone every file page whose file is no longer on disk.
+
+ :func:`mark_tombstone_pages` works from a diff, so it only ever sees a
+ file that was deleted or renamed *between two commits this run compared*.
+ ``repowise init`` compares nothing — it indexes a checkout as it stands —
+ so it has never tombstoned anything, and a page written before a file was
+ deleted keeps ``freshness_status='fresh'`` through every later index.
+
+ That page is the trap the tombstone exists to close: retrieval serves it,
+ agents cite it, and the index-age metadata says nothing is wrong.
+
+ The check is a plain existence test against the checkout rather than a
+ comparison with what this run parsed. A file can be present but unparsed —
+ an unsupported extension, a parse failure, a changed exclude list — and
+ such a file is stale, not deleted. Calling it deleted would put a
+ ``successor_paths: []`` on a page whose file a reader can still open.
+
+ Returns the page ids marked, so the caller can drop them from the
+ full-text index once its session has closed, on the same terms as
+ :func:`mark_tombstone_pages`.
+ """
+ from sqlalchemy import select
+
+ from repowise.core.persistence.models import Page
+
+ root = Path(repo_path)
+ res = await session.execute(
+ select(Page).where(
+ Page.repository_id == repo_id,
+ Page.page_type == "file_page",
+ Page.freshness_status != "tombstone",
+ )
+ )
+ live = list(res.scalars().all())
+ if not live:
+ return []
+
+ absent = [p for p in live if not (root / (p.target_path or "")).exists()]
+ if not absent:
+ return []
+
+ # Every page absent means the paths are not being resolved against the
+ # tree they were written from — a wrong root, a bare-repo checkout, a
+ # working copy that has not been restored yet. Tombstoning the whole wiki
+ # on that reading is worse than leaving it, and unlike a partial mistake
+ # it is not recoverable by re-indexing a file. Refuse, loudly, and let the
+ # rest of the run proceed.
+ if len(absent) == len(live):
+ _log.error(
+ "tombstone_sweep_refused repo_id=%s file_pages=%d root=%s: every file "
+ "page's path is missing from the checkout, which reads as a wrong "
+ "root rather than a deleted repository. Nothing was tombstoned.",
+ repo_id,
+ len(live),
+ root,
+ )
+ return []
+
+ marked: list[str] = []
+ for page in absent:
+ page.freshness_status = "tombstone"
+ try:
+ meta = json.loads(page.metadata_json or "{}")
+ except (json.JSONDecodeError, TypeError):
+ meta = {}
+ # Deleted, not renamed: rename detection is the diff-driven sweep's
+ # job and it has evidence this one does not.
+ meta["successor_paths"] = []
+ page.metadata_json = json.dumps(meta)
+ marked.append(page.id)
+
+ logger.info(
+ "file_pages_tombstoned_absent",
+ repo_id=repo_id,
+ count=len(marked),
+ sample=[p.target_path for p in absent[:3]],
+ )
+ return marked
+
+
async def mark_stale_pages(session: Any, repo_id: str, paths: list[str]) -> int:
"""Decay weakly-affected file pages to ``freshness_status='stale'``.
@@ -678,9 +771,7 @@ async def sweep_superseded_generated_pages(
produced_ids.setdefault(page_type, set()).add(page.page_id)
metadata = getattr(page, "metadata", None) or {}
members = metadata.get("file_paths") or metadata.get("files") or []
- covered.setdefault(page_type, set()).update(
- m for m in members if isinstance(m, str)
- )
+ covered.setdefault(page_type, set()).update(m for m in members if isinstance(m, str))
swept: list[str] = []
for page_type, current in produced_ids.items():
@@ -722,9 +813,7 @@ async def sweep_superseded_generated_pages(
swept.extend(stale)
if swept:
- logger.info(
- "superseded_generated_pages_swept", repo_id=repo_id, count=len(swept)
- )
+ logger.info("superseded_generated_pages_swept", repo_id=repo_id, count=len(swept))
return swept
diff --git a/tests/unit/cli/test_persist_result_tombstones_absent.py b/tests/unit/cli/test_persist_result_tombstones_absent.py
new file mode 100644
index 000000000..fb4d5a733
--- /dev/null
+++ b/tests/unit/cli/test_persist_result_tombstones_absent.py
@@ -0,0 +1,136 @@
+"""``repowise init`` tombstones a page whose file is gone.
+
+The helper doing the work is unit-tested next door. This drives the CLI
+persistence path instead, because the helper existing is not the fix — being
+called from both of ``persist_result``'s branches is. The normal single-repo
+init persists the index phase during the run and so takes the
+``index_persisted_incrementally=True`` branch, which never reached the
+diff-driven tombstone path at all.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from types import SimpleNamespace
+
+from sqlalchemy import select
+
+from repowise.cli._repo_session import open_repo_db
+from repowise.cli.commands.init_cmd.persistence import persist_result
+from repowise.core.generation.models import GeneratedPage
+from repowise.core.persistence import FullTextSearch, get_session
+from repowise.core.persistence.models import Page
+
+
+def _generated_page(page_type: str, target: str) -> GeneratedPage:
+ now = datetime.now(UTC).isoformat()
+ return GeneratedPage(
+ page_id=f"{page_type}:{target}",
+ page_type=page_type,
+ title=target,
+ content=f"content for {target}",
+ source_hash="x" * 64,
+ model_name="mock",
+ provider_name="mock",
+ input_tokens=1,
+ output_tokens=1,
+ cached_tokens=0,
+ generation_level=1,
+ target_path=target,
+ created_at=now,
+ updated_at=now,
+ )
+
+
+def _result(repo_name: str, generated_pages: list[GeneratedPage]) -> SimpleNamespace:
+ return SimpleNamespace(
+ repo_name=repo_name,
+ index_persisted_incrementally=True,
+ generated_pages=generated_pages,
+ tech_stack=None,
+ vector_store=None,
+ dead_code_report=None,
+ health_report=None,
+ decision_report=None,
+ git_metadata_list=[],
+ knowledge_graph_result=None,
+ authoritative_page_types=set(),
+ preserved_page_ids=set(),
+ )
+
+
+async def _seed_file_pages(repo_path, *paths: str) -> str:
+ """A fresh file page per path, seeded as a previous run would leave it."""
+ engine, sf, repo_id = await open_repo_db(repo_path, repo_name="r")
+ try:
+ now = datetime.now(UTC)
+ async with get_session(sf) as session:
+ for path in paths:
+ session.add(
+ Page(
+ id=f"file_page:{path}",
+ repository_id=repo_id,
+ page_type="file_page",
+ title=f"File: {path}",
+ content=f"# Overview\n\n{path} handles xylophone tuning.",
+ target_path=path,
+ source_hash="x" * 64,
+ model_name="mock",
+ provider_name="mock",
+ created_at=now,
+ updated_at=now,
+ )
+ )
+ finally:
+ await engine.dispose()
+ return repo_id
+
+
+async def _status(repo_path, page_id: str) -> str:
+ engine, sf, _ = await open_repo_db(repo_path, repo_name="r")
+ try:
+ async with get_session(sf) as session:
+ res = await session.execute(select(Page).where(Page.id == page_id))
+ return res.scalar_one().freshness_status
+ finally:
+ await engine.dispose()
+
+
+async def test_init_tombstones_the_page_of_a_deleted_file(tmp_path):
+ """No diff anywhere in this run — just a page whose file is not there."""
+ repo_path = tmp_path / "repo"
+ repo_path.mkdir()
+ (repo_path / "kept.py").write_text("x", encoding="utf-8")
+ await _seed_file_pages(repo_path, "kept.py", "gone.py")
+
+ await persist_result(_result("r", [_generated_page("module_page", "community-75")]), repo_path)
+
+ assert await _status(repo_path, "file_page:gone.py") == "tombstone"
+ assert await _status(repo_path, "file_page:kept.py") == "fresh"
+
+
+async def test_the_tombstoned_page_leaves_the_full_text_index(tmp_path):
+ """A tombstone still occupies one of the fixed number of rows retrieval
+ fetches before any filter runs, so the row has to go, not just the flag."""
+ repo_path = tmp_path / "repo"
+ repo_path.mkdir()
+ (repo_path / "kept.py").write_text("x", encoding="utf-8")
+ await _seed_file_pages(repo_path, "kept.py", "gone.py")
+
+ engine, _sf, _ = await open_repo_db(repo_path, repo_name="r")
+ fts = FullTextSearch(engine)
+ await fts.ensure_index()
+ for path in ("kept.py", "gone.py"):
+ await fts.index(f"file_page:{path}", path, f"{path} handles xylophone tuning.")
+ await engine.dispose()
+
+ await persist_result(_result("r", [_generated_page("module_page", "community-75")]), repo_path)
+
+ engine, _sf, _ = await open_repo_db(repo_path, repo_name="r")
+ fts = FullTextSearch(engine)
+ await fts.ensure_index()
+ hits = {r.page_id for r in await fts.search("xylophone", limit=10)}
+ await engine.dispose()
+
+ assert "file_page:gone.py" not in hits
+ assert "file_page:kept.py" in hits
diff --git a/tests/unit/pipeline/test_tombstone_absent_files.py b/tests/unit/pipeline/test_tombstone_absent_files.py
new file mode 100644
index 000000000..b7a236c9f
--- /dev/null
+++ b/tests/unit/pipeline/test_tombstone_absent_files.py
@@ -0,0 +1,172 @@
+"""A page for a file that is simply gone becomes a tombstone.
+
+The existing tombstone path reads a diff, so it only ever sees a file that
+was deleted or renamed between two commits a run compared. ``repowise init``
+compares nothing — it indexes a checkout as it stands — so it has never
+tombstoned anything, and a page written before its file was deleted keeps
+``freshness_status='fresh'`` through every later index.
+
+That page is the trap the tombstone exists to close: retrieval serves it,
+agents cite it, and the index-age metadata says nothing is wrong.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+from sqlalchemy.pool import StaticPool
+
+from repowise.core.persistence.crud import upsert_page, upsert_repository
+from repowise.core.persistence.database import init_db
+from repowise.core.persistence.models import Page
+from repowise.core.pipeline.persist import tombstone_absent_file_pages
+
+
+async def engine():
+ eng = create_async_engine(
+ "sqlite+aiosqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ await init_db(eng)
+ yield eng
+ await eng.dispose()
+
+
+async def session(engine):
+ factory = async_sessionmaker(engine, expire_on_commit=False)
+ async with factory() as sess:
+ yield sess
+
+
+async def _seed(session, repo_root: Path, *paths: str, page_type: str = "file_page") -> str:
+ """One page per path. Only the paths written to disk actually exist."""
+ repo = await upsert_repository(session, name="r", local_path=str(repo_root))
+ await session.commit()
+ for path in paths:
+ await upsert_page(
+ session,
+ page_id=f"{page_type}:{path}",
+ repository_id=repo.id,
+ page_type=page_type,
+ title=f"File: {path}",
+ content=f"# Overview\n\nWhat {path} does.",
+ summary=f"What {path} does.",
+ target_path=path,
+ source_hash="h",
+ model_name="mock",
+ provider_name="mock",
+ )
+ await session.commit()
+ return repo.id
+
+
+def _write(repo_root: Path, *paths: str) -> None:
+ for path in paths:
+ target = repo_root / path
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text("x", encoding="utf-8")
+
+
+async def _status(session, page_id: str) -> str:
+ res = await session.execute(select(Page).where(Page.id == page_id))
+ return res.scalar_one().freshness_status
+
+
+async def test_a_page_for_a_deleted_file_is_tombstoned(session, tmp_path: Path):
+ """The whole point: no diff involved, just a file that is not there."""
+ repo_id = await _seed(session, tmp_path, "kept.py", "gone.py")
+ _write(tmp_path, "kept.py")
+
+ marked = await tombstone_absent_file_pages(session, repo_id, tmp_path)
+
+ assert marked == ["file_page:gone.py"]
+ assert await _status(session, "file_page:gone.py") == "tombstone"
+
+
+async def test_a_page_for_a_file_still_on_disk_is_untouched(session, tmp_path: Path):
+ repo_id = await _seed(session, tmp_path, "kept.py", "gone.py")
+ _write(tmp_path, "kept.py")
+
+ await tombstone_absent_file_pages(session, repo_id, tmp_path)
+
+ assert await _status(session, "file_page:kept.py") == "fresh"
+
+
+async def test_the_tombstone_claims_no_successor(session, tmp_path: Path):
+ """Deleted, not renamed.
+
+ Rename detection is the diff-driven sweep's job and it has evidence this
+ one does not. Guessing a successor here would redirect a reader to a file
+ that has nothing to do with the one they asked for.
+ """
+ repo_id = await _seed(session, tmp_path, "kept.py", "gone.py")
+ _write(tmp_path, "kept.py")
+
+ await tombstone_absent_file_pages(session, repo_id, tmp_path)
+
+ res = await session.execute(select(Page).where(Page.id == "file_page:gone.py"))
+ assert json.loads(res.scalar_one().metadata_json)["successor_paths"] == []
+
+
+async def test_every_page_absent_is_refused_rather_than_obeyed(
+ session, tmp_path: Path, caplog: pytest.LogCaptureFixture
+):
+ """A wrong root reads exactly like a repository that lost every file.
+
+ One of those readings wipes the wiki, and unlike a partial mistake it is
+ not recoverable by re-indexing a file, so the sweep declines to act on
+ it — at ERROR, which is the only level that survives ``repowise init``.
+ """
+ repo_id = await _seed(session, tmp_path, "a.py", "b.py", "c.py")
+ # Nothing written to disk: as far as the sweep can see, all three are gone.
+
+ with caplog.at_level("ERROR"):
+ marked = await tombstone_absent_file_pages(session, repo_id, tmp_path)
+
+ assert marked == []
+ assert await _status(session, "file_page:a.py") == "fresh"
+ assert "tombstone_sweep_refused" in caplog.text
+
+
+async def test_only_file_pages_are_swept(session, tmp_path: Path):
+ """A module page's target_path is a grouping, not a file that must exist.
+
+ Several page types carry a target_path that never named a file — a
+ clustering ordinal, a layer's curated id — and testing those against the
+ filesystem would tombstone every one of them.
+ """
+ repo_id = await _seed(session, tmp_path, "community-7", page_type="module_page")
+ await _seed(session, tmp_path, "kept.py", "gone.py")
+ _write(tmp_path, "kept.py")
+
+ marked = await tombstone_absent_file_pages(session, repo_id, tmp_path)
+
+ assert marked == ["file_page:gone.py"]
+ assert await _status(session, "module_page:community-7") == "fresh"
+
+
+async def test_an_already_tombstoned_page_is_not_marked_again(session, tmp_path: Path):
+ """The returned ids drive a full-text delete, so re-reporting one every
+ run means re-deleting a row that left the index the first time."""
+ repo_id = await _seed(session, tmp_path, "kept.py", "gone.py")
+ _write(tmp_path, "kept.py")
+
+ first = await tombstone_absent_file_pages(session, repo_id, tmp_path)
+ second = await tombstone_absent_file_pages(session, repo_id, tmp_path)
+
+ assert first == ["file_page:gone.py"]
+ assert second == []
+
+
+async def test_a_repository_with_no_file_pages_does_nothing(session, tmp_path: Path):
+ repo = await upsert_repository(session, name="r", local_path=str(tmp_path))
+ await session.commit()
+
+ assert await tombstone_absent_file_pages(session, repo.id, tmp_path) == []