Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(update): prune deleted files on a docs-mode update too (#3412)

Raghav Chamadiya1c41d17cfa56
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
index 3c6fd5376..4b339de7b 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
@@ -1311,6 +1311,23 @@ async def _persist_full_update_async(
except Exception as exc:
_skip("External systems refresh", exc)
+ # Last of the store writes, for the reason the index-only path
+ # gives: the steps above upsert rows for deleted paths too.
+ from repowise.core.pipeline.incremental import prune_deleted_files
+
+ await prune_deleted_files(
+ session,
+ repo_id,
+ repo_path,
+ graph_builder,
+ {parsed.file_info.path for parsed in parsed_files or []},
+ health_report=partial_health_report,
+ skip=_skip,
+ log=console.print,
+ degraded=degraded,
+ timings=timings,
+ )
+
# Record a GenerationJob so the web UI "last synced" timestamp updates.
try:
from datetime import UTC as _UTC
@@ -1399,7 +1416,11 @@ async def _persist_full_update_async(
if timings is not None:
timings.stop("persist.fts")
- vector_cleanup_ids = load_cleanup_debt(Path(repo_path))["vectors"]
+ # The union closes the mark_tombstone_pages path: the scope branches
+ # above already drop their ids, but a deleted file's embedding did not.
+ vector_cleanup_ids = (
+ set(tombstoned_page_ids) | load_cleanup_debt(Path(repo_path))["vectors"]
+ )
if vector_cleanup_ids and decision_vector_store is not None:
try:
await decision_vector_store.delete_many(sorted(vector_cleanup_ids))
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.py
index 0b8fb88c1..0ff898858 100644
--- a/packages/core/src/repowise/core/pipeline/incremental.py
+++ b/packages/core/src/repowise/core/pipeline/incremental.py
@@ -1822,6 +1822,79 @@ async def refresh_external_systems(
return True
+async def prune_deleted_files(
+ session: Any,
+ repo_id: str,
+ repo_path: Any,
+ graph_builder: Any,
+ live_paths: Iterable[str],
+ *,
+ health_report: Any,
+ skip: Callable[[str, Exception], None],
+ log: LogFn,
+ degraded: list[str] | None,
+ timings: PhaseTimings | None,
+) -> None:
+ """Drop file-scoped rows for deleted files, then rebuild the queues built from them.
+
+ Shared by every update writer. Liveness is asked of disk and git, never of
+ this run's parse (see prune_deleted_file_rows).
+
+ Call it last in the session: the steps before it upsert, and the git step
+ indexes deleted paths too, so an earlier prune would be undone.
+
+ The graph's file nodes join *live_paths* because resolver passes mint
+ file-shaped nodes that name no file (`external:`, `framework:`, Spring SPI
+ sources); they fail every liveness test, and only changed files' edges are
+ rebuilt afterwards. A graph that cannot be read skips the prune.
+ """
+ pruned = 0
+ try:
+ from repowise.core.pipeline.persist import prune_deleted_file_rows
+
+ live_hint = set(live_paths) | {
+ node
+ for node, data in graph_builder.graph().nodes(data=True)
+ if data.get("node_type", "file") == "file"
+ }
+ with timed(timings, "persist.prune"):
+ pruned, refusals = await prune_deleted_file_rows(
+ session, repo_id, repo_path, live_hint=live_hint
+ )
+ if pruned:
+ log(f"Pruned rows for [cyan]{pruned}[/cyan] deleted file(s)")
+ for refusal in refusals:
+ log(f"[yellow]{refusal}[/yellow]")
+ if degraded is not None:
+ degraded.append(refusal)
+ except Exception as exc:
+ skip("Deleted-file prune", exc)
+
+ # The performance and refactoring queues are materialized from the health
+ # findings the prune just deleted; without this they keep serving a cause
+ # whose evidence is gone, on a file the store no longer claims exists.
+ if not pruned:
+ return
+ try:
+ from repowise.core.persistence.crud import (
+ finalize_performance_opportunities,
+ finalize_refactoring_opportunities,
+ )
+ from repowise.core.pipeline.persist import _analyzed_commit
+
+ analyzed_commit = await _analyzed_commit(session, repo_id)
+ await finalize_performance_opportunities(
+ session,
+ repo_id,
+ analyzed_commit=analyzed_commit,
+ plan_policy=getattr(health_report, "performance_plan_policy", None),
+ execution_roles=getattr(health_report, "execution_roles", None),
+ )
+ await finalize_refactoring_opportunities(session, repo_id, analyzed_commit=analyzed_commit)
+ except Exception as exc:
+ skip("Queue rebuild after prune", exc)
+
+
async def persist_incremental_index(
repo_path: Any,
graph_builder: Any,
@@ -2174,90 +2247,25 @@ async def persist_incremental_index(
except Exception as exc:
_skip("Decision purge", exc)
- pruned = 0
- # Drop file-scoped rows for files that have actually been deleted.
- # Without this an incremental update tombstones the deleted file's
- # page and leaves everything else: graph nodes, edges, metrics,
- # symbols, health rows and git metadata all keep serving a file
- # that is gone. The liveness question is asked of the filesystem
- # and git, never of this run's parse, so a transient read failure
- # cannot masquerade as a deletion (see prune_deleted_file_rows).
- #
- # Last in the session on purpose. Everything above upserts rather
- # than deleting, and the git step in particular indexes the changed
- # *paths*, which includes the deleted ones: pruning first left a
- # fresh git_metadata row for every file this run watched disappear.
- # Running last means the prune has the final say on what the store
- # claims exists.
- try:
- from repowise.core.pipeline.persist import prune_deleted_file_rows
-
- live_hint = set(
+ # Last in the session on purpose: everything above upserts, and the
+ # git step indexes the changed *paths*, deleted ones included, so
+ # the prune has to have the final say on what the store claims.
+ await prune_deleted_files(
+ session,
+ repo_id,
+ repo_path,
+ graph_builder,
+ (
current_graph_file_paths
if current_graph_file_paths is not None
else {pf.file_info.path for pf in parsed_files or []}
- )
- # Plus every file node the rebuilt graph still holds, which is
- # the only reliable way to know a node names no file at all.
- # Framework and resolver passes mint file-shaped nodes for
- # things that were never on disk: `external:` imports,
- # `framework:` anchors, and Spring's `META-INF/services/<iface>`
- # SPI source, which carries no prefix to recognise it by. All
- # of them fail every liveness test there is, so a prune that
- # asked only about disk and git would delete them and take
- # their edges with them, and only *changed* files' edges are
- # rebuilt afterwards. A node this run's graph build still
- # contains is live by construction, whatever it names.
- #
- # Deliberately not guarded: a graph this run cannot read is a
- # run that has no business deciding what was deleted, and the
- # outer handler skips the prune entirely.
- graph = graph_builder.graph()
- live_hint |= {
- node
- for node, data in graph.nodes(data=True)
- if data.get("node_type", "file") == "file"
- }
- with timed(timings, "persist.prune"):
- pruned, refusals = await prune_deleted_file_rows(
- session, repo_id, repo_path, live_hint=live_hint
- )
- if pruned:
- log(f"Pruned rows for [cyan]{pruned}[/cyan] deleted file(s)")
- for refusal in refusals:
- log(f"[yellow]{refusal}[/yellow]")
- if degraded is not None:
- degraded.append(refusal)
- except Exception as exc:
- _skip("Deleted-file prune", exc)
-
- # The prune deletes the health findings of files that are gone, and
- # the performance and refactoring queues are materialized from those
- # findings. Without this they keep serving a cause whose evidence was
- # removed a moment ago, on a file the store no longer claims exists.
- # After the prune for the same reason the prune runs last: only now
- # is the surviving finding set final.
- if pruned:
- try:
- from repowise.core.persistence.crud import (
- finalize_performance_opportunities,
- finalize_refactoring_opportunities,
- )
- from repowise.core.pipeline.persist import _analyzed_commit
-
- analyzed_commit = await _analyzed_commit(session, repo_id)
- await finalize_performance_opportunities(
- session,
- repo_id,
- analyzed_commit=analyzed_commit,
- plan_policy=getattr(partial_health_report, "performance_plan_policy", None),
- execution_roles=getattr(partial_health_report, "execution_roles", None),
- )
- await finalize_refactoring_opportunities(
- session, repo_id, analyzed_commit=analyzed_commit
- )
- except Exception as exc:
- _skip("Queue rebuild after prune", exc)
+ ),
+ health_report=partial_health_report,
+ skip=_skip,
+ log=log,
+ degraded=degraded,
+ timings=timings,
+ )
# After the session closes: on SQLite the full-text index shares the
# database file, so writing to it while the session holds a write lock
diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.py
index 55dc13cd9..111f3b9bf 100644
--- a/packages/core/src/repowise/core/pipeline/persist.py
+++ b/packages/core/src/repowise/core/pipeline/persist.py
@@ -1262,9 +1262,7 @@ async def prune_deleted_file_rows(
# documents it read, so a deleted one is never in scope and its rows would
# outlive the file without this. ``_FileLiveness`` asks disk and
# ``git ls-files`` rather than the parse, so a ``.md`` path is judged
- # correctly here. Note the LLM-regenerating update path never reaches this
- # function, so drift rows for a document deleted there wait for a reindex,
- # as dead-code and health rows already do.
+ # correctly here.
await _prune_table(DocDriftFinding, DocDriftFinding.file_path, "doc_drift_findings")
# On the document only. A reference to a deleted *target* is drift, not a
# dead row: the next pass re-resolves it into a finding, and pruning by
diff --git a/tests/unit/cli/test_update_persist_reliability.py b/tests/unit/cli/test_update_persist_reliability.py
index ba265b6ec..9556d7fb0 100644
--- a/tests/unit/cli/test_update_persist_reliability.py
+++ b/tests/unit/cli/test_update_persist_reliability.py
@@ -694,6 +694,121 @@ def test_update_sweeps_pages_retired_since_the_index_was_built(tmp_path: Path) -
assert survivor in fts_left
+# ---------------------------------------------------------------------------
+# A docs-mode update prunes a deleted file like the index-only one does
+# ---------------------------------------------------------------------------
+
+
+class _GraphOnly:
+ """Just enough builder for the prune: the rebuilt graph holds the kept file."""
+
+ def __init__(self, *files: str) -> None:
+ import networkx as nx
+
+ self._graph = nx.DiGraph()
+ self._graph.add_nodes_from(files, node_type="file")
+
+ def graph(self):
+ return self._graph
+
+
+class _RecordingVectors:
+ def __init__(self) -> None:
+ self.deleted: set[str] = set()
+
+ async def delete_many(self, ids) -> None:
+ self.deleted.update(ids)
+
+
+def test_full_update_leaves_no_rows_for_a_deleted_file(tmp_path: Path) -> None:
+ from repowise.cli.helpers import get_db_url_for_repo
+ from repowise.core.analysis.health.refactoring.payoff import FILE_DELETED
+ from repowise.core.ingestion.change_detector import FileDiff
+ from repowise.core.persistence import (
+ create_engine,
+ create_session_factory,
+ get_session,
+ upsert_repository,
+ )
+ from repowise.core.persistence.crud import (
+ save_refactoring_suggestions,
+ write_function_facts,
+ )
+ from repowise.core.persistence.models import (
+ FunctionFact,
+ GraphNode,
+ RefactoringPayoff,
+ RefactoringSuggestion,
+ )
+ from tests.unit.persistence.test_persist_prune import KEPT, STALE, _paths, _seed
+ from tests.unit.persistence.test_prune_deleted_files import FILE_SCOPED
+ from tests.unit.persistence.test_refactoring_payoff import _fact, _plan
+
+ (tmp_path / ".repowise").mkdir()
+ (tmp_path / Path(KEPT).parent).mkdir(parents=True)
+ (tmp_path / KEPT).write_text("export const main = () => {};\n", encoding="utf-8")
+ stale_page = f"file_page:{STALE}"
+ asyncio.run(_seed_pages(tmp_path, "repo", [stale_page]))
+
+ async def _with_session(work):
+ engine = create_engine(get_db_url_for_repo(tmp_path))
+ try:
+ async with get_session(create_session_factory(engine)) as session:
+ repo = await upsert_repository(session, name="repo", local_path=str(tmp_path))
+ return await work(session, repo.id)
+ finally:
+ await engine.dispose()
+
+ async def _seed_rows(session, repo_id: str) -> None:
+ await _seed(session, repo_id)
+ await save_refactoring_suggestions(session, repo_id, [_plan(KEPT), _plan(STALE)])
+ await write_function_facts(
+ session, repo_id, [_fact(f"{STALE}::run", 14, 50), _fact(f"{KEPT}::run", 3, 5)]
+ )
+
+ asyncio.run(_with_session(_seed_rows))
+
+ vectors = _RecordingVectors()
+ degraded: list[str] = []
+ asyncio.run(
+ _persist_full_update_async(
+ repo_path=tmp_path,
+ repo_name="repo",
+ generated_pages=[_page(f"file_page:{KEPT}")],
+ file_diffs=[FileDiff(STALE, "deleted", None, None, None, None)],
+ git_meta_map={},
+ new_decision_markers=[],
+ decision_vector_store=vectors,
+ provider=None,
+ partial_health_report=None,
+ dead_code_report=None,
+ graph_builder=_GraphOnly(KEPT),
+ knowledge_graph_result=None,
+ degraded=degraded,
+ parsed_files=[],
+ )
+ )
+
+ async def _check(session, repo_id: str) -> None:
+ node_ids = await _paths(session, GraphNode, GraphNode.node_id, repo_id)
+ assert not {n for n in node_ids if n.startswith(STALE)}
+ for model, column in FILE_SCOPED:
+ assert await _paths(session, model, column, repo_id) == {KEPT}, model.__name__
+ facts = set((await session.execute(select(FunctionFact.symbol_id))).scalars())
+ assert facts == {f"{KEPT}::run"}
+ plans = {
+ r.file_path: r for r in (await session.execute(select(RefactoringSuggestion))).scalars()
+ }
+ assert plans[KEPT].status == "open"
+ assert plans[STALE].status == "resolved"
+ (payoff,) = (await session.execute(select(RefactoringPayoff))).scalars()
+ assert (payoff.suggestion_id, payoff.outcome) == (plans[STALE].id, FILE_DELETED)
+
+ asyncio.run(_with_session(_check))
+ assert not any(entry.startswith("Deleted-file prune") for entry in degraded)
+ assert stale_page in vectors.deleted
+
+
# ---------------------------------------------------------------------------
# Lock contention: exactly one winner
# ---------------------------------------------------------------------------
diff --git a/tests/unit/persistence/test_prune_deleted_files.py b/tests/unit/persistence/test_prune_deleted_files.py
index 63b811024..20d5d3282 100644
--- a/tests/unit/persistence/test_prune_deleted_files.py
+++ b/tests/unit/persistence/test_prune_deleted_files.py
@@ -262,3 +262,63 @@ async def test_function_facts_of_a_deleted_file_are_pruned(async_session, repo_w
assert await _paths(async_session, FunctionFact, FunctionFact.symbol_id, repo.id) == {
f"{KEPT}::main"
}
+
+
+class _Graph:
+ def __init__(self, *files: str) -> None:
+ import networkx as nx
+
+ self._graph = nx.DiGraph()
+ self._graph.add_nodes_from(files, node_type="file")
+
+ def graph(self):
+ return self._graph
+
+
+def _shared_prune(session, repo_id, repo, live_paths, degraded, graph=()):
+ from repowise.core.pipeline.incremental import prune_deleted_files
+
+ def _skip(step: str, exc: Exception) -> None:
+ raise AssertionError(f"{step}: {exc}")
+
+ return prune_deleted_files(
+ session,
+ repo_id,
+ repo,
+ _Graph(*graph),
+ live_paths,
+ health_report=None,
+ skip=_skip,
+ log=lambda _m: None,
+ degraded=degraded,
+ timings=None,
+ )
+
+
+async def test_a_parsed_file_the_graph_lacks_is_not_pruned(async_session, repo_with_kept_file):
+ """This run parsed STALE, so it is live even though disk, git and the graph
+ all miss it: the parse half of the live set is not only an optimisation."""
+ repo = await insert_repo(async_session)
+ await _seed(async_session, repo.id)
+
+ degraded: list[str] = []
+ await _shared_prune(async_session, repo.id, repo_with_kept_file, {STALE}, degraded, (KEPT,))
+ await async_session.commit()
+
+ assert degraded == []
+ for model, column in FILE_SCOPED:
+ assert await _paths(async_session, model, column, repo.id) == {KEPT, STALE}, model
+
+
+async def test_a_refused_prune_lands_in_degraded(async_session, repo_with_kept_file):
+ repo = await insert_repo(async_session)
+ for i in range(40):
+ async_session.add(GitMetadata(repository_id=repo.id, file_path=f"src/module_{i}.js"))
+ await async_session.flush()
+
+ degraded: list[str] = []
+ await _shared_prune(async_session, repo.id, repo_with_kept_file, set(), degraded)
+ await async_session.commit()
+
+ assert len(degraded) == 1
+ assert "git_metadata" in degraded[0]