Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(update): prune deleted files on a docs-mode update too (#3412)
Raghav Chamadiya1c41d17cfa56
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.pyindex 3c6fd5376..4b339de7b 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py@@ -1311,6 +1311,23 @@ async def _persist_full_update_async(except Exception as exc:_skip("External systems refresh", exc)+ # Last of the store writes, for the reason the index-only path+ # gives: the steps above upsert rows for deleted paths too.+ from repowise.core.pipeline.incremental import prune_deleted_files++ await prune_deleted_files(+ session,+ repo_id,+ repo_path,+ graph_builder,+ {parsed.file_info.path for parsed in parsed_files or []},+ health_report=partial_health_report,+ skip=_skip,+ log=console.print,+ degraded=degraded,+ timings=timings,+ )+# Record a GenerationJob so the web UI "last synced" timestamp updates.try:from datetime import UTC as _UTC@@ -1399,7 +1416,11 @@ async def _persist_full_update_async(if timings is not None:timings.stop("persist.fts")- vector_cleanup_ids = load_cleanup_debt(Path(repo_path))["vectors"]+ # The union closes the mark_tombstone_pages path: the scope branches+ # above already drop their ids, but a deleted file's embedding did not.+ vector_cleanup_ids = (+ set(tombstoned_page_ids) | load_cleanup_debt(Path(repo_path))["vectors"]+ )if vector_cleanup_ids and decision_vector_store is not None:try:await decision_vector_store.delete_many(sorted(vector_cleanup_ids))diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.pyindex 0b8fb88c1..0ff898858 100644--- a/packages/core/src/repowise/core/pipeline/incremental.py+++ b/packages/core/src/repowise/core/pipeline/incremental.py@@ -1822,6 +1822,79 @@ async def refresh_external_systems(return True+async def prune_deleted_files(+ session: Any,+ repo_id: str,+ repo_path: Any,+ graph_builder: Any,+ live_paths: Iterable[str],+ *,+ health_report: Any,+ skip: Callable[[str, Exception], None],+ log: LogFn,+ degraded: list[str] | None,+ timings: PhaseTimings | None,+) -> None:+ """Drop file-scoped rows for deleted files, then rebuild the queues built from them.++ Shared by every update writer. Liveness is asked of disk and git, never of+ this run's parse (see prune_deleted_file_rows).++ Call it last in the session: the steps before it upsert, and the git step+ indexes deleted paths too, so an earlier prune would be undone.++ The graph's file nodes join *live_paths* because resolver passes mint+ file-shaped nodes that name no file (`external:`, `framework:`, Spring SPI+ sources); they fail every liveness test, and only changed files' edges are+ rebuilt afterwards. A graph that cannot be read skips the prune.+ """+ pruned = 0+ try:+ from repowise.core.pipeline.persist import prune_deleted_file_rows++ live_hint = set(live_paths) | {+ node+ for node, data in graph_builder.graph().nodes(data=True)+ if data.get("node_type", "file") == "file"+ }+ with timed(timings, "persist.prune"):+ pruned, refusals = await prune_deleted_file_rows(+ session, repo_id, repo_path, live_hint=live_hint+ )+ if pruned:+ log(f"Pruned rows for [cyan]{pruned}[/cyan] deleted file(s)")+ for refusal in refusals:+ log(f"[yellow]{refusal}[/yellow]")+ if degraded is not None:+ degraded.append(refusal)+ except Exception as exc:+ skip("Deleted-file prune", exc)++ # The performance and refactoring queues are materialized from the health+ # findings the prune just deleted; without this they keep serving a cause+ # whose evidence is gone, on a file the store no longer claims exists.+ if not pruned:+ return+ try:+ from repowise.core.persistence.crud import (+ finalize_performance_opportunities,+ finalize_refactoring_opportunities,+ )+ from repowise.core.pipeline.persist import _analyzed_commit++ analyzed_commit = await _analyzed_commit(session, repo_id)+ await finalize_performance_opportunities(+ session,+ repo_id,+ analyzed_commit=analyzed_commit,+ plan_policy=getattr(health_report, "performance_plan_policy", None),+ execution_roles=getattr(health_report, "execution_roles", None),+ )+ await finalize_refactoring_opportunities(session, repo_id, analyzed_commit=analyzed_commit)+ except Exception as exc:+ skip("Queue rebuild after prune", exc)++async def persist_incremental_index(repo_path: Any,graph_builder: Any,@@ -2174,90 +2247,25 @@ async def persist_incremental_index(except Exception as exc:_skip("Decision purge", exc)- pruned = 0- # Drop file-scoped rows for files that have actually been deleted.- # Without this an incremental update tombstones the deleted file's- # page and leaves everything else: graph nodes, edges, metrics,- # symbols, health rows and git metadata all keep serving a file- # that is gone. The liveness question is asked of the filesystem- # and git, never of this run's parse, so a transient read failure- # cannot masquerade as a deletion (see prune_deleted_file_rows).- #- # Last in the session on purpose. Everything above upserts rather- # than deleting, and the git step in particular indexes the changed- # *paths*, which includes the deleted ones: pruning first left a- # fresh git_metadata row for every file this run watched disappear.- # Running last means the prune has the final say on what the store- # claims exists.- try:- from repowise.core.pipeline.persist import prune_deleted_file_rows-- live_hint = set(+ # Last in the session on purpose: everything above upserts, and the+ # git step indexes the changed *paths*, deleted ones included, so+ # the prune has to have the final say on what the store claims.+ await prune_deleted_files(+ session,+ repo_id,+ repo_path,+ graph_builder,+ (current_graph_file_pathsif current_graph_file_paths is not Noneelse {pf.file_info.path for pf in parsed_files or []}- )- # Plus every file node the rebuilt graph still holds, which is- # the only reliable way to know a node names no file at all.- # Framework and resolver passes mint file-shaped nodes for- # things that were never on disk: `external:` imports,- # `framework:` anchors, and Spring's `META-INF/services/<iface>`- # SPI source, which carries no prefix to recognise it by. All- # of them fail every liveness test there is, so a prune that- # asked only about disk and git would delete them and take- # their edges with them, and only *changed* files' edges are- # rebuilt afterwards. A node this run's graph build still- # contains is live by construction, whatever it names.- #- # Deliberately not guarded: a graph this run cannot read is a- # run that has no business deciding what was deleted, and the- # outer handler skips the prune entirely.- graph = graph_builder.graph()- live_hint |= {- node- for node, data in graph.nodes(data=True)- if data.get("node_type", "file") == "file"- }- with timed(timings, "persist.prune"):- pruned, refusals = await prune_deleted_file_rows(- session, repo_id, repo_path, live_hint=live_hint- )- if pruned:- log(f"Pruned rows for [cyan]{pruned}[/cyan] deleted file(s)")- for refusal in refusals:- log(f"[yellow]{refusal}[/yellow]")- if degraded is not None:- degraded.append(refusal)- except Exception as exc:- _skip("Deleted-file prune", exc)-- # The prune deletes the health findings of files that are gone, and- # the performance and refactoring queues are materialized from those- # findings. Without this they keep serving a cause whose evidence was- # removed a moment ago, on a file the store no longer claims exists.- # After the prune for the same reason the prune runs last: only now- # is the surviving finding set final.- if pruned:- try:- from repowise.core.persistence.crud import (- finalize_performance_opportunities,- finalize_refactoring_opportunities,- )- from repowise.core.pipeline.persist import _analyzed_commit-- analyzed_commit = await _analyzed_commit(session, repo_id)- await finalize_performance_opportunities(- session,- repo_id,- analyzed_commit=analyzed_commit,- plan_policy=getattr(partial_health_report, "performance_plan_policy", None),- execution_roles=getattr(partial_health_report, "execution_roles", None),- )- await finalize_refactoring_opportunities(- session, repo_id, analyzed_commit=analyzed_commit- )- except Exception as exc:- _skip("Queue rebuild after prune", exc)+ ),+ health_report=partial_health_report,+ skip=_skip,+ log=log,+ degraded=degraded,+ timings=timings,+ )# After the session closes: on SQLite the full-text index shares the# database file, so writing to it while the session holds a write lockdiff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex 55dc13cd9..111f3b9bf 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -1262,9 +1262,7 @@ async def prune_deleted_file_rows(# documents it read, so a deleted one is never in scope and its rows would# outlive the file without this. ``_FileLiveness`` asks disk and# ``git ls-files`` rather than the parse, so a ``.md`` path is judged- # correctly here. Note the LLM-regenerating update path never reaches this- # function, so drift rows for a document deleted there wait for a reindex,- # as dead-code and health rows already do.+ # correctly here.await _prune_table(DocDriftFinding, DocDriftFinding.file_path, "doc_drift_findings")# On the document only. A reference to a deleted *target* is drift, not a# dead row: the next pass re-resolves it into a finding, and pruning bydiff --git a/tests/unit/cli/test_update_persist_reliability.py b/tests/unit/cli/test_update_persist_reliability.pyindex ba265b6ec..9556d7fb0 100644--- a/tests/unit/cli/test_update_persist_reliability.py+++ b/tests/unit/cli/test_update_persist_reliability.py@@ -694,6 +694,121 @@ def test_update_sweeps_pages_retired_since_the_index_was_built(tmp_path: Path) -assert survivor in fts_left+# ---------------------------------------------------------------------------+# A docs-mode update prunes a deleted file like the index-only one does+# ---------------------------------------------------------------------------+++class _GraphOnly:+ """Just enough builder for the prune: the rebuilt graph holds the kept file."""++ def __init__(self, *files: str) -> None:+ import networkx as nx++ self._graph = nx.DiGraph()+ self._graph.add_nodes_from(files, node_type="file")++ def graph(self):+ return self._graph+++class _RecordingVectors:+ def __init__(self) -> None:+ self.deleted: set[str] = set()++ async def delete_many(self, ids) -> None:+ self.deleted.update(ids)+++def test_full_update_leaves_no_rows_for_a_deleted_file(tmp_path: Path) -> None:+ from repowise.cli.helpers import get_db_url_for_repo+ from repowise.core.analysis.health.refactoring.payoff import FILE_DELETED+ from repowise.core.ingestion.change_detector import FileDiff+ from repowise.core.persistence import (+ create_engine,+ create_session_factory,+ get_session,+ upsert_repository,+ )+ from repowise.core.persistence.crud import (+ save_refactoring_suggestions,+ write_function_facts,+ )+ from repowise.core.persistence.models import (+ FunctionFact,+ GraphNode,+ RefactoringPayoff,+ RefactoringSuggestion,+ )+ from tests.unit.persistence.test_persist_prune import KEPT, STALE, _paths, _seed+ from tests.unit.persistence.test_prune_deleted_files import FILE_SCOPED+ from tests.unit.persistence.test_refactoring_payoff import _fact, _plan++ (tmp_path / ".repowise").mkdir()+ (tmp_path / Path(KEPT).parent).mkdir(parents=True)+ (tmp_path / KEPT).write_text("export const main = () => {};\n", encoding="utf-8")+ stale_page = f"file_page:{STALE}"+ asyncio.run(_seed_pages(tmp_path, "repo", [stale_page]))++ async def _with_session(work):+ engine = create_engine(get_db_url_for_repo(tmp_path))+ try:+ async with get_session(create_session_factory(engine)) as session:+ repo = await upsert_repository(session, name="repo", local_path=str(tmp_path))+ return await work(session, repo.id)+ finally:+ await engine.dispose()++ async def _seed_rows(session, repo_id: str) -> None:+ await _seed(session, repo_id)+ await save_refactoring_suggestions(session, repo_id, [_plan(KEPT), _plan(STALE)])+ await write_function_facts(+ session, repo_id, [_fact(f"{STALE}::run", 14, 50), _fact(f"{KEPT}::run", 3, 5)]+ )++ asyncio.run(_with_session(_seed_rows))++ vectors = _RecordingVectors()+ degraded: list[str] = []+ asyncio.run(+ _persist_full_update_async(+ repo_path=tmp_path,+ repo_name="repo",+ generated_pages=[_page(f"file_page:{KEPT}")],+ file_diffs=[FileDiff(STALE, "deleted", None, None, None, None)],+ git_meta_map={},+ new_decision_markers=[],+ decision_vector_store=vectors,+ provider=None,+ partial_health_report=None,+ dead_code_report=None,+ graph_builder=_GraphOnly(KEPT),+ knowledge_graph_result=None,+ degraded=degraded,+ parsed_files=[],+ )+ )++ async def _check(session, repo_id: str) -> None:+ node_ids = await _paths(session, GraphNode, GraphNode.node_id, repo_id)+ assert not {n for n in node_ids if n.startswith(STALE)}+ for model, column in FILE_SCOPED:+ assert await _paths(session, model, column, repo_id) == {KEPT}, model.__name__+ facts = set((await session.execute(select(FunctionFact.symbol_id))).scalars())+ assert facts == {f"{KEPT}::run"}+ plans = {+ r.file_path: r for r in (await session.execute(select(RefactoringSuggestion))).scalars()+ }+ assert plans[KEPT].status == "open"+ assert plans[STALE].status == "resolved"+ (payoff,) = (await session.execute(select(RefactoringPayoff))).scalars()+ assert (payoff.suggestion_id, payoff.outcome) == (plans[STALE].id, FILE_DELETED)++ asyncio.run(_with_session(_check))+ assert not any(entry.startswith("Deleted-file prune") for entry in degraded)+ assert stale_page in vectors.deleted++# ---------------------------------------------------------------------------# Lock contention: exactly one winner# ---------------------------------------------------------------------------diff --git a/tests/unit/persistence/test_prune_deleted_files.py b/tests/unit/persistence/test_prune_deleted_files.pyindex 63b811024..20d5d3282 100644--- a/tests/unit/persistence/test_prune_deleted_files.py+++ b/tests/unit/persistence/test_prune_deleted_files.py@@ -262,3 +262,63 @@ async def test_function_facts_of_a_deleted_file_are_pruned(async_session, repo_wassert await _paths(async_session, FunctionFact, FunctionFact.symbol_id, repo.id) == {f"{KEPT}::main"}+++class _Graph:+ def __init__(self, *files: str) -> None:+ import networkx as nx++ self._graph = nx.DiGraph()+ self._graph.add_nodes_from(files, node_type="file")++ def graph(self):+ return self._graph+++def _shared_prune(session, repo_id, repo, live_paths, degraded, graph=()):+ from repowise.core.pipeline.incremental import prune_deleted_files++ def _skip(step: str, exc: Exception) -> None:+ raise AssertionError(f"{step}: {exc}")++ return prune_deleted_files(+ session,+ repo_id,+ repo,+ _Graph(*graph),+ live_paths,+ health_report=None,+ skip=_skip,+ log=lambda _m: None,+ degraded=degraded,+ timings=None,+ )+++async def test_a_parsed_file_the_graph_lacks_is_not_pruned(async_session, repo_with_kept_file):+ """This run parsed STALE, so it is live even though disk, git and the graph+ all miss it: the parse half of the live set is not only an optimisation."""+ repo = await insert_repo(async_session)+ await _seed(async_session, repo.id)++ degraded: list[str] = []+ await _shared_prune(async_session, repo.id, repo_with_kept_file, {STALE}, degraded, (KEPT,))+ await async_session.commit()++ assert degraded == []+ for model, column in FILE_SCOPED:+ assert await _paths(async_session, model, column, repo.id) == {KEPT, STALE}, model+++async def test_a_refused_prune_lands_in_degraded(async_session, repo_with_kept_file):+ repo = await insert_repo(async_session)+ for i in range(40):+ async_session.add(GitMetadata(repository_id=repo.id, file_path=f"src/module_{i}.js"))+ await async_session.flush()++ degraded: list[str] = []+ await _shared_prune(async_session, repo.id, repo_with_kept_file, set(), degraded)+ await async_session.commit()++ assert len(degraded) == 1+ assert "git_metadata" in degraded[0]