Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(persist): let a parser change reach edges on unchanged files (#1619)
Raghav Chamadiya0a0390bd42eb
diff --git a/packages/core/alembic/versions/0052_graph_edges_parser_fingerprint.py b/packages/core/alembic/versions/0052_graph_edges_parser_fingerprint.pynew file mode 100644index 000000000..225264018--- /dev/null+++ b/packages/core/alembic/versions/0052_graph_edges_parser_fingerprint.py@@ -0,0 +1,48 @@+"""Let a parser change reach edges on files git considers unchanged.++``persist_incremental_edges`` rewrites only the git-changed files' rows, so a+tree-sitter query or extractor change reached a file only when that file next+happened to change — every other file kept the old build's edges indefinitely.+The one existing escape hatch (``_edges_predate_cohesion``) probes store content+for one specific past fix, which does not generalise.++``repositories.graph_edges_parser_fingerprint`` records the+``parser_fingerprint()`` of the build that last wrote the repo's edges. When it+differs from the running build's, the next update widens its reconcile to every+parsed file once and re-stamps, so this class of staleness cannot recur.++NULL on every store written before this, which reads as a mismatch and heals on+the next update.++``init_db``'s additive schema reconciler picks the column up from the model for+local SQLite stores that never run Alembic; this migration covers the managed+Postgres ones.++Revision ID: 0052+Revises: 0051+Create Date: 2026-08-16+"""++from __future__ import annotations++from collections.abc import Sequence++import sqlalchemy as sa+from alembic import op++# revision identifiers+revision: str = "0052"+down_revision: str | None = "0051"+branch_labels: str | Sequence[str] | None = None+depends_on: str | Sequence[str] | None = None+++def upgrade() -> None:+ op.add_column(+ "repositories",+ sa.Column("graph_edges_parser_fingerprint", sa.String(length=64), nullable=True),+ )+++def downgrade() -> None:+ op.drop_column("repositories", "graph_edges_parser_fingerprint")diff --git a/packages/core/src/repowise/core/persistence/models.py b/packages/core/src/repowise/core/persistence/models.pyindex 52f2cc093..014f95ef6 100644--- a/packages/core/src/repowise/core/persistence/models.py+++ b/packages/core/src/repowise/core/persistence/models.py@@ -78,6 +78,13 @@ class Repository(Base):# reconcile. NULL on indexes written before this, which just means the next# capture walks once and anchors itself.churn_anchor_sha: Mapped[str | None] = mapped_column(String(40), nullable=True)+ # ``parser_fingerprint()`` of the build that last wrote this repo's+ # ``graph_edges``. An incremental update only rewrites the git-changed+ # files' edges, so a query/extractor change would otherwise reach a file+ # only when that file happened to change. A mismatch here widens the next+ # update's edge reconcile to every parsed file, once. NULL on stores written+ # before this, which is treated as a mismatch and heals the same way.+ graph_edges_parser_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)settings_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_now_utcdiff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex dd122aa5e..9204a4d0e 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -533,6 +533,47 @@ async def _edges_predate_cohesion(session: Any, repo_id: str, graph_builder: Anyreturn row is None+async def _stored_edges_parser_fingerprint(session: Any, repo_id: str) -> str | None:+ """The ``parser_fingerprint()`` recorded when this repo's edges were written.++ ``None`` on a store written before the column existed, which is what makes+ every already-shipped store take the widen once.+ """+ from sqlalchemy import select++ from repowise.core.persistence.models import Repository++ return (+ await session.execute(+ select(Repository.graph_edges_parser_fingerprint).where(Repository.id == repo_id)+ )+ ).scalar_one_or_none()+++async def stamp_edges_parser_fingerprint(session: Any, repo_id: str, fingerprint: str) -> None:+ """Record the parser build that just wrote this repo's ``graph_edges``.++ Written only by :func:`persist_incremental_edges`, and only after a widen.+ Deliberately *not* written by the full :func:`persist_ingestion` path, which+ upserts edges (:func:`batch_upsert_graph_edges`) and prunes only the nodes+ of files that vanished: a re-index over an existing store therefore adds the+ new build's edges without removing the old build's surplus ones. Stamping+ there would mark such a store parser-current and permanently suppress the+ delete-then-insert reconcile that is the only thing which clears them. The+ cost of not stamping is one widen on the first update after an index, which+ is wasted on a fresh store and exactly the repair on a re-indexed one.+ """+ from sqlalchemy import update++ from repowise.core.persistence.models import Repository++ await session.execute(+ update(Repository)+ .where(Repository.id == repo_id)+ .values(graph_edges_parser_fingerprint=fingerprint)+ )++async def persist_incremental_edges(session: Any,repo_id: str,@@ -549,16 +590,34 @@ async def persist_incremental_edges(straight from this table, so they decayed on every incremental update. Thisdelete-then-inserts the changed files' outgoing edges (dropping edges thosefiles no longer have). Scoped to the changed set for cost.++ That scoping is also this function's one structural blind spot: an+ extraction change (a tree-sitter query edit, an extractor fix) alters the+ edges of *every* file, but only the git-changed ones get rewritten, so the+ rest keep the old build's edges until they happen to change. The parse cache+ self-invalidates on such a change (``parser_fingerprint`` hashes the ``.scm``+ sources) and the update rebuilds the whole graph in memory anyway, so the+ correct edges are already computed and only the write is too narrow. Widen+ it whenever the fingerprint that last wrote the table differs from the+ running one, then re-stamp. One comparison covers every future extraction+ change; :func:`_edges_predate_cohesion` is the bespoke predecessor, kept+ until shipped stores have all been stamped."""if graph_builder is None or not parsed_files:return+ from repowise.core.ingestion.parse_cache import parser_fingerprintfrom repowise.core.persistence.crud import reconcile_edges_for_files- if await _edges_predate_cohesion(session, repo_id, graph_builder):- # Widen the reconcile to the whole parsed set, once. Same code path,- # so the delete-then-insert still drops edges a file no longer has —- # which is what clears the pre-fix resolution mistakes, an upsert- # alone would leave them behind.+ # Widen the reconcile to the whole parsed set, once. Same code path, so the+ # delete-then-insert still drops edges a file no longer has — which is what+ # clears the old build's resolution mistakes, an upsert alone would leave+ # them behind.+ fingerprint = parser_fingerprint()+ parser_changed = await _stored_edges_parser_fingerprint(session, repo_id) != fingerprint+ if parser_changed:+ changed_paths = [pf.file_info.path for pf in parsed_files]+ logger.info("graph_edges_parser_backfill", repo_id=repo_id, files=len(changed_paths))+ elif await _edges_predate_cohesion(session, repo_id, graph_builder):changed_paths = [pf.file_info.path for pf in parsed_files]logger.info("graph_edges_cohesion_backfill", repo_id=repo_id, files=len(changed_paths))@@ -566,6 +625,11 @@ async def persist_incremental_edges(if not reconcile_paths:returnawait reconcile_edges_for_files(session, repo_id, reconcile_paths, edges)+ # Only after the widen actually ran to completion. Raising here (or in any+ # caller before the commit) leaves the old stamp, so the next update retries+ # rather than stranding the files it never reached.+ if parser_changed:+ await stamp_edges_parser_fingerprint(session, repo_id, fingerprint)# Chunk size for IN (...) deletes — stays under SQLite's host-parameter limit.diff --git a/tests/unit/persistence/test_incremental_edge_persist.py b/tests/unit/persistence/test_incremental_edge_persist.pyindex 31b16178d..0a2bf5fcc 100644--- a/tests/unit/persistence/test_incremental_edge_persist.py+++ b/tests/unit/persistence/test_incremental_edge_persist.py@@ -13,13 +13,20 @@ from __future__ import annotationsimport jsonfrom pathlib import Path+from types import SimpleNamespacefrom sqlalchemy import selectfrom repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder+from repowise.core.ingestion.parse_cache import parser_fingerprintfrom repowise.core.persistence import batch_upsert_graph_edges-from repowise.core.persistence.models import GraphEdge-from repowise.core.pipeline.persist import persist_graph_nodes, persist_incremental_edges+from repowise.core.persistence.models import GraphEdge, Repository+from repowise.core.pipeline.persist import (+ persist_graph_nodes,+ persist_incremental_edges,+ persist_ingestion,+ stamp_edges_parser_fingerprint,+)from tests.unit.persistence.helpers import insert_repo@@ -59,10 +66,18 @@ async def _db_edge_set(session, repo_id: str) -> set[tuple[str, str, str]]:return {(r.source_node_id, r.target_node_id, r.edge_type) for r in rows}-async def _seed_full(session, repo_id: str, gb: GraphBuilder) -> None:- """Mirror the full-init persist: nodes first, then the whole edge set."""+async def _seed_full(+ session, repo_id: str, gb: GraphBuilder, *, fingerprint: str | None = None+) -> None:+ """Mirror the full-init persist: nodes first, then the whole edge set.++ *fingerprint* is what the build that wrote those edges stamped. The real+ full path never stamps (see ``stamp_edges_parser_fingerprint``); this seeds+ the stamp directly so a test can pick the store state it wants.+ """await persist_graph_nodes(session, repo_id, gb)await batch_upsert_graph_edges(session, repo_id, _graph_edges(gb))+ await stamp_edges_parser_fingerprint(session, repo_id, fingerprint or parser_fingerprint())async def test_incremental_update_reconciles_changed_file_edges(async_session, tmp_path):@@ -128,3 +143,139 @@ async def test_incremental_edge_persist_leaves_unchanged_files_untouched(async_s# Both survive: b.py re-inserted its (unchanged) edge, c.py was never touched.assert ("b.py", "a.py", "imports") in afterassert ("c.py", "a.py", "imports") in after+++async def _stored_fingerprint(session, repo_id: str) -> str | None:+ return (+ await session.execute(+ select(Repository.graph_edges_parser_fingerprint).where(Repository.id == repo_id)+ )+ ).scalar_one_or_none()+++async def _seed_missing_edge(session, tmp_path: Path, fingerprint: str | None):+ """A store whose *unchanged* c.py is missing an edge the current parser finds.++ Stands in for what a parser fix does to a shipped store: the in-memory graph+ gains edges on every file, but the table only has the ones the old build+ extracted. *fingerprint* is what that old build stamped (``None`` for a+ store predating the stamp entirely).+ """+ repo = await insert_repo(session)+ (tmp_path / "a.py").write_text("x = 1\n")+ (tmp_path / "b.py").write_text("from a import x\n")+ (tmp_path / "c.py").write_text("from a import x\n")++ gb, parsed = _build_graph(tmp_path)+ await _seed_full(session, repo.id, gb, fingerprint=fingerprint or parser_fingerprint())+ if fingerprint is None:+ await session.execute(+ Repository.__table__.update()+ .where(Repository.id == repo.id)+ .values(graph_edges_parser_fingerprint=None)+ )+ # Drop the edge the old build failed to extract, leaving c.py stale.+ await session.execute(+ GraphEdge.__table__.delete().where(+ GraphEdge.repository_id == repo.id, GraphEdge.source_node_id == "c.py"+ )+ )+ await session.commit()+ assert ("c.py", "a.py", "imports") not in await _db_edge_set(session, repo.id)+ return repo, gb, parsed+++async def test_parser_change_widens_reconcile_to_unchanged_files(async_session, tmp_path):+ """A parser fix must reach files git considers unchanged, without a reindex.++ Scoping the write to the changed set is right for content changes and wrong+ for extraction changes: the latter alter every file's edges at once. The+ stored parser fingerprint is what tells the two apart.+ """+ repo, gb, parsed = await _seed_missing_edge(async_session, tmp_path, "older-parser-build")++ # Only b.py changed on disk, yet the run is under a different parser build.+ await persist_incremental_edges(async_session, repo.id, gb, parsed, ["b.py"])+ await async_session.commit()++ assert ("c.py", "a.py", "imports") in await _db_edge_set(async_session, repo.id)+ # Stamped, so the next update goes back to the cheap changed-set scoping.+ assert await _stored_fingerprint(async_session, repo.id) == parser_fingerprint()+++async def test_store_predating_the_stamp_is_backfilled_once(async_session, tmp_path):+ """A NULL stamp is a mismatch: every already-shipped store takes the widen."""+ repo, gb, parsed = await _seed_missing_edge(async_session, tmp_path, None)++ await persist_incremental_edges(async_session, repo.id, gb, parsed, ["b.py"])+ await async_session.commit()++ assert ("c.py", "a.py", "imports") in await _db_edge_set(async_session, repo.id)+ assert await _stored_fingerprint(async_session, repo.id) == parser_fingerprint()+++async def test_matching_fingerprint_does_not_widen(async_session, tmp_path):+ """Same parser build: stay scoped to the changed set, so the widen is one-shot."""+ repo, gb, parsed = await _seed_missing_edge(async_session, tmp_path, parser_fingerprint())++ await persist_incremental_edges(async_session, repo.id, gb, parsed, ["b.py"])+ await async_session.commit()++ # c.py was not in the changed set and the parser did not change, so its+ # rows are left exactly as they were.+ assert ("c.py", "a.py", "imports") not in await _db_edge_set(async_session, repo.id)+++async def test_full_reindex_does_not_suppress_the_widen(async_session, tmp_path):+ """A re-index must not mark a store parser-current — it cannot clear surplus edges.++ ``persist_ingestion`` upserts edges and prunes only vanished files' nodes,+ so re-indexing an old store adds the new build's edges while leaving the old+ build's wrong ones behind. If it stamped the fingerprint, the widen — the+ only path that deletes those — would never run again for that store.+ """+ repo = await insert_repo(async_session)+ (tmp_path / "a.py").write_text("x = 1\n")+ (tmp_path / "b.py").write_text("from a import x\n")+ (tmp_path / "c.py").write_text("x = 3\n") # imports nothing++ gb, parsed = _build_graph(tmp_path)+ # An older build resolved a c.py -> a.py import the current parser does not.+ await _seed_full(async_session, repo.id, gb, fingerprint="older-parser-build")+ await batch_upsert_graph_edges(+ async_session,+ repo.id,+ [+ {+ "source_node_id": "c.py",+ "target_node_id": "a.py",+ "imported_names_json": "[]",+ "edge_type": "imports",+ "confidence": 1.0,+ }+ ],+ )+ await async_session.commit()+ assert ("c.py", "a.py", "imports") in await _db_edge_set(async_session, repo.id)++ # A full re-index on the new build: every edge rewritten, none deleted.+ await persist_ingestion(+ SimpleNamespace(+ parsed_files=parsed,+ graph_builder=gb,+ external_systems=[],+ execution_flow_report=None,+ source_map={},+ ),+ async_session,+ repo.id,+ )+ await async_session.commit()+ # The surplus edge survived the re-index, and the store is still unstamped.+ assert ("c.py", "a.py", "imports") in await _db_edge_set(async_session, repo.id)+ assert await _stored_fingerprint(async_session, repo.id) == "older-parser-build"++ # So the next update still widens, and that is what clears it.+ await persist_incremental_edges(async_session, repo.id, gb, parsed, ["b.py"])+ await async_session.commit()+ assert ("c.py", "a.py", "imports") not in await _db_edge_set(async_session, repo.id)