Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(update): persist wiki_symbols on incremental updates (#795)
Raghav Chamadiya2cc5589e7f76
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.pyindex c4050fec3..368f6db98 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py@@ -716,6 +716,7 @@ def run_update([fd.path for fd in file_diffs],file_diffs=file_diffs,knowledge_graph_result=knowledge_graph_result,+ parsed_files=parsed_files,degraded=degraded,)except Exception as exc:@@ -1126,6 +1127,7 @@ def run_update(knowledge_graph_result=knowledge_graph_result,degraded=degraded,decay_paths=affected.decay_only,+ parsed_files=parsed_files,)except Exception as exc:if emitter is not None:diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.pyindex 4bbefdc68..61953d60a 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py@@ -139,9 +139,10 @@ def _persist_index_only_update(changed_paths: list[str],file_diffs: list | None = None,knowledge_graph_result: Any | None = None,+ parsed_files: list | None = None,degraded: list[str] | None = None,) -> None:- """Persist the index-only update (graph + git + dead-code + health + KG),+ """Persist the index-only update (graph + symbols + git + dead-code + health + KG),save state, and print the completion line. No LLM regeneration.DB persistence delegates to :mod:`repowise.core.pipeline.incremental`;@@ -161,6 +162,7 @@ def _persist_index_only_update(changed_paths,file_diffs=file_diffs,knowledge_graph_result=knowledge_graph_result,+ parsed_files=parsed_files,log=console.print,degraded=degraded,)@@ -292,6 +294,7 @@ def _persist_full_update(knowledge_graph_result: Any | None,degraded: list[str],decay_paths: list[str] | None = None,+ parsed_files: list | None = None,) -> int:"""Persist a full (LLM-regenerating) update in one transaction.@@ -324,6 +327,7 @@ def _persist_full_update(knowledge_graph_result=knowledge_graph_result,degraded=degraded,decay_paths=decay_paths,+ parsed_files=parsed_files,))@@ -344,6 +348,7 @@ async def _persist_full_update_async(knowledge_graph_result: Any | None,degraded: list[str],decay_paths: list[str] | None = None,+ parsed_files: list | None = None,) -> int:from repowise.cli.helpers import get_db_url_for_repofrom repowise.core.persistence import (@@ -532,6 +537,18 @@ async def _persist_full_update_async(except Exception as exc:_skip("Graph nodes persist", exc)+ # Refresh wiki_symbols for the changed files (upsert + within-file+ # prune). Without this, symbol bounds fossilize at the last full+ # index and the get_answer hydrator serves drifted signatures.+ try:+ from repowise.core.pipeline.persist import persist_incremental_symbols++ await persist_incremental_symbols(+ session, repo_id, parsed_files, [fd.path for fd in file_diffs]+ )+ except Exception as exc:+ _skip("Symbol persist", exc)+# Record a GenerationJob so the web UI "last synced" timestamp updates.try:from datetime import UTC as _UTCdiff --git a/packages/core/src/repowise/core/persistence/crud/external_systems.py b/packages/core/src/repowise/core/persistence/crud/external_systems.pyindex 5766e39ea..c7d7c742f 100644--- a/packages/core/src/repowise/core/persistence/crud/external_systems.py+++ b/packages/core/src/repowise/core/persistence/crud/external_systems.py@@ -128,6 +128,28 @@ def _symbol_id(sym: Any) -> str:return getattr(sym, "id", None) or f"{sym.file_path}::{sym.name}"+def _new_wiki_symbol(repository_id: str, sym: Any) -> WikiSymbol:+ """Build a WikiSymbol row from an ingestion Symbol (shared insert recipe)."""+ return WikiSymbol(+ id=_new_uuid(),+ repository_id=repository_id,+ file_path=getattr(sym, "file_path", ""),+ symbol_id=_symbol_id(sym),+ name=sym.name,+ qualified_name=getattr(sym, "qualified_name", sym.name),+ kind=sym.kind,+ signature=getattr(sym, "signature", ""),+ start_line=getattr(sym, "start_line", 0),+ end_line=getattr(sym, "end_line", 0),+ docstring=getattr(sym, "docstring", None),+ visibility=getattr(sym, "visibility", "public"),+ is_async=getattr(sym, "is_async", False),+ complexity_estimate=getattr(sym, "complexity_estimate", 0),+ language=getattr(sym, "language", ""),+ parent_name=getattr(sym, "parent_name", None),+ )++def _update_wiki_symbol(existing: WikiSymbol, sym: Any) -> None:existing.name = sym.nameexisting.qualified_name = getattr(sym, "qualified_name", sym.name)@@ -161,22 +183,75 @@ async def batch_upsert_symbols(item_key_fn=_symbol_id,row_key_fn=lambda row: row.symbol_id,update_fn=_update_wiki_symbol,- insert_fn=lambda sym: WikiSymbol(- id=_new_uuid(),- repository_id=repository_id,- file_path=getattr(sym, "file_path", ""),- symbol_id=_symbol_id(sym),- name=sym.name,- qualified_name=getattr(sym, "qualified_name", sym.name),- kind=sym.kind,- signature=getattr(sym, "signature", ""),- start_line=getattr(sym, "start_line", 0),- end_line=getattr(sym, "end_line", 0),- docstring=getattr(sym, "docstring", None),- visibility=getattr(sym, "visibility", "public"),- is_async=getattr(sym, "is_async", False),- complexity_estimate=getattr(sym, "complexity_estimate", 0),- language=getattr(sym, "language", ""),- parent_name=getattr(sym, "parent_name", None),- ),+ insert_fn=lambda sym: _new_wiki_symbol(repository_id, sym),)+++# Chunk size for the scoped existence SELECT — stays under SQLite's+# host-parameter limit when a wide catch-up update touches many files.+_SYMBOL_RECONCILE_CHUNK = 400+++async def reconcile_symbols_for_files(+ session: AsyncSession,+ repository_id: str,+ file_paths: list[str],+ symbols: list, # fresh parse of exactly those files+) -> int:+ """Make ``wiki_symbols`` for *file_paths* match a fresh parse (*symbols*).++ The incremental update path re-parses changed files but historically never+ persisted their symbols, so bounds fossilized at the last full index and+ the get_answer hydrator served drifted signatures. The repo-wide+ :func:`batch_upsert_symbols` reloads *every* symbol row for the repo, so+ calling it per update would SELECT the whole table. This scopes the+ existence query to the changed *file_paths*, upserts their symbols, and+ prunes rows for symbols that vanished from a still-existing file (a symbol+ deleted/renamed inside a file a pure upsert would otherwise leave behind as+ a stale row). Whole-file deletions are handled separately by the stale-row+ pruner; this only touches the files it was handed.++ Returns the number of pruned rows.+ """+ scoped = [p for p in dict.fromkeys(file_paths) if p]+ if not scoped:+ return 0++ existing_rows: list[WikiSymbol] = []+ for i in range(0, len(scoped), _SYMBOL_RECONCILE_CHUNK):+ chunk = scoped[i : i + _SYMBOL_RECONCILE_CHUNK]+ rows = (+ (+ await session.execute(+ select(WikiSymbol).where(+ WikiSymbol.repository_id == repository_id,+ WikiSymbol.file_path.in_(chunk),+ )+ )+ )+ .scalars()+ .all()+ )+ existing_rows.extend(rows)+ by_id: dict[str, WikiSymbol] = {row.symbol_id: row for row in existing_rows}++ fresh_ids: set[str] = set()+ for sym in symbols:+ sid = _symbol_id(sym)+ # Within-batch duplicate ids: first inserts, later ones update it.+ fresh_ids.add(sid)+ existing = by_id.get(sid)+ if existing is not None:+ _update_wiki_symbol(existing, sym)+ else:+ obj = _new_wiki_symbol(repository_id, sym)+ session.add(obj)+ by_id[sid] = obj++ pruned = 0+ for sid, row in list(by_id.items()):+ if sid not in fresh_ids:+ await session.delete(row)+ pruned += 1+ await session.flush()+ return pruneddiff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.pyindex 3f4efb6aa..531b4dadc 100644--- a/packages/core/src/repowise/core/pipeline/incremental.py+++ b/packages/core/src/repowise/core/pipeline/incremental.py@@ -538,10 +538,11 @@ async def persist_incremental_index(current_graph_file_paths: set[str] | None = None,file_diffs: list[Any] | None = None,knowledge_graph_result: Any | None = None,+ parsed_files: list[Any] | None = None,log: LogFn | None = None,degraded: list[str] | None = None,) -> None:- """Persist an incremental index refresh (graph + git + dead-code + health).+ """Persist an incremental index refresh (graph + symbols + git + dead-code + health).Upsert-only: unchanged files keep their existing rows, unlike``persist_pipeline_result``'s delete-then-insert. State-file updates stay@@ -647,6 +648,18 @@ async def persist_incremental_index(except Exception as exc:_skip("Graph nodes persist", exc)+ # Refresh wiki_symbols for the changed files. Historically the+ # incremental path re-parsed but never persisted symbols, so their+ # start/end bounds fossilized at the last full index and the+ # get_answer hydrator served drifted signatures. Scoped to the+ # changed set for cost.+ try:+ from repowise.core.pipeline.persist import persist_incremental_symbols++ await persist_incremental_symbols(session, repo_id, parsed_files, changed_paths)+ except Exception as exc:+ _skip("Symbol persist", exc)+if knowledge_graph_result is not None:try:from repowise.core.pipeline.persist import persist_kgdiff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex 4b47995bf..6f0d42a05 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -251,6 +251,57 @@ async def persist_graph_nodes(logger.warning("graph_node_membership_materialize_skipped", error=str(exc))+def _changed_file_symbols(+ parsed_files: list[Any] | None, changed_paths: list[str]+) -> tuple[list[str], list[Any]]:+ """``(reconcile_paths, symbols)`` for files that both changed and parsed.++ Restricting to the intersection means a changed file that failed to parse+ this run keeps its existing symbol rows (mirrors the graph, which skips+ unparsed files) rather than having them wrongly pruned on a transient+ failure. Mutates ``sym.file_path`` where the parser left it unset, same as+ the full persist path.+ """+ changed = set(changed_paths or [])+ reconcile_paths: list[str] = []+ symbols: list[Any] = []+ for pf in parsed_files or []:+ path = pf.file_info.path+ if path not in changed:+ continue+ reconcile_paths.append(path)+ for sym in pf.symbols:+ if not getattr(sym, "file_path", None):+ sym.file_path = path+ symbols.append(sym)+ return reconcile_paths, symbols+++async def persist_incremental_symbols(+ session: Any,+ repo_id: str,+ parsed_files: list[Any] | None,+ changed_paths: list[str],+) -> None:+ """Refresh ``wiki_symbols`` for changed+parsed files on an incremental update.++ The incremental update path re-parses changed files but never persisted+ their symbols, so wiki_symbols bounds fossilized at the last full index and+ the get_answer hydrator served drifted signatures/bodies. This upserts the+ changed files' fresh symbols and prunes symbols that vanished from a+ still-existing file. Scoped to the changed set for cost — the repo-wide+ ``batch_upsert_symbols`` reloads every symbol row.+ """+ if not parsed_files:+ return+ from repowise.core.persistence.crud import reconcile_symbols_for_files++ reconcile_paths, symbols = _changed_file_symbols(parsed_files, changed_paths)+ if not reconcile_paths:+ return+ await reconcile_symbols_for_files(session, repo_id, reconcile_paths, symbols)++# Chunk size for IN (...) deletes — stays under SQLite's host-parameter limit._PRUNE_CHUNK = 500diff --git a/packages/core/src/repowise/core/workspace/update.py b/packages/core/src/repowise/core/workspace/update.pyindex 9d19acf01..483f2fbe6 100644--- a/packages/core/src/repowise/core/workspace/update.py+++ b/packages/core/src/repowise/core/workspace/update.py@@ -386,6 +386,7 @@ async def _incremental_repo_update(# until the next full regeneration.file_diffs=file_diffs,knowledge_graph_result=kg,+ parsed_files=parsed_files,log=_log.info,)diff --git a/tests/unit/persistence/test_incremental_symbol_persist.py b/tests/unit/persistence/test_incremental_symbol_persist.pynew file mode 100644index 000000000..7afa3fadf--- /dev/null+++ b/tests/unit/persistence/test_incremental_symbol_persist.py@@ -0,0 +1,123 @@+"""Incremental updates must refresh ``wiki_symbols`` for changed files.++The incremental update path re-parses changed files but historically never+persisted their symbols, so symbol start/end bounds fossilized at the last+full index. The get_answer hydrator then read signatures/bodies from the live+file at those stale bounds and served garbled text. These tests drive the real+parser over a file whose symbol moves (lines inserted above it) and assert the+persisted bounds and ``updated_at`` follow, that a symbol deleted from a+still-existing file is pruned, and that unchanged files are left alone.+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta+from pathlib import Path++from sqlalchemy import select++from repowise.core.ingestion import ASTParser, FileTraverser+from repowise.core.persistence.models import WikiSymbol+from repowise.core.pipeline.persist import persist_incremental_symbols+from tests.unit.persistence.helpers import insert_repo+++def _parse(repo_dir: Path) -> list:+ """Parse every file under *repo_dir* into ParsedFile objects."""+ traverser = FileTraverser(repo_dir)+ parser = ASTParser()+ return [parser.parse_file(fi, Path(fi.abs_path).read_bytes()) for fi in traverser.traverse()]+++async def _symbol_row(session, repo_id: str, symbol_id: str) -> WikiSymbol | None:+ return (+ await session.execute(+ select(WikiSymbol).where(+ WikiSymbol.repository_id == repo_id,+ WikiSymbol.symbol_id == symbol_id,+ )+ )+ ).scalar_one_or_none()+++async def test_incremental_update_refreshes_moved_symbol(async_session, tmp_path):+ repo = await insert_repo(async_session)++ # v1: foo near the top, bar below it.+ (tmp_path / "m.py").write_text(+ "import os\n\n\ndef foo():\n return 1\n\n\ndef bar():\n return 2\n"+ )+ parsed_v1 = _parse(tmp_path)+ await persist_incremental_symbols(async_session, repo.id, parsed_v1, ["m.py"])+ await async_session.commit()++ foo_v1 = await _symbol_row(async_session, repo.id, "m.py::foo")+ assert foo_v1 is not None+ v1_start = foo_v1.start_line+ assert await _symbol_row(async_session, repo.id, "m.py::bar") is not None++ # Backdate updated_at so the refresh is detectable as a real re-write.+ old_ts = datetime.now(UTC) - timedelta(days=7)+ foo_v1.updated_at = old_ts+ await async_session.commit()++ # v2: five blank lines inserted above the code moves foo down; bar removed.+ (tmp_path / "m.py").write_text("import os\n" + "\n" * 5 + "\n\ndef foo():\n return 1\n")+ parsed_v2 = _parse(tmp_path)+ from repowise.core.persistence.crud import reconcile_symbols_for_files++ # Sanity: the fresh parse really moved foo before we assert the DB followed.+ v2_foo = next(s for pf in parsed_v2 for s in pf.symbols if s.name == "foo")+ assert v2_foo.start_line > v1_start++ await persist_incremental_symbols(async_session, repo.id, parsed_v2, ["m.py"])+ await async_session.commit()++ foo_v2 = await _symbol_row(async_session, repo.id, "m.py::foo")+ assert foo_v2.start_line == v2_foo.start_line+ assert foo_v2.start_line > v1_start+ assert foo_v2.updated_at > old_ts+ # bar was deleted from a still-existing file, so its row must be pruned.+ assert await _symbol_row(async_session, repo.id, "m.py::bar") is None++ # reconcile returns the prune count directly for the changed-file scope.+ _reparse = _parse(tmp_path)+ syms = [s for pf in _reparse for s in pf.symbols]+ for s in syms:+ s.file_path = "m.py"+ pruned = await reconcile_symbols_for_files(async_session, repo.id, ["m.py"], syms)+ assert pruned == 0 # nothing new to prune on a no-op re-run+++async def test_incremental_update_leaves_unchanged_files_untouched(async_session, tmp_path):+ """Only files in the changed set are re-written; others keep their rows."""+ repo = await insert_repo(async_session)++ # Seed an unchanged file's symbol directly with an old timestamp.+ old_ts = datetime.now(UTC) - timedelta(days=30)+ async_session.add(+ WikiSymbol(+ repository_id=repo.id,+ file_path="other.py",+ symbol_id="other.py::helper",+ name="helper",+ qualified_name="helper",+ kind="function",+ start_line=10,+ end_line=12,+ updated_at=old_ts,+ )+ )+ await async_session.commit()++ (tmp_path / "m.py").write_text("def foo():\n return 1\n")+ parsed = _parse(tmp_path)+ # changed set names only m.py, even though other.py exists in the DB.+ await persist_incremental_symbols(async_session, repo.id, parsed, ["m.py"])+ await async_session.commit()++ other = await _symbol_row(async_session, repo.id, "other.py::helper")+ assert other is not None+ # SQLite drops tzinfo on read; compare against the naive UTC wall clock.+ assert other.updated_at.replace(tzinfo=None) == old_ts.replace(tzinfo=None)+ assert other.start_line == 10 # untouched by the scoped refresh