Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(update): persist wiki_symbols on incremental updates (#795)

Raghav Chamadiya2cc5589e7f76
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
index c4050fec3..368f6db98 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
@@ -716,6 +716,7 @@ def run_update(
[fd.path for fd in file_diffs],
file_diffs=file_diffs,
knowledge_graph_result=knowledge_graph_result,
+ parsed_files=parsed_files,
degraded=degraded,
)
except Exception as exc:
@@ -1126,6 +1127,7 @@ def run_update(
knowledge_graph_result=knowledge_graph_result,
degraded=degraded,
decay_paths=affected.decay_only,
+ parsed_files=parsed_files,
)
except Exception as exc:
if emitter is not None:
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
index 4bbefdc68..61953d60a 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
@@ -139,9 +139,10 @@ def _persist_index_only_update(
changed_paths: list[str],
file_diffs: list | None = None,
knowledge_graph_result: Any | None = None,
+ parsed_files: list | None = None,
degraded: list[str] | None = None,
) -> None:
- """Persist the index-only update (graph + git + dead-code + health + KG),
+ """Persist the index-only update (graph + symbols + git + dead-code + health + KG),
save state, and print the completion line. No LLM regeneration.
DB persistence delegates to :mod:`repowise.core.pipeline.incremental`;
@@ -161,6 +162,7 @@ def _persist_index_only_update(
changed_paths,
file_diffs=file_diffs,
knowledge_graph_result=knowledge_graph_result,
+ parsed_files=parsed_files,
log=console.print,
degraded=degraded,
)
@@ -292,6 +294,7 @@ def _persist_full_update(
knowledge_graph_result: Any | None,
degraded: list[str],
decay_paths: list[str] | None = None,
+ parsed_files: list | None = None,
) -> int:
"""Persist a full (LLM-regenerating) update in one transaction.
@@ -324,6 +327,7 @@ def _persist_full_update(
knowledge_graph_result=knowledge_graph_result,
degraded=degraded,
decay_paths=decay_paths,
+ parsed_files=parsed_files,
)
)
@@ -344,6 +348,7 @@ async def _persist_full_update_async(
knowledge_graph_result: Any | None,
degraded: list[str],
decay_paths: list[str] | None = None,
+ parsed_files: list | None = None,
) -> int:
from repowise.cli.helpers import get_db_url_for_repo
from repowise.core.persistence import (
@@ -532,6 +537,18 @@ async def _persist_full_update_async(
except Exception as exc:
_skip("Graph nodes persist", exc)
+ # Refresh wiki_symbols for the changed files (upsert + within-file
+ # prune). Without this, symbol bounds fossilize at the last full
+ # index and the get_answer hydrator serves drifted signatures.
+ try:
+ from repowise.core.pipeline.persist import persist_incremental_symbols
+
+ await persist_incremental_symbols(
+ session, repo_id, parsed_files, [fd.path for fd in file_diffs]
+ )
+ except Exception as exc:
+ _skip("Symbol persist", exc)
+
# Record a GenerationJob so the web UI "last synced" timestamp updates.
try:
from datetime import UTC as _UTC
diff --git a/packages/core/src/repowise/core/persistence/crud/external_systems.py b/packages/core/src/repowise/core/persistence/crud/external_systems.py
index 5766e39ea..c7d7c742f 100644
--- a/packages/core/src/repowise/core/persistence/crud/external_systems.py
+++ b/packages/core/src/repowise/core/persistence/crud/external_systems.py
@@ -128,6 +128,28 @@ def _symbol_id(sym: Any) -> str:
return getattr(sym, "id", None) or f"{sym.file_path}::{sym.name}"
+def _new_wiki_symbol(repository_id: str, sym: Any) -> WikiSymbol:
+ """Build a WikiSymbol row from an ingestion Symbol (shared insert recipe)."""
+ return WikiSymbol(
+ id=_new_uuid(),
+ repository_id=repository_id,
+ file_path=getattr(sym, "file_path", ""),
+ symbol_id=_symbol_id(sym),
+ name=sym.name,
+ qualified_name=getattr(sym, "qualified_name", sym.name),
+ kind=sym.kind,
+ signature=getattr(sym, "signature", ""),
+ start_line=getattr(sym, "start_line", 0),
+ end_line=getattr(sym, "end_line", 0),
+ docstring=getattr(sym, "docstring", None),
+ visibility=getattr(sym, "visibility", "public"),
+ is_async=getattr(sym, "is_async", False),
+ complexity_estimate=getattr(sym, "complexity_estimate", 0),
+ language=getattr(sym, "language", ""),
+ parent_name=getattr(sym, "parent_name", None),
+ )
+
+
def _update_wiki_symbol(existing: WikiSymbol, sym: Any) -> None:
existing.name = sym.name
existing.qualified_name = getattr(sym, "qualified_name", sym.name)
@@ -161,22 +183,75 @@ async def batch_upsert_symbols(
item_key_fn=_symbol_id,
row_key_fn=lambda row: row.symbol_id,
update_fn=_update_wiki_symbol,
- insert_fn=lambda sym: WikiSymbol(
- id=_new_uuid(),
- repository_id=repository_id,
- file_path=getattr(sym, "file_path", ""),
- symbol_id=_symbol_id(sym),
- name=sym.name,
- qualified_name=getattr(sym, "qualified_name", sym.name),
- kind=sym.kind,
- signature=getattr(sym, "signature", ""),
- start_line=getattr(sym, "start_line", 0),
- end_line=getattr(sym, "end_line", 0),
- docstring=getattr(sym, "docstring", None),
- visibility=getattr(sym, "visibility", "public"),
- is_async=getattr(sym, "is_async", False),
- complexity_estimate=getattr(sym, "complexity_estimate", 0),
- language=getattr(sym, "language", ""),
- parent_name=getattr(sym, "parent_name", None),
- ),
+ insert_fn=lambda sym: _new_wiki_symbol(repository_id, sym),
)
+
+
+# Chunk size for the scoped existence SELECT — stays under SQLite's
+# host-parameter limit when a wide catch-up update touches many files.
+_SYMBOL_RECONCILE_CHUNK = 400
+
+
+async def reconcile_symbols_for_files(
+ session: AsyncSession,
+ repository_id: str,
+ file_paths: list[str],
+ symbols: list, # fresh parse of exactly those files
+) -> int:
+ """Make ``wiki_symbols`` for *file_paths* match a fresh parse (*symbols*).
+
+ The incremental update path re-parses changed files but historically never
+ persisted their symbols, so bounds fossilized at the last full index and
+ the get_answer hydrator served drifted signatures. The repo-wide
+ :func:`batch_upsert_symbols` reloads *every* symbol row for the repo, so
+ calling it per update would SELECT the whole table. This scopes the
+ existence query to the changed *file_paths*, upserts their symbols, and
+ prunes rows for symbols that vanished from a still-existing file (a symbol
+ deleted/renamed inside a file a pure upsert would otherwise leave behind as
+ a stale row). Whole-file deletions are handled separately by the stale-row
+ pruner; this only touches the files it was handed.
+
+ Returns the number of pruned rows.
+ """
+ scoped = [p for p in dict.fromkeys(file_paths) if p]
+ if not scoped:
+ return 0
+
+ existing_rows: list[WikiSymbol] = []
+ for i in range(0, len(scoped), _SYMBOL_RECONCILE_CHUNK):
+ chunk = scoped[i : i + _SYMBOL_RECONCILE_CHUNK]
+ rows = (
+ (
+ await session.execute(
+ select(WikiSymbol).where(
+ WikiSymbol.repository_id == repository_id,
+ WikiSymbol.file_path.in_(chunk),
+ )
+ )
+ )
+ .scalars()
+ .all()
+ )
+ existing_rows.extend(rows)
+ by_id: dict[str, WikiSymbol] = {row.symbol_id: row for row in existing_rows}
+
+ fresh_ids: set[str] = set()
+ for sym in symbols:
+ sid = _symbol_id(sym)
+ # Within-batch duplicate ids: first inserts, later ones update it.
+ fresh_ids.add(sid)
+ existing = by_id.get(sid)
+ if existing is not None:
+ _update_wiki_symbol(existing, sym)
+ else:
+ obj = _new_wiki_symbol(repository_id, sym)
+ session.add(obj)
+ by_id[sid] = obj
+
+ pruned = 0
+ for sid, row in list(by_id.items()):
+ if sid not in fresh_ids:
+ await session.delete(row)
+ pruned += 1
+ await session.flush()
+ return pruned
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.py
index 3f4efb6aa..531b4dadc 100644
--- a/packages/core/src/repowise/core/pipeline/incremental.py
+++ b/packages/core/src/repowise/core/pipeline/incremental.py
@@ -538,10 +538,11 @@ async def persist_incremental_index(
current_graph_file_paths: set[str] | None = None,
file_diffs: list[Any] | None = None,
knowledge_graph_result: Any | None = None,
+ parsed_files: list[Any] | None = None,
log: LogFn | None = None,
degraded: list[str] | None = None,
) -> None:
- """Persist an incremental index refresh (graph + git + dead-code + health).
+ """Persist an incremental index refresh (graph + symbols + git + dead-code + health).
Upsert-only: unchanged files keep their existing rows, unlike
``persist_pipeline_result``'s delete-then-insert. State-file updates stay
@@ -647,6 +648,18 @@ async def persist_incremental_index(
except Exception as exc:
_skip("Graph nodes persist", exc)
+ # Refresh wiki_symbols for the changed files. Historically the
+ # incremental path re-parsed but never persisted symbols, so their
+ # start/end bounds fossilized at the last full index and the
+ # get_answer hydrator served drifted signatures. Scoped to the
+ # changed set for cost.
+ try:
+ from repowise.core.pipeline.persist import persist_incremental_symbols
+
+ await persist_incremental_symbols(session, repo_id, parsed_files, changed_paths)
+ except Exception as exc:
+ _skip("Symbol persist", exc)
+
if knowledge_graph_result is not None:
try:
from repowise.core.pipeline.persist import persist_kg
diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.py
index 4b47995bf..6f0d42a05 100644
--- a/packages/core/src/repowise/core/pipeline/persist.py
+++ b/packages/core/src/repowise/core/pipeline/persist.py
@@ -251,6 +251,57 @@ async def persist_graph_nodes(
logger.warning("graph_node_membership_materialize_skipped", error=str(exc))
+def _changed_file_symbols(
+ parsed_files: list[Any] | None, changed_paths: list[str]
+) -> tuple[list[str], list[Any]]:
+ """``(reconcile_paths, symbols)`` for files that both changed and parsed.
+
+ Restricting to the intersection means a changed file that failed to parse
+ this run keeps its existing symbol rows (mirrors the graph, which skips
+ unparsed files) rather than having them wrongly pruned on a transient
+ failure. Mutates ``sym.file_path`` where the parser left it unset, same as
+ the full persist path.
+ """
+ changed = set(changed_paths or [])
+ reconcile_paths: list[str] = []
+ symbols: list[Any] = []
+ for pf in parsed_files or []:
+ path = pf.file_info.path
+ if path not in changed:
+ continue
+ reconcile_paths.append(path)
+ for sym in pf.symbols:
+ if not getattr(sym, "file_path", None):
+ sym.file_path = path
+ symbols.append(sym)
+ return reconcile_paths, symbols
+
+
+async def persist_incremental_symbols(
+ session: Any,
+ repo_id: str,
+ parsed_files: list[Any] | None,
+ changed_paths: list[str],
+) -> None:
+ """Refresh ``wiki_symbols`` for changed+parsed files on an incremental update.
+
+ The incremental update path re-parses changed files but never persisted
+ their symbols, so wiki_symbols bounds fossilized at the last full index and
+ the get_answer hydrator served drifted signatures/bodies. This upserts the
+ changed files' fresh symbols and prunes symbols that vanished from a
+ still-existing file. Scoped to the changed set for cost — the repo-wide
+ ``batch_upsert_symbols`` reloads every symbol row.
+ """
+ if not parsed_files:
+ return
+ from repowise.core.persistence.crud import reconcile_symbols_for_files
+
+ reconcile_paths, symbols = _changed_file_symbols(parsed_files, changed_paths)
+ if not reconcile_paths:
+ return
+ await reconcile_symbols_for_files(session, repo_id, reconcile_paths, symbols)
+
+
# Chunk size for IN (...) deletes — stays under SQLite's host-parameter limit.
_PRUNE_CHUNK = 500
diff --git a/packages/core/src/repowise/core/workspace/update.py b/packages/core/src/repowise/core/workspace/update.py
index 9d19acf01..483f2fbe6 100644
--- a/packages/core/src/repowise/core/workspace/update.py
+++ b/packages/core/src/repowise/core/workspace/update.py
@@ -386,6 +386,7 @@ async def _incremental_repo_update(
# until the next full regeneration.
file_diffs=file_diffs,
knowledge_graph_result=kg,
+ parsed_files=parsed_files,
log=_log.info,
)
diff --git a/tests/unit/persistence/test_incremental_symbol_persist.py b/tests/unit/persistence/test_incremental_symbol_persist.py
new file mode 100644
index 000000000..7afa3fadf
--- /dev/null
+++ b/tests/unit/persistence/test_incremental_symbol_persist.py
@@ -0,0 +1,123 @@
+"""Incremental updates must refresh ``wiki_symbols`` for changed files.
+
+The incremental update path re-parses changed files but historically never
+persisted their symbols, so symbol start/end bounds fossilized at the last
+full index. The get_answer hydrator then read signatures/bodies from the live
+file at those stale bounds and served garbled text. These tests drive the real
+parser over a file whose symbol moves (lines inserted above it) and assert the
+persisted bounds and ``updated_at`` follow, that a symbol deleted from a
+still-existing file is pruned, and that unchanged files are left alone.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+from sqlalchemy import select
+
+from repowise.core.ingestion import ASTParser, FileTraverser
+from repowise.core.persistence.models import WikiSymbol
+from repowise.core.pipeline.persist import persist_incremental_symbols
+from tests.unit.persistence.helpers import insert_repo
+
+
+def _parse(repo_dir: Path) -> list:
+ """Parse every file under *repo_dir* into ParsedFile objects."""
+ traverser = FileTraverser(repo_dir)
+ parser = ASTParser()
+ return [parser.parse_file(fi, Path(fi.abs_path).read_bytes()) for fi in traverser.traverse()]
+
+
+async def _symbol_row(session, repo_id: str, symbol_id: str) -> WikiSymbol | None:
+ return (
+ await session.execute(
+ select(WikiSymbol).where(
+ WikiSymbol.repository_id == repo_id,
+ WikiSymbol.symbol_id == symbol_id,
+ )
+ )
+ ).scalar_one_or_none()
+
+
+async def test_incremental_update_refreshes_moved_symbol(async_session, tmp_path):
+ repo = await insert_repo(async_session)
+
+ # v1: foo near the top, bar below it.
+ (tmp_path / "m.py").write_text(
+ "import os\n\n\ndef foo():\n return 1\n\n\ndef bar():\n return 2\n"
+ )
+ parsed_v1 = _parse(tmp_path)
+ await persist_incremental_symbols(async_session, repo.id, parsed_v1, ["m.py"])
+ await async_session.commit()
+
+ foo_v1 = await _symbol_row(async_session, repo.id, "m.py::foo")
+ assert foo_v1 is not None
+ v1_start = foo_v1.start_line
+ assert await _symbol_row(async_session, repo.id, "m.py::bar") is not None
+
+ # Backdate updated_at so the refresh is detectable as a real re-write.
+ old_ts = datetime.now(UTC) - timedelta(days=7)
+ foo_v1.updated_at = old_ts
+ await async_session.commit()
+
+ # v2: five blank lines inserted above the code moves foo down; bar removed.
+ (tmp_path / "m.py").write_text("import os\n" + "\n" * 5 + "\n\ndef foo():\n return 1\n")
+ parsed_v2 = _parse(tmp_path)
+ from repowise.core.persistence.crud import reconcile_symbols_for_files
+
+ # Sanity: the fresh parse really moved foo before we assert the DB followed.
+ v2_foo = next(s for pf in parsed_v2 for s in pf.symbols if s.name == "foo")
+ assert v2_foo.start_line > v1_start
+
+ await persist_incremental_symbols(async_session, repo.id, parsed_v2, ["m.py"])
+ await async_session.commit()
+
+ foo_v2 = await _symbol_row(async_session, repo.id, "m.py::foo")
+ assert foo_v2.start_line == v2_foo.start_line
+ assert foo_v2.start_line > v1_start
+ assert foo_v2.updated_at > old_ts
+ # bar was deleted from a still-existing file, so its row must be pruned.
+ assert await _symbol_row(async_session, repo.id, "m.py::bar") is None
+
+ # reconcile returns the prune count directly for the changed-file scope.
+ _reparse = _parse(tmp_path)
+ syms = [s for pf in _reparse for s in pf.symbols]
+ for s in syms:
+ s.file_path = "m.py"
+ pruned = await reconcile_symbols_for_files(async_session, repo.id, ["m.py"], syms)
+ assert pruned == 0 # nothing new to prune on a no-op re-run
+
+
+async def test_incremental_update_leaves_unchanged_files_untouched(async_session, tmp_path):
+ """Only files in the changed set are re-written; others keep their rows."""
+ repo = await insert_repo(async_session)
+
+ # Seed an unchanged file's symbol directly with an old timestamp.
+ old_ts = datetime.now(UTC) - timedelta(days=30)
+ async_session.add(
+ WikiSymbol(
+ repository_id=repo.id,
+ file_path="other.py",
+ symbol_id="other.py::helper",
+ name="helper",
+ qualified_name="helper",
+ kind="function",
+ start_line=10,
+ end_line=12,
+ updated_at=old_ts,
+ )
+ )
+ await async_session.commit()
+
+ (tmp_path / "m.py").write_text("def foo():\n return 1\n")
+ parsed = _parse(tmp_path)
+ # changed set names only m.py, even though other.py exists in the DB.
+ await persist_incremental_symbols(async_session, repo.id, parsed, ["m.py"])
+ await async_session.commit()
+
+ other = await _symbol_row(async_session, repo.id, "other.py::helper")
+ assert other is not None
+ # SQLite drops tzinfo on read; compare against the naive UTC wall clock.
+ assert other.updated_at.replace(tzinfo=None) == old_ts.replace(tzinfo=None)
+ assert other.start_line == 10 # untouched by the scoped refresh