Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf: cache parsing and betweenness across incremental updates (#369)

Raghav Chamadiya2ef3f76cfd79
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd.py b/packages/cli/src/repowise/cli/commands/update_cmd.py
index 66f6ed916..1dd532e8c 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd.py
@@ -346,22 +346,45 @@ def _build_repo_graph(
"""
from pathlib import Path as PathlibPath
- from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder
+ from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder, compute_content_hash
traverser = FileTraverser(repo_path, extra_exclude_patterns=exclude_patterns or None)
file_infos = list(traverser.traverse())
repo_structure = traverser.get_repo_structure()
- parser = ASTParser()
+ # Content-hash parse cache: an incremental update re-ingests the whole
+ # repo, but only the changed files actually need a tree-sitter parse.
+ # Best-effort — any cache failure falls back to a full parse.
+ parse_cache = None
+ try:
+ from repowise.core.ingestion.parse_cache import ParseCache
+
+ parse_cache = ParseCache(PathlibPath(repo_path) / ".repowise")
+ parse_cache.load()
+ except Exception:
+ parse_cache = None
+
+ parser: Any = None # constructed lazily — every-file-cached updates skip query compilation
parsed_files: list = []
source_map: dict[str, bytes] = {}
- graph_builder = GraphBuilder(repo_path, exclude_patterns=exclude_patterns)
+ graph_builder = GraphBuilder(
+ repo_path,
+ exclude_patterns=exclude_patterns,
+ centrality_cache_dir=PathlibPath(repo_path) / ".repowise",
+ )
skipped = 0
for fi in file_infos:
try:
source = PathlibPath(fi.abs_path).read_bytes()
- parsed = parser.parse_file(fi, source)
+ content_hash = compute_content_hash(source)
+ parsed = parse_cache.get(fi, content_hash) if parse_cache is not None else None
+ if parsed is None:
+ if parser is None:
+ parser = ASTParser()
+ parsed = parser.parse_file(fi, source)
+ if parse_cache is not None:
+ parse_cache.put(parsed, content_hash)
except Exception:
skipped += 1
continue
@@ -370,6 +393,8 @@ def _build_repo_graph(
source_map[fi.path] = source
graph_builder.add_file(parsed)
graph_builder.build()
+ if parse_cache is not None:
+ parse_cache.save()
if skipped:
console.print(f"[yellow]Skipped {skipped} file(s) that failed to parse.[/yellow]")
@@ -393,10 +418,16 @@ def _rebuild_graph_and_git(
file_diffs: list,
cfg: dict,
exclude_patterns: list[str],
+ git_tier: str | None = None,
) -> tuple[list, dict[str, bytes], Any, Any, int, dict[str, dict]]:
"""Re-traverse + parse the repo, rebuild the graph (+ framework edges), and
re-index git metadata for the changed files.
+ ``git_tier`` is the persisted ``state.json:git_tier`` value: a fast-mode
+ (ESSENTIAL) repo must not pay per-file blame on every update for signals
+ its index never had. Unknown/missing values fall back to FULL, matching
+ the historical behavior for legacy state files.
+
Returns ``(parsed_files, source_map, graph_builder, repo_structure,
file_count, git_meta_map)``.
"""
@@ -409,7 +440,12 @@ def _rebuild_graph_and_git(
git_meta_map: dict[str, dict] = {}
try:
from repowise.core.ingestion.git_indexer import GitIndexer
+ from repowise.core.ingestion.git_indexer.tiers import GitIndexTier
+ try:
+ tier = GitIndexTier(git_tier) if git_tier else GitIndexTier.FULL
+ except ValueError:
+ tier = GitIndexTier.FULL
_commit_limit = cfg.get("commit_limit")
_follow_renames = cfg.get("follow_renames", False)
git_indexer = GitIndexer(
@@ -417,6 +453,7 @@ def _rebuild_graph_and_git(
commit_limit=_commit_limit,
follow_renames=_follow_renames,
exclude_patterns=exclude_patterns or None,
+ tier=tier,
)
changed_paths = _build_filtered_changed_paths(file_diffs, exclude_patterns)
updated_meta = run_async(git_indexer.index_changed_files(changed_paths))
@@ -1101,26 +1138,15 @@ def update_command(
console.print(f" [{color}]{fd.status:>10}[/{color}] {fd.path}")
# Re-parse changed files and rebuild graph for affected pages
- from repowise.core.generation import ContextAssembler, GenerationConfig, PageGenerator
-
cfg = load_config(repo_path)
- language = cfg.get("language", "en")
- # Config-driven (saved by `repowise init`); CLI override not surfaced
- # on update yet — defaults to on to keep the onboarding collection
- # fresh as the codebase evolves.
- enable_onboarding_cfg = bool(cfg.get("enable_onboarding", True))
- config = GenerationConfig(
- max_concurrency=concurrency,
- language=language,
- reasoning=resolve_reasoning(reasoning, cfg),
- enable_onboarding=enable_onboarding_cfg,
- )
# Read exclude patterns from config (set during init or via web UI)
exclude_patterns: list[str] = list(cfg.get("exclude_patterns") or [])
parsed_files, source_map, graph_builder, repo_structure, file_count, git_meta_map = (
- _rebuild_graph_and_git(repo_path, file_diffs, cfg, exclude_patterns)
+ _rebuild_graph_and_git(
+ repo_path, file_diffs, cfg, exclude_patterns, git_tier=state.get("git_tier")
+ )
)
# Determine affected pages (auto-scale budget if not explicitly set)
@@ -1164,6 +1190,23 @@ def update_command(
)
return
+ # The generation/LLM layer is only needed past this point — importing it
+ # above the index-only branch would make every index-only update (the
+ # post-commit hook's hot path) pay the import for code it never runs.
+ from repowise.core.generation import ContextAssembler, GenerationConfig, PageGenerator
+
+ language = cfg.get("language", "en")
+ # Config-driven (saved by `repowise init`); CLI override not surfaced
+ # on update yet — defaults to on to keep the onboarding collection
+ # fresh as the codebase evolves.
+ enable_onboarding_cfg = bool(cfg.get("enable_onboarding", True))
+ config = GenerationConfig(
+ max_concurrency=concurrency,
+ language=language,
+ reasoning=resolve_reasoning(reasoning, cfg),
+ enable_onboarding=enable_onboarding_cfg,
+ )
+
provider = resolve_provider(provider_name, model, repo_path=repo_path)
# Attach a DB-backed CostTracker so every LLM call made during this update
diff --git a/packages/core/src/repowise/core/generation/__init__.py b/packages/core/src/repowise/core/generation/__init__.py
index db00b92d2..0f891ca5f 100644
--- a/packages/core/src/repowise/core/generation/__init__.py
+++ b/packages/core/src/repowise/core/generation/__init__.py
@@ -5,105 +5,79 @@ Jinja2-templated prompts and BaseProvider.generate().
Import direction (strictly one-way):
ingestion.models ← generation.models ← context_assembler ← page_generator
+
+Exports resolve lazily (PEP 562): importing a light leaf module such as
+``generation.report`` from the index-only update path no longer pays for
+``context_assembler``/``page_generator``/``editor_files`` — they load on first
+attribute access instead.
"""
-from .context_assembler import (
- ApiContractContext,
- ArchitectureDiagramContext,
- ContextAssembler,
- FilePageContext,
- InfraPageContext,
- ModulePageContext,
- RepoOverviewContext,
- SccPageContext,
- SymbolSpotlightContext,
-)
-from .editor_files import (
- ClaudeMdGenerator,
- DecisionSummary,
- EditorFileData,
- EditorFileDataFetcher,
- HotspotFile,
- KeyModule,
- TechStackItem,
-)
-from .job_system import Checkpoint, JobStatus, JobSystem
-from .models import (
- GENERATION_LEVELS,
- ConfidenceDecayResult,
- DeadCodeConfig,
- FreshnessStatus,
- GeneratedPage,
- GenerationConfig,
- GitConfig,
- PageType,
- compute_confidence_decay_with_git,
- compute_freshness,
- compute_page_id,
- compute_source_hash,
- decay_confidence,
-)
-from .api_contract_detector import detect_code_api_contracts
-from .interlinking import (
- LinkIndex,
- WikiLink,
- attach_wiki_links_and_backlinks,
- resolve_wiki_links,
-)
-from .page_generator import SYSTEM_PROMPTS, PageGenerator
-from .selection import (
- BucketAllocation,
- ModuleGroup,
- Selection,
- SelectionInputs,
- select_pages,
- summarize_selection,
-)
+from importlib import import_module
+from typing import Any
+
+# Public name → defining submodule. Names import lazily on first access so
+# light consumers (e.g. the index-only update path importing ``.report``)
+# don't pull the heavy assembler/generator stack via this ``__init__``.
+_EXPORTS = {
+ "ApiContractContext": ".context_assembler",
+ "ArchitectureDiagramContext": ".context_assembler",
+ "ContextAssembler": ".context_assembler",
+ "FilePageContext": ".context_assembler",
+ "InfraPageContext": ".context_assembler",
+ "ModulePageContext": ".context_assembler",
+ "RepoOverviewContext": ".context_assembler",
+ "SccPageContext": ".context_assembler",
+ "SymbolSpotlightContext": ".context_assembler",
+ "ClaudeMdGenerator": ".editor_files",
+ "DecisionSummary": ".editor_files",
+ "EditorFileData": ".editor_files",
+ "EditorFileDataFetcher": ".editor_files",
+ "HotspotFile": ".editor_files",
+ "KeyModule": ".editor_files",
+ "TechStackItem": ".editor_files",
+ "Checkpoint": ".job_system",
+ "JobStatus": ".job_system",
+ "JobSystem": ".job_system",
+ "GENERATION_LEVELS": ".models",
+ "ConfidenceDecayResult": ".models",
+ "DeadCodeConfig": ".models",
+ "FreshnessStatus": ".models",
+ "GeneratedPage": ".models",
+ "GenerationConfig": ".models",
+ "GitConfig": ".models",
+ "PageType": ".models",
+ "compute_confidence_decay_with_git": ".models",
+ "compute_freshness": ".models",
+ "compute_page_id": ".models",
+ "compute_source_hash": ".models",
+ "decay_confidence": ".models",
+ "detect_code_api_contracts": ".api_contract_detector",
+ "LinkIndex": ".interlinking",
+ "WikiLink": ".interlinking",
+ "attach_wiki_links_and_backlinks": ".interlinking",
+ "resolve_wiki_links": ".interlinking",
+ "SYSTEM_PROMPTS": ".page_generator",
+ "PageGenerator": ".page_generator",
+ "BucketAllocation": ".selection",
+ "ModuleGroup": ".selection",
+ "Selection": ".selection",
+ "SelectionInputs": ".selection",
+ "select_pages": ".selection",
+ "summarize_selection": ".selection",
+}
+
+__all__ = sorted(_EXPORTS)
+
+
+def __getattr__(name: str) -> Any:
+ try:
+ module = _EXPORTS[name]
+ except KeyError:
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
+ value = getattr(import_module(module, __name__), name)
+ globals()[name] = value # cache: subsequent access skips __getattr__
+ return value
+
-__all__ = [
- "BucketAllocation",
- "GENERATION_LEVELS",
- "LinkIndex",
- "ModuleGroup",
- "SYSTEM_PROMPTS",
- "Selection",
- "SelectionInputs",
- "WikiLink",
- "attach_wiki_links_and_backlinks",
- "resolve_wiki_links",
- "select_pages",
- "summarize_selection",
- "ApiContractContext",
- "ArchitectureDiagramContext",
- "Checkpoint",
- "ClaudeMdGenerator",
- "ConfidenceDecayResult",
- "ContextAssembler",
- "DeadCodeConfig",
- "DecisionSummary",
- "EditorFileData",
- "EditorFileDataFetcher",
- "FilePageContext",
- "FreshnessStatus",
- "GeneratedPage",
- "GenerationConfig",
- "GitConfig",
- "HotspotFile",
- "InfraPageContext",
- "JobStatus",
- "JobSystem",
- "KeyModule",
- "ModulePageContext",
- "PageGenerator",
- "PageType",
- "RepoOverviewContext",
- "SccPageContext",
- "SymbolSpotlightContext",
- "TechStackItem",
- "compute_confidence_decay_with_git",
- "compute_freshness",
- "compute_page_id",
- "compute_source_hash",
- "decay_confidence",
- "detect_code_api_contracts",
-]
+def __dir__() -> list[str]:
+ return sorted(set(globals()) | set(_EXPORTS))
diff --git a/packages/core/src/repowise/core/ingestion/graph/_centrality_cache.py b/packages/core/src/repowise/core/ingestion/graph/_centrality_cache.py
new file mode 100644
index 000000000..4d64ce545
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/_centrality_cache.py
@@ -0,0 +1,113 @@
+"""Structure-keyed disk cache for betweenness centrality.
+
+Exact betweenness (Brandes) is the most expensive metric kernel on every
+ingest — and an incremental ``repowise update`` recomputes it for the whole
+graph even when the docstring-only edit it processed didn't change a single
+edge. Betweenness is a pure function of the *structure* of the (unweighted)
+subgraph it runs on, so values cache safely keyed by a signature over the
+sorted node and edge sets: content edits that don't move call/heritage/import
+edges hit; any structural change misses and recomputes.
+
+One entry per kind (``file`` / ``symbol``) — the cache answers "is the graph
+still the one I last scored?", not a history. Thread-safe because the init
+pipeline computes both kinds concurrently. Best-effort by design: any
+load/save error degrades to a fresh computation.
+
+Note for the >30k-node sampled path (``nx.betweenness_centrality(k=...)``):
+sampling is seedless, so recomputing the same graph yields slightly
+different values run to run. A cache hit returns the previously sampled
+values — deterministic where the status quo was not.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import os
+import pickle
+import tempfile
+import threading
+from pathlib import Path
+
+import structlog
+
+log = structlog.get_logger(__name__)
+
+_CACHE_VERSION = 1
+_CACHE_FILENAME = "centrality_cache.pkl"
+
+__all__ = ["CentralityCache", "subgraph_signature"]
+
+
+def subgraph_signature(graph) -> str:
+ """Hash the structure (sorted nodes + edges) of *graph*.
+
+ Attributes are deliberately excluded — betweenness on these subgraphs is
+ unweighted, and the subgraph constructors already filtered edges by type.
+ """
+ h = hashlib.sha256()
+ for node in sorted(graph.nodes()):
+ h.update(node.encode("utf-8", "replace"))
+ h.update(b"\x00")
+ h.update(b"\x01")
+ for u, v in sorted(graph.edges()):
+ h.update(u.encode("utf-8", "replace"))
+ h.update(b"\x00")
+ h.update(v.encode("utf-8", "replace"))
+ h.update(b"\x00")
+ return h.hexdigest()
+
+
+class CentralityCache:
+ """Pickle-backed ``kind -> (signature, values)`` store."""
+
+ def __init__(self, cache_dir: Path | str) -> None:
+ self._path = Path(cache_dir) / _CACHE_FILENAME
+ self._entries: dict[str, tuple[str, dict[str, float]]] = {}
+ self._lock = threading.Lock()
+ self._loaded = False
+
+ def _ensure_loaded(self) -> None:
+ if self._loaded:
+ return
+ self._loaded = True
+ try:
+ with self._path.open("rb") as fh:
+ payload = pickle.load(fh)
+ if payload.get("version") != _CACHE_VERSION:
+ return
+ self._entries = payload.get("entries", {})
+ except FileNotFoundError:
+ return
+ except Exception as exc: # corrupt / unreadable cache -> recompute
+ log.debug("centrality_cache_load_failed", error=str(exc))
+
+ def get(self, kind: str, signature: str) -> dict[str, float] | None:
+ with self._lock:
+ self._ensure_loaded()
+ entry = self._entries.get(kind)
+ if entry is None or entry[0] != signature:
+ return None
+ return dict(entry[1])
+
+ def put(self, kind: str, signature: str, values: dict[str, float]) -> None:
+ """Store *values* for *kind* and persist atomically."""
+ with self._lock:
+ self._ensure_loaded()
+ self._entries[kind] = (signature, dict(values))
+ try:
+ self._path.parent.mkdir(parents=True, exist_ok=True)
+ payload = {"version": _CACHE_VERSION, "entries": self._entries}
+ fd, tmp_name = tempfile.mkstemp(
+ dir=str(self._path.parent), prefix=_CACHE_FILENAME, suffix=".tmp"
+ )
+ try:
+ with os.fdopen(fd, "wb") as fh:
+ pickle.dump(payload, fh, protocol=pickle.HIGHEST_PROTOCOL)
+ os.replace(tmp_name, self._path)
+ except BaseException:
+ with contextlib.suppress(OSError):
+ os.unlink(tmp_name)
+ raise
+ except Exception as exc:
+ log.debug("centrality_cache_save_failed", error=str(exc))
diff --git a/packages/core/src/repowise/core/ingestion/graph/_metrics.py b/packages/core/src/repowise/core/ingestion/graph/_metrics.py
index 4cfbcdbd0..377c52f17 100644
--- a/packages/core/src/repowise/core/ingestion/graph/_metrics.py
+++ b/packages/core/src/repowise/core/ingestion/graph/_metrics.py
@@ -164,17 +164,10 @@ class MetricsMixin:
if self._betweenness_cache is not None:
return self._betweenness_cache
g = self.file_subgraph()
- n = g.number_of_nodes()
- if n == 0:
+ if g.number_of_nodes() == 0:
self._betweenness_cache = {}
return self._betweenness_cache
- if n > _LARGE_REPO_THRESHOLD:
- k = min(500, n)
- self._betweenness_cache = nx.betweenness_centrality(g, k=k, normalized=True)
- else:
- from ._betweenness import betweenness_centrality_fast
-
- self._betweenness_cache = betweenness_centrality_fast(g, normalized=True)
+ self._betweenness_cache = self._betweenness_with_disk_cache("file", g)
return self._betweenness_cache
def in_degree(self) -> dict[str, int]:
@@ -269,18 +262,51 @@ class MetricsMixin:
if self._symbol_betweenness_cache is not None:
return self._symbol_betweenness_cache
sub = self.symbol_subgraph()
- n = sub.number_of_nodes()
- if n == 0:
+ if sub.number_of_nodes() == 0:
self._symbol_betweenness_cache = {}
return self._symbol_betweenness_cache
+ self._symbol_betweenness_cache = self._betweenness_with_disk_cache("symbol", sub)
+ return self._symbol_betweenness_cache
+
+ def _betweenness_with_disk_cache(self, kind: str, g: nx.DiGraph) -> dict[str, float]:
+ """Compute betweenness for *g*, consulting the structure-keyed disk cache.
+
+ With a cache attached (see ``GraphBuilder(centrality_cache_dir=...)``)
+ and an unchanged subgraph structure, the previous run's values are
+ returned without re-running Brandes — the dominant metric cost of an
+ incremental update whose change didn't move any edges. Structural
+ changes, cache errors, or no cache all fall through to the exact
+ computation used before.
+ """
+ cache = getattr(self, "_centrality_cache", None)
+ signature: str | None = None
+ if cache is not None:
+ try:
+ from ._centrality_cache import subgraph_signature
+
+ signature = subgraph_signature(g)
+ hit = cache.get(kind, signature)
+ if hit is not None:
+ log.info("betweenness_reused_from_cache", kind=kind, nodes=len(hit))
+ return hit
+ except Exception as exc:
+ log.debug("centrality_cache_lookup_failed", kind=kind, error=str(exc))
+ signature = None
+
+ n = g.number_of_nodes()
if n > _LARGE_REPO_THRESHOLD:
k = min(500, n)
- self._symbol_betweenness_cache = nx.betweenness_centrality(sub, k=k, normalized=True)
+ values = nx.betweenness_centrality(g, k=k, normalized=True)
else:
from ._betweenness import betweenness_centrality_fast
- self._symbol_betweenness_cache = betweenness_centrality_fast(sub, normalized=True)
- return self._symbol_betweenness_cache
+ values = betweenness_centrality_fast(g, normalized=True)
+ if cache is not None and signature is not None:
+ try:
+ cache.put(kind, signature, values)
+ except Exception as exc:
+ log.debug("centrality_cache_store_failed", kind=kind, error=str(exc))
+ return values
# ------------------------------------------------------------------
# Execution flows + bulk priming
diff --git a/packages/core/src/repowise/core/ingestion/graph/builder.py b/packages/core/src/repowise/core/ingestion/graph/builder.py
index 692e49e64..862b5ef03 100644
--- a/packages/core/src/repowise/core/ingestion/graph/builder.py
+++ b/packages/core/src/repowise/core/ingestion/graph/builder.py
@@ -45,12 +45,24 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
repo_path: Path | str | None = None,
*,
exclude_patterns: list[str] | None = None,
+ centrality_cache_dir: Path | str | None = None,
) -> None:
self._graph: nx.DiGraph = nx.DiGraph()
self._parsed_files: dict[str, ParsedFile] = {} # path → ParsedFile
self._built = False
self._repo_path: Path | None = Path(repo_path) if repo_path else None
self._tsconfig_resolver: Any | None = None # TsconfigResolver (lazy import)
+ # Optional structure-keyed disk cache for betweenness (the most
+ # expensive metric kernel). Opt-in via *centrality_cache_dir* — the
+ # ingest paths pass ``<repo>/.repowise``; ad-hoc builders are unchanged.
+ self._centrality_cache: Any | None = None
+ if centrality_cache_dir is not None:
+ try:
+ from ._centrality_cache import CentralityCache
+
+ self._centrality_cache = CentralityCache(centrality_cache_dir)
+ except Exception:
+ self._centrality_cache = None
import pathspec
diff --git a/packages/core/src/repowise/core/ingestion/parse_cache.py b/packages/core/src/repowise/core/ingestion/parse_cache.py
new file mode 100644
index 000000000..aec7d78f6
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/parse_cache.py
@@ -0,0 +1,160 @@
+"""Content-hash keyed cache for ParsedFile results.
+
+Parsing is the repo-proportional term of every ingest: ``repowise update``
+re-parses the whole repo when one file changed, and large-repo inits pay
+minutes of tree-sitter time. A ParsedFile is a pure function of the file
+*bytes* and the parser itself, so it caches safely keyed by
+``(relative path, content hash)`` under a parser fingerprint (the compiled
+``.scm`` query sources + the package version) that invalidates the whole
+cache when extraction rules change.
+
+Entries are stored as pickle *bytes* snapshotted at parse time — downstream
+graph builds mutate ParsedFile in place (``Import.resolved_file``,
+``NamedBinding.source_file`` are back-filled during resolution), so a hit
+must deserialize a fresh object graph rather than share one. The path is
+part of the key because symbol IDs embed it; a rename is a miss and
+re-parses (producing IDs for the new path). On a hit the *current*
+traversal ``FileInfo`` is rebound so git hash / mtime stay fresh.
+
+Best-effort by design: any load/save/deserialize error degrades to a full
+parse. The cache file is rewritten with only the entries touched this run,
+so deleted files age out naturally.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import os
+import pickle
+import tempfile
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from .models import FileInfo, ParsedFile
+
+log = structlog.get_logger(__name__)
+
+_CACHE_VERSION = 1
+_CACHE_FILENAME = "parse_cache.pkl"
+
+__all__ = ["ParseCache", "parser_fingerprint"]
+
+
+@lru_cache(maxsize=1)
+def parser_fingerprint() -> str:
+ """Fingerprint of everything that determines a ParsedFile besides bytes.
+
+ Hashes every ``.scm`` query source (extraction rules) plus the package
+ version (parser/extractor code changes ship as releases). A mismatch
+ invalidates the whole cache — correctness over reuse.
+ """
+ h = hashlib.sha256()
+ h.update(f"cache-version:{_CACHE_VERSION}".encode())
+ try:
+ from repowise.core import __version__
+
+ h.update(f"version:{__version__}".encode())
+ except Exception:
+ h.update(b"version:unknown")
+ queries_dir = Path(__file__).parent / "queries"
+ try:
+ for scm in sorted(queries_dir.glob("*.scm")):
+ h.update(scm.name.encode())
+ h.update(scm.read_bytes())
+ except Exception:
+ h.update(b"queries:unreadable")
+ return h.hexdigest()
+
+
+class ParseCache:
+ """Pickle-backed ``(path, content_hash) -> ParsedFile-bytes`` store."""
+
+ def __init__(self, cache_dir: Path | str) -> None:
+ self._path = Path(cache_dir) / _CACHE_FILENAME
+ self._fingerprint = parser_fingerprint()
+ self._entries: dict[tuple[str, str], bytes] = {}
+ self._fresh: dict[tuple[str, str], bytes] = {}
+ self.hits = 0
+ self.misses = 0
+
+ # -- lifecycle ---------------------------------------------------------
+
+ def load(self) -> None:
+ try:
+ with self._path.open("rb") as fh:
+ payload = pickle.load(fh)
+ if (
+ payload.get("version") != _CACHE_VERSION
+ or payload.get("fingerprint") != self._fingerprint
+ ):
+ return
+ self._entries = payload.get("files", {})
+ except FileNotFoundError:
+ return
+ except Exception as exc: # corrupt / unreadable cache -> full parse
+ log.debug("parse_cache_load_failed", error=str(exc))
+
+ def save(self) -> None:
+ """Atomically persist the entries used or created this run."""
+ try:
+ self._path.parent.mkdir(parents=True, exist_ok=True)
+ payload = {
+ "version": _CACHE_VERSION,
+ "fingerprint": self._fingerprint,
+ "files": self._fresh,
+ }
+ fd, tmp_name = tempfile.mkstemp(
+ dir=str(self._path.parent), prefix=_CACHE_FILENAME, suffix=".tmp"
+ )
+ try:
+ with os.fdopen(fd, "wb") as fh:
+ pickle.dump(payload, fh, protocol=pickle.HIGHEST_PROTOCOL)
+ os.replace(tmp_name, self._path)
+ except BaseException:
+ with contextlib.suppress(OSError):
+ os.unlink(tmp_name)
+ raise
+ except Exception as exc:
+ log.debug("parse_cache_save_failed", error=str(exc))
+
+ # -- access ------------------------------------------------------------
+
+ def get(self, file_info: FileInfo, content_hash: str) -> ParsedFile | None:
+ """Return a fresh ParsedFile for *content_hash* or None on miss.
+
+ Hits deserialize a new object graph (downstream resolution mutates
+ ParsedFile in place) and rebind the current *file_info* so traversal
+ metadata (git hash, mtime) stays current. *content_hash* is
+ :func:`compute_content_hash` of the raw bytes — callers compute it
+ once and reuse it for the matching :meth:`put`.
+ """
+ key = (file_info.path, content_hash)
+ blob = self._entries.get(key)
+ if blob is None:
+ self.misses += 1
+ return None
+ try:
+ parsed: Any = pickle.loads(blob)
+ except Exception as exc: # corrupt entry -> re-parse this file
+ log.debug("parse_cache_entry_corrupt", path=file_info.path, error=str(exc))
+ self.misses += 1
+ return None
+ self.hits += 1
+ self._fresh[key] = blob
+ parsed.file_info = file_info
+ return parsed
+
+ def put(self, parsed: ParsedFile, content_hash: str) -> None:
+ """Snapshot *parsed* (call before any graph build mutates it)."""
+ try:
+ blob = pickle.dumps(parsed, protocol=pickle.HIGHEST_PROTOCOL)
+ except Exception as exc:
+ log.debug("parse_cache_put_failed", path=parsed.file_info.path, error=str(exc))
+ return
+ key = (parsed.file_info.path, content_hash)
+ self._entries[key] = blob
+ self._fresh[key] = blob
diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
index c6191932d..525a5a362 100644
--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py
+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
@@ -7,6 +7,7 @@ orchestrator.py) imports these phase functions. No CLI/click/rich imports.
from __future__ import annotations
import asyncio
+import contextlib
import os
import time
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
@@ -85,6 +86,57 @@ def _parse_one(path_and_fi_and_bytes: tuple) -> Any:
return (fi.abs_path, str(exc))
+def _split_cached(
+ repo_path: Path,
+ fi_and_bytes: list[tuple],
+ progress: ProgressCallback | None,
+) -> tuple[Any, dict[int, Any], list[tuple[int, tuple, str]]]:
+ """Partition pre-read files into parse-cache hits and misses.
+
+ Returns ``(cache, hits, misses)`` where *hits* maps the position in
+ *fi_and_bytes* to a cached ParsedFile and *misses* keeps ``(position,
+ (FileInfo, bytes), content_hash)`` triples that still need a real parse
+ (the hash rides along so the post-parse ``put`` doesn't re-digest).
+ Positions let the caller reassemble results in traversal order. Hits
+ tick the parse progress bar here (misses tick when their worker
+ finishes). Cache failures degrade to all-miss.
+ """
+ from repowise.core.ingestion import compute_content_hash
+ from repowise.core.ingestion.parse_cache import ParseCache
+
+ cache = None
+ hits: dict[int, Any] = {}
+ misses: list[tuple[int, tuple, str]] = []
+ try:
+ cache = ParseCache(repo_path / ".repowise")
+ cache.load()
+ except Exception as exc:
+ logger.debug("parse_cache_init_failed", error=str(exc))
+ return None, hits, [(idx, item, "") for idx, item in enumerate(fi_and_bytes)]
+
+ for idx, (fi, source) in enumerate(fi_and_bytes):
+ content_hash = compute_content_hash(source)
+ parsed = cache.get(fi, content_hash)
+ if parsed is not None:
+ hits[idx] = parsed
+ if progress:
+ progress.on_item_done("parse")
+ else:
+ misses.append((idx, (fi, source), content_hash))
+ if hits:
+ logger.info("parse_cache_used", hits=cache.hits, misses=cache.misses)
+ return cache, hits, misses
+
+
+def _cache_parsed(cache: Any, parsed: Any, content_hash: str) -> None:
+ """Best-effort put of a freshly parsed file into the parse cache."""
+ if cache is None or not content_hash:
+ return
+ # Cache failures must never fail the parse phase.
+ with contextlib.suppress(Exception):
+ cache.put(parsed, content_hash)
+
+
def _read_sources(
file_infos: list[Any],
progress: ProgressCallback | None,
@@ -190,57 +242,76 @@ async def _run_ingestion(
parsed_files: list[Any] = []
source_map: dict[str, bytes] = {}
- graph_builder = GraphBuilder(repo_path=repo_path, exclude_patterns=exclude_patterns)
+ graph_builder = GraphBuilder(
+ repo_path=repo_path,
+ exclude_patterns=exclude_patterns,
+ centrality_cache_dir=repo_path / ".repowise",
+ )
loop = asyncio.get_running_loop()
- workers = max(1, os.cpu_count() or 4)
- _use_process_pool = True
+ # Content-hash parse cache: unchanged files skip the worker pool
+ # entirely (an incremental update re-ingests the whole repo for one
+ # changed file). Misses parse exactly as before.
+ parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)
+ workers = max(1, min(os.cpu_count() or 4, len(to_parse) or 1))
+
parse_results: list[Any] = []
- try:
- with ProcessPoolExecutor(max_workers=workers) as pool:
- tasks = [loop.run_in_executor(pool, _parse_one, item) for item in fi_and_bytes]
- # Tick the parse-progress bar as each worker finishes —
- # ``asyncio.gather`` would otherwise hold every event back
- # until the last file is done, which on PowerToys-scale
- # repos looked like a hang at ``0/N`` for many minutes.
- # Per-task done-callbacks fire on the event loop thread and
- # preserve gather's ordered results, so the aggregation
- # loop below still indexes ``fi_and_bytes`` correctly.
- if progress is not None:
- _parse_tick = lambda _fut: progress.on_item_done("parse") # noqa: E731
- for fut in tasks:
- fut.add_done_callback(_parse_tick)
- parse_results = await asyncio.gather(*tasks, return_exceptions=True)
- except Exception as pool_exc:
- logger.warning(
- "process_pool_parse_failed_falling_back",
- error=str(pool_exc),
- )
- _use_process_pool = False
- # Fallback: in-process sequential parse
- _fallback_parser = ASTParser()
- for i, (fi, source) in enumerate(fi_and_bytes):
- try:
- result = _fallback_parser.parse_file(fi, source)
- parse_results.append(result)
- except Exception as exc:
- parse_results.append((fi.abs_path, str(exc)))
- if progress:
- progress.on_item_done("parse")
- if i % 50 == 49:
- await asyncio.sleep(0)
+ if to_parse:
+ try:
+ with ProcessPoolExecutor(max_workers=workers) as pool:
+ tasks = [
+ loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse
+ ]
+ # Tick the parse-progress bar as each worker finishes —
+ # ``asyncio.gather`` would otherwise hold every event back
+ # until the last file is done, which on PowerToys-scale
+ # repos looked like a hang at ``0/N`` for many minutes.
+ # Per-task done-callbacks fire on the event loop thread and
+ # preserve gather's ordered results, so the aggregation
+ # loop below still indexes ``to_parse`` correctly.
+ if progress is not None:
+ _parse_tick = lambda _fut: progress.on_item_done("parse") # noqa: E731
+ for fut in tasks:
+ fut.add_done_callback(_parse_tick)
+ parse_results = await asyncio.gather(*tasks, return_exceptions=True)
+ except Exception as pool_exc:
+ logger.warning(
+ "process_pool_parse_failed_falling_back",
+ error=str(pool_exc),
+ )
+ # Fallback: in-process sequential parse
+ parse_results = []
+ _fallback_parser = ASTParser()
+ for i, (_idx, (fi, source), _h) in enumerate(to_parse):
+ try:
+ result = _fallback_parser.parse_file(fi, source)
+ parse_results.append(result)
+ except Exception as exc:
+ parse_results.append((fi.abs_path, str(exc)))
+ if progress:
+ progress.on_item_done("parse")
+ if i % 50 == 49:
+ await asyncio.sleep(0)
+
+ # Merge fresh parses back into traversal order alongside cache hits.
+ merged: dict[int, Any] = dict(cached_hits)
+ for (idx, _item, content_hash), result in zip(to_parse, parse_results, strict=True):
+ merged[idx] = result
+ if not isinstance(result, tuple | Exception):
+ _cache_parsed(parse_cache, result, content_hash)
# Aggregate results into GraphBuilder on the main loop (not thread-safe).
- for idx, result in enumerate(parse_results):
+ for idx in range(len(fi_and_bytes)):
fi, source = fi_and_bytes[idx]
+ result = merged.get(idx)
if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], str):
# Error tuple: (abs_path_str, error_str)
logger.debug("parse_error_in_worker", path=result[0], error=result[1])
elif isinstance(result, Exception):
logger.debug("parse_exception_in_worker", path=fi.abs_path, error=str(result))
- else:
+ elif result is not None:
parsed_files.append(result)
source_map[fi.path] = source
graph_builder.add_file(result)
@@ -248,6 +319,8 @@ async def _run_ingestion(
# attached above; only the fallback path ticks here (handled in
# its own loop). No tick needed in aggregation.
+ if parse_cache is not None:
+ parse_cache.save()
_phase_done(progress, "parse")
# ---- tsconfig path-alias resolver (before graph build) ------------------
@@ -460,39 +533,55 @@ async def reparse_for_resume(
parsed_files: list[Any] = []
source_map: dict[str, bytes] = {}
loop = asyncio.get_running_loop()
- workers = max(1, os.cpu_count() or 4)
- parse_results: list[Any] = []
- try:
- with ProcessPoolExecutor(max_workers=workers) as pool:
- tasks = [loop.run_in_executor(pool, _parse_one, item) for item in fi_and_bytes]
- if progress is not None:
- _tick = lambda _fut: progress.on_item_done("parse") # noqa: E731
- for fut in tasks:
- fut.add_done_callback(_tick)
- parse_results = await asyncio.gather(*tasks, return_exceptions=True)
- except Exception as pool_exc:
- logger.warning("resume_reparse_pool_failed_falling_back", error=str(pool_exc))
- _fallback_parser = ASTParser()
- for i, (fi, source) in enumerate(fi_and_bytes):
- try:
- parse_results.append(_fallback_parser.parse_file(fi, source))
- except Exception as exc:
- parse_results.append((fi.abs_path, str(exc)))
- if progress:
- progress.on_item_done("parse")
- if i % 50 == 49:
- await asyncio.sleep(0)
+ parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)
+ workers = max(1, min(os.cpu_count() or 4, len(to_parse) or 1))
+ parse_results: list[Any] = []
- for idx, result in enumerate(parse_results):
+ if to_parse:
+ try:
+ with ProcessPoolExecutor(max_workers=workers) as pool:
+ tasks = [
+ loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse
+ ]
+ if progress is not None:
+ _tick = lambda _fut: progress.on_item_done("parse") # noqa: E731
+ for fut in tasks:
+ fut.add_done_callback(_tick)
+ parse_results = await asyncio.gather(*tasks, return_exceptions=True)
+ except Exception as pool_exc:
+ logger.warning("resume_reparse_pool_failed_falling_back", error=str(pool_exc))
+ parse_results = []
+ _fallback_parser = ASTParser()
+ for i, (_idx, (fi, source), _h) in enumerate(to_parse):
+ try:
+ parse_results.append(_fallback_parser.parse_file(fi, source))
+ except Exception as exc:
+ parse_results.append((fi.abs_path, str(exc)))
+ if progress:
+ progress.on_item_done("parse")
+ if i % 50 == 49:
+ await asyncio.sleep(0)
+
+ merged: dict[int, Any] = dict(cached_hits)
+ for (idx, _item, content_hash), result in zip(to_parse, parse_results, strict=True):
+ merged[idx] = result
+ if not isinstance(result, tuple | Exception):
+ _cache_parsed(parse_cache, result, content_hash)
+
+ for idx in range(len(fi_and_bytes)):
fi, source = fi_and_bytes[idx]
+ result = merged.get(idx)
if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], str):
logger.debug("parse_error_in_worker", path=result[0], error=result[1])
elif isinstance(result, Exception):
logger.debug("parse_exception_in_worker", path=fi.abs_path, error=str(result))
- else:
+ elif result is not None:
parsed_files.append(result)
source_map[fi.path] = source
+
+ if parse_cache is not None:
+ parse_cache.save()
_phase_done(progress, "parse")
tech_items: list = []
diff --git a/tests/unit/cli/test_update_git_tier.py b/tests/unit/cli/test_update_git_tier.py
new file mode 100644
index 000000000..5cf197cb6
--- /dev/null
+++ b/tests/unit/cli/test_update_git_tier.py
@@ -0,0 +1,95 @@
+"""`repowise update` must honor the persisted git tier.
+
+A fast-mode (ESSENTIAL-tier) repo records ``git_tier: essential`` in
+state.json; its updates must not pay per-file ``git blame`` for signals the
+index never had. ``_rebuild_graph_and_git`` previously constructed GitIndexer
+without a tier — silently FULL for every repo.
+"""
+
+from __future__ import annotations
+
+from typing import ClassVar
+
+import pytest
+
+from repowise.cli.commands.update_cmd import _rebuild_graph_and_git
+from repowise.core.ingestion.git_indexer.tiers import GitIndexTier
+
+
+def _init_repo(tmp_path):
+ import git as gitpython
+
+ repo = gitpython.Repo.init(tmp_path)
+ with repo.config_writer() as cw:
+ cw.set_value("user", "name", "Alice")
+ cw.set_value("user", "email", "[email protected]")
+ (tmp_path / "a.py").write_text("x = 1\n")
+ repo.index.add(["a.py"])
+ repo.index.commit("feat: add module a")
+ repo.close()
+
+
+class _RecordingIndexer:
+ """Stands in for GitIndexer; records the tier the CLI passed."""
+
+ instances: ClassVar[list] = []
+
+ def __init__(self, repo_path, **kwargs):
+ self.kwargs = kwargs
+ _RecordingIndexer.instances.append(self)
+
+ async def index_changed_files(self, changed_paths):
+ return []
+
+
[email protected](autouse=True)
+def _reset_recorder():
+ _RecordingIndexer.instances = []
+
+
+ ("state_tier", "expected"),
+ [
+ ("essential", GitIndexTier.ESSENTIAL),
+ ("full", GitIndexTier.FULL),
+ (None, GitIndexTier.FULL), # legacy state without git_tier
+ ("bogus-tier", GitIndexTier.FULL), # corrupt value falls back
+ ],
+)
+def test_rebuild_threads_state_git_tier(tmp_path, monkeypatch, state_tier, expected):
+ _init_repo(tmp_path)
+ monkeypatch.setattr(
+ "repowise.core.ingestion.git_indexer.GitIndexer", _RecordingIndexer
+ )
+
+ _rebuild_graph_and_git(tmp_path, [], {}, [], git_tier=state_tier)
+
+ assert len(_RecordingIndexer.instances) == 1
+ assert _RecordingIndexer.instances[0].kwargs["tier"] is expected
+
+
+async def test_essential_tier_update_never_blames(tmp_path, monkeypatch):
+ """End-to-end through the real GitIndexer: ESSENTIAL updates skip blame."""
+ from repowise.core.ingestion.git_indexer import GitIndexer
+
+ _init_repo(tmp_path)
+ calls: list[str] = []
+
+ def _no_blame(*args, **kwargs): # pragma: no cover - failure path
+ calls.append("blame")
+ raise AssertionError("blame must not run on ESSENTIAL updates")
+
+ monkeypatch.setattr(
+ "repowise.core.ingestion.git_indexer.file_history.build_blame_index", _no_blame
+ )
+ monkeypatch.setattr(
+ "repowise.core.ingestion.git_indexer.file_history.get_blame_ownership", _no_blame
+ )
+
+ indexer = GitIndexer(tmp_path, tier=GitIndexTier.ESSENTIAL)
+ meta = await indexer.index_changed_files(["a.py"])
+
+ assert calls == []
+ assert meta and meta[0]["file_path"] == "a.py"
+ # Ownership still present via the commit-author fallback.
+ assert meta[0].get("primary_owner_name") == "Alice"
diff --git a/tests/unit/generation/test_lazy_generation_exports.py b/tests/unit/generation/test_lazy_generation_exports.py
new file mode 100644
index 000000000..644ff18f4
--- /dev/null
+++ b/tests/unit/generation/test_lazy_generation_exports.py
@@ -0,0 +1,54 @@
+"""Guard tests for the lazy (PEP 562) ``repowise.core.generation`` exports.
+
+The index-only update path imports light leaves (``generation.report``) and
+must not pay for the assembler/generator stack via the package ``__init__``.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+
+import pytest
+
+import repowise.core.generation as generation
+
+
+def test_all_exports_resolve():
+ for name in generation.__all__:
+ assert getattr(generation, name) is not None, name
+
+
+def test_dir_includes_lazy_exports():
+ listing = dir(generation)
+ assert "PageGenerator" in listing
+ assert "GenerationConfig" in listing
+
+
+def test_unknown_attribute_raises():
+ with pytest.raises(AttributeError, match="does_not_exist"):
+ _ = generation.does_not_exist
+
+
+def test_light_leaf_import_does_not_load_heavy_stack():
+ """``generation.report`` (index-only update path) must not pull the
+ assembler/generator/editor_files stack through the package ``__init__``."""
+ code = (
+ "import sys; import repowise.core.generation.report; "
+ "heavy = [m for m in sys.modules if m.startswith('repowise.core.generation.') "
+ "and m.split('.')[3] in ('context_assembler', 'page_generator', 'editor_files')]; "
+ "sys.exit(1 if heavy else 0)"
+ )
+ result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
+ assert result.returncode == 0, result.stderr
+
+
+def test_lazy_attribute_access_imports_on_demand():
+ code = (
+ "from repowise.core.generation import PageGenerator; "
+ "import sys; "
+ "assert 'repowise.core.generation.page_generator' in sys.modules; "
+ "assert PageGenerator.__name__ == 'PageGenerator'"
+ )
+ result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
+ assert result.returncode == 0, result.stderr
diff --git a/tests/unit/ingestion/test_centrality_cache.py b/tests/unit/ingestion/test_centrality_cache.py
new file mode 100644
index 000000000..5c9f85afd
--- /dev/null
+++ b/tests/unit/ingestion/test_centrality_cache.py
@@ -0,0 +1,133 @@
+"""Structure-keyed betweenness cache: reuse iff the subgraph is unchanged.
+
+A content edit that doesn't move call/heritage/import edges must reuse the
+previous run's betweenness values exactly; any structural change must
+recompute. No cache dir -> behavior (and filesystem) unchanged.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from repowise.core.ingestion import ASTParser, GraphBuilder
+from repowise.core.ingestion.graph._centrality_cache import (
+ CentralityCache,
+ subgraph_signature,
+)
+from repowise.core.ingestion.models import FileInfo
+
+_MAIN = "from util import helper\n\n\ndef main():\n return helper(2)\n"
+_UTIL = "def helper(x):\n return x + 1\n"
+
+
+def _parse(path: str, source: str):
+ fi = FileInfo(
+ path=path,
+ abs_path=f"C:/fake/{path}",
+ language="python",
+ size_bytes=len(source),
+ git_hash="",
+ last_modified=datetime.now(UTC),
+ is_test=False,
+ is_config=False,
+ is_api_contract=False,
+ is_entry_point=False,
+ )
+ return ASTParser().parse_file(fi, source.encode())
+
+
+def _build(cache_dir, files):
+ gb = GraphBuilder("C:/fake", centrality_cache_dir=cache_dir)
+ for path, source in files:
+ gb.add_file(_parse(path, source))
+ gb.build()
+ return gb
+
+
+_FILES = [("main.py", _MAIN), ("util.py", _UTIL)]
+
+
+def test_unchanged_structure_reuses_values(tmp_path, monkeypatch):
+ gb1 = _build(tmp_path, _FILES)
+ file_bc = gb1.betweenness_centrality()
+ sym_bc = gb1.symbol_betweenness_centrality()
+ assert (tmp_path / "centrality_cache.pkl").exists()
+
+ def _boom(*args, **kwargs):
+ raise AssertionError("betweenness must not recompute on an unchanged graph")
+
+ monkeypatch.setattr(
+ "repowise.core.ingestion.graph._betweenness.betweenness_centrality_fast", _boom
+ )
+ gb2 = _build(tmp_path, _FILES)
+ assert gb2.betweenness_centrality() == file_bc
+ assert gb2.symbol_betweenness_centrality() == sym_bc
+
+
+def test_structural_change_recomputes_and_matches_fresh(tmp_path):
+ _build(tmp_path, _FILES).symbol_betweenness_centrality()
+
+ edited = _MAIN + "\n\ndef extra():\n return main()\n"
+ cached_run = _build(tmp_path, [("main.py", edited), ("util.py", _UTIL)])
+ fresh_run = _build(None, [("main.py", edited), ("util.py", _UTIL)])
+
+ assert cached_run.symbol_betweenness_centrality() == fresh_run.symbol_betweenness_centrality()
+ assert cached_run.betweenness_centrality() == fresh_run.betweenness_centrality()
+
+
+def test_no_cache_dir_writes_nothing(tmp_path):
+ gb = _build(None, _FILES)
+ gb.betweenness_centrality()
+ gb.symbol_betweenness_centrality()
+ assert not (tmp_path / "centrality_cache.pkl").exists()
+
+
+def test_corrupt_cache_recomputes(tmp_path):
+ _build(tmp_path, _FILES).symbol_betweenness_centrality()
+ (tmp_path / "centrality_cache.pkl").write_bytes(b"\x00garbage")
+
+ cached_run = _build(tmp_path, _FILES)
+ fresh_run = _build(None, _FILES)
+ assert cached_run.symbol_betweenness_centrality() == fresh_run.symbol_betweenness_centrality()
+
+
+def test_signature_is_structure_only():
+ import networkx as nx
+
+ g1 = nx.DiGraph()
+ g1.add_edge("a", "b")
+ g1.add_node("c")
+
+ g2 = nx.DiGraph() # same structure, different insertion order + attrs
+ g2.add_node("c")
+ g2.add_edge("a", "b", edge_type="calls")
+ g2.nodes["a"]["node_type"] = "symbol"
+
+ assert subgraph_signature(g1) == subgraph_signature(g2)
+
+ g2.add_edge("b", "c")
+ assert subgraph_signature(g1) != subgraph_signature(g2)
+
+
+def test_signature_mismatch_returns_none(tmp_path):
+ cache = CentralityCache(tmp_path)
+ cache.put("symbol", "sig-1", {"n": 0.5})
+ assert cache.get("symbol", "sig-1") == {"n": 0.5}
+ assert cache.get("symbol", "sig-2") is None
+ assert cache.get("file", "sig-1") is None
+
+ # A fresh instance reads back from disk.
+ cache2 = CentralityCache(tmp_path)
+ assert cache2.get("symbol", "sig-1") == {"n": 0.5}
+
+
+async def test_compute_metrics_parallel_with_cache(tmp_path):
+ """Both kinds computed concurrently must land in one cache file."""
+ gb = _build(tmp_path, _FILES)
+ await gb.compute_metrics_parallel()
+
+ cache = CentralityCache(tmp_path)
+ file_sig = subgraph_signature(gb.file_subgraph())
+ sym_sig = subgraph_signature(gb.symbol_subgraph())
+ assert cache.get("file", file_sig) == gb.betweenness_centrality()
+ assert cache.get("symbol", sym_sig) == gb.symbol_betweenness_centrality()
diff --git a/tests/unit/ingestion/test_parse_cache.py b/tests/unit/ingestion/test_parse_cache.py
new file mode 100644
index 000000000..c85c153b0
--- /dev/null
+++ b/tests/unit/ingestion/test_parse_cache.py
@@ -0,0 +1,198 @@
+"""Parse-cache equivalence: cached ingest must match a fresh parse exactly.
+
+The cache snapshots ParsedFiles at parse time keyed by (path, content hash)
+under a parser fingerprint. Every scenario asserts the cached run is
+ParsedFile-equal AND graph-isomorphic to a from-scratch build — after file
+edit, add, delete, rename, and an import-changing edit. Corrupt or
+stale-fingerprint caches must degrade silently to a full parse.
+"""
+
+from __future__ import annotations
+
+import pickle
+
+import networkx as nx
+import pytest
+
+from repowise.cli.commands.update_cmd import _build_repo_graph
+from repowise.core.ingestion import compute_content_hash
+from repowise.core.ingestion.parse_cache import ParseCache, parser_fingerprint
+
+
+def _write_fixture(repo) -> None:
+ (repo / "util.py").write_text(
+ "def helper(x):\n return x + 1\n",
+ encoding="utf-8",
+ )
+ (repo / "main.py").write_text(
+ "from util import helper\n\n\ndef main():\n return helper(2)\n",
+ encoding="utf-8",
+ )
+ (repo / "other.py").write_text(
+ "VALUE = 3\n",
+ encoding="utf-8",
+ )
+
+
+def _cache_path(repo) -> object:
+ return repo / ".repowise" / "parse_cache.pkl"
+
+
+def _build(repo):
+ """Run the update path's full ingest; returns (parsed_files, graph)."""
+ parsed, _src, gb, _structure, _count = _build_repo_graph(repo, [], collect_sources=False)
+ return sorted(parsed, key=lambda p: p.file_info.path), gb.graph()
+
+
+def _fresh_build(repo):
+ """Ground truth: same build with the cache file removed."""
+ cache_file = _cache_path(repo)
+ if cache_file.exists():
+ cache_file.unlink()
+ return _build(repo)
+
+
+def _assert_equivalent(repo) -> None:
+ """Cached run == from-scratch run: ParsedFile-equal and graph-isomorphic."""
+ cached_parsed, cached_graph = _build(repo) # warm cache from prior run
+ fresh_parsed, fresh_graph = _fresh_build(repo)
+
+ assert [p.file_info.path for p in cached_parsed] == [p.file_info.path for p in fresh_parsed]
+ for cp, fp in zip(cached_parsed, fresh_parsed, strict=True):
+ assert cp == fp, cp.file_info.path
+
+ assert nx.utils.graphs_equal(cached_graph, fresh_graph)
+
+
+def repo(tmp_path):
+ _write_fixture(tmp_path)
+ (tmp_path / ".repowise").mkdir()
+ return tmp_path
+
+
+def test_warm_cache_equivalent_unchanged(repo):
+ _build(repo) # cold run populates the cache
+ _assert_equivalent(repo)
+
+
+def test_equivalent_after_edit(repo):
+ _build(repo)
+ (repo / "util.py").write_text(
+ "def helper(x):\n return x * 2\n\n\ndef extra(y):\n return y\n",
+ encoding="utf-8",
+ )
+ _assert_equivalent(repo)
+
+
+def test_equivalent_after_add(repo):
+ _build(repo)
+ (repo / "added.py").write_text("def added_fn():\n return 9\n", encoding="utf-8")
+ _assert_equivalent(repo)
+
+
+def test_equivalent_after_delete(repo):
+ _build(repo)
+ (repo / "other.py").unlink()
+ _assert_equivalent(repo)
+
+
+def test_equivalent_after_rename(repo):
+ _build(repo)
+ (repo / "other.py").rename(repo / "renamed.py")
+ _assert_equivalent(repo)
+ # Symbol IDs embed the path — the renamed file must carry the new path.
+ parsed, _ = _build(repo)
+ by_path = {p.file_info.path: p for p in parsed}
+ assert "renamed.py" in by_path
+ assert "other.py" not in by_path
+
+
+def test_equivalent_after_import_changing_edit(repo):
+ _build(repo)
+ (repo / "main.py").write_text(
+ "from other import VALUE\n\n\ndef main():\n return VALUE\n",
+ encoding="utf-8",
+ )
+ _assert_equivalent(repo)
+
+
+def test_corrupt_cache_falls_back_to_full_parse(repo):
+ _build(repo)
+ _cache_path(repo).write_bytes(b"\x00not a pickle")
+ cached_parsed, cached_graph = _build(repo)
+ fresh_parsed, fresh_graph = _fresh_build(repo)
+ for cp, fp in zip(cached_parsed, fresh_parsed, strict=True):
+ assert cp == fp
+ assert nx.utils.graphs_equal(cached_graph, fresh_graph)
+
+
+def test_fingerprint_mismatch_invalidates(repo, tmp_path):
+ _build(repo)
+ cache_file = _cache_path(repo)
+ payload = pickle.loads(cache_file.read_bytes())
+ assert payload["fingerprint"] == parser_fingerprint()
+ payload["fingerprint"] = "stale-fingerprint"
+ cache_file.write_bytes(pickle.dumps(payload))
+
+ cache = ParseCache(repo / ".repowise")
+ cache.load()
+ fi = _make_file_info(repo / "other.py", "other.py")
+ source = (repo / "other.py").read_bytes()
+ assert cache.get(fi, compute_content_hash(source)) is None
+ assert cache.misses == 1
+
+
+def _make_file_info(abs_path, rel_path):
+ from datetime import UTC, datetime
+
+ from repowise.core.ingestion.models import FileInfo
+
+ return FileInfo(
+ path=rel_path,
+ abs_path=str(abs_path),
+ language="python",
+ size_bytes=abs_path.stat().st_size,
+ git_hash="",
+ last_modified=datetime.now(UTC),
+ is_test=False,
+ is_config=False,
+ is_api_contract=False,
+ is_entry_point=False,
+ )
+
+
+def test_hit_rebinds_fresh_file_info_and_isolates_mutations(repo):
+ from repowise.core.ingestion import ASTParser
+
+ fi = _make_file_info(repo / "main.py", "main.py")
+ source = (repo / "main.py").read_bytes()
+ chash = compute_content_hash(source)
+ parsed = ASTParser().parse_file(fi, source)
+
+ cache = ParseCache(repo / ".repowise")
+ cache.put(parsed, chash)
+
+ fresh_fi = _make_file_info(repo / "main.py", "main.py")
+ fresh_fi.git_hash = "abc123"
+ hit1 = cache.get(fresh_fi, chash)
+ hit2 = cache.get(fresh_fi, chash)
+
+ # Fresh FileInfo rebound on the hit.
+ assert hit1.file_info is fresh_fi
+ assert hit1.file_info.git_hash == "abc123"
+ # Distinct object graphs: graph-build mutations cannot leak between runs.
+ assert hit1 is not hit2 and hit1 is not parsed
+ assert hit1.imports and hit1.imports[0].resolved_file is None
+ hit1.imports[0].resolved_file = "mutated.py"
+ assert hit2.imports[0].resolved_file is None
+
+
+def test_deleted_files_age_out_of_cache(repo):
+ _build(repo)
+ (repo / "other.py").unlink()
+ _build(repo) # rewrite keeps only entries touched this run
+ payload = pickle.loads(_cache_path(repo).read_bytes())
+ cached_paths = {path for path, _hash in payload["files"]}
+ assert "other.py" not in cached_paths
+ assert "main.py" in cached_paths
diff --git a/tests/unit/pipeline/test_ingestion_parse_cache.py b/tests/unit/pipeline/test_ingestion_parse_cache.py
new file mode 100644
index 000000000..2ee2cb2f9
--- /dev/null
+++ b/tests/unit/pipeline/test_ingestion_parse_cache.py
@@ -0,0 +1,64 @@
+"""Init-pipeline parse-cache wiring: warm-cache ingest == cold ingest.
+
+`_run_ingestion` splits pre-read files into cache hits (skip the worker
+pool) and misses (parse exactly as before), then merges back in traversal
+order. The warm run must produce identical ParsedFiles, source maps, and
+graphs — including with a stale cache after edits/adds/deletes.
+"""
+
+from __future__ import annotations
+
+import networkx as nx
+
+from repowise.core.pipeline.phases import ingestion as ingestion_phase
+
+
+def _write_fixture(repo) -> None:
+ (repo / "util.py").write_text("def helper(x):\n return x + 1\n", encoding="utf-8")
+ (repo / "main.py").write_text(
+ "from util import helper\n\n\ndef main():\n return helper(2)\n",
+ encoding="utf-8",
+ )
+
+
+async def _ingest(repo):
+ parsed, _fis, _structure, source_map, gb, _stats, _tech = (
+ await ingestion_phase._run_ingestion(
+ repo,
+ exclude_patterns=None,
+ skip_tests=False,
+ skip_infra=False,
+ progress=None,
+ )
+ )
+ return sorted(parsed, key=lambda p: p.file_info.path), source_map, gb.graph()
+
+
+async def _fresh_ingest(repo):
+ cache_file = repo / ".repowise" / "parse_cache.pkl"
+ if cache_file.exists():
+ cache_file.unlink()
+ return await _ingest(repo)
+
+
+async def test_run_ingestion_warm_cache_equivalent(tmp_path):
+ _write_fixture(tmp_path)
+
+ await _ingest(tmp_path) # cold: populates cache
+ assert (tmp_path / ".repowise" / "parse_cache.pkl").exists()
+
+ # Mixed hit/miss: one edit + one add + one delete against the warm cache.
+ (tmp_path / "util.py").write_text(
+ "def helper(x):\n return x * 3\n",
+ encoding="utf-8",
+ )
+ (tmp_path / "added.py").write_text("NEW = 1\n", encoding="utf-8")
+
+ warm_parsed, warm_sources, warm_graph = await _ingest(tmp_path)
+ fresh_parsed, fresh_sources, fresh_graph = await _fresh_ingest(tmp_path)
+
+ assert [p.file_info.path for p in warm_parsed] == [p.file_info.path for p in fresh_parsed]
+ for wp, fp in zip(warm_parsed, fresh_parsed, strict=True):
+ assert wp == fp, wp.file_info.path
+ assert warm_sources == fresh_sources
+ assert nx.utils.graphs_equal(warm_graph, fresh_graph)