Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(graph): stop re-reading the repo during graph build (#1648)
Raghav Chamadiyacef5f83df655
diff --git a/packages/core/src/repowise/core/fs_walk.py b/packages/core/src/repowise/core/fs_walk.pyindex 4ddfeb2ea..33ee37d89 100644--- a/packages/core/src/repowise/core/fs_walk.py+++ b/packages/core/src/repowise/core/fs_walk.py@@ -377,6 +377,25 @@ class WalkSnapshot:yield dirpath / name+def glob_via(+ snapshot: WalkSnapshot | None,+ root: Path | str,+ patterns: str | Iterable[str],+ *,+ prune_nested_git: bool = True,+) -> Iterator[Path]:+ """:func:`iter_glob`, answered from *snapshot* when the caller has one.++ The seam for scans that run several queries against one tree and are+ reached from a context that can hold the walk. Yield order and matching+ are identical either way, so a caller cannot behave differently for+ having been given a snapshot.+ """+ if snapshot is not None:+ return snapshot.iter_glob(root, patterns)+ return iter_glob(root, patterns, prune_nested_git=prune_nested_git)++def iter_glob(root: Path | str,patterns: str | Iterable[str],diff --git a/packages/core/src/repowise/core/ingestion/graph/_resolvers.py b/packages/core/src/repowise/core/ingestion/graph/_resolvers.pyindex adf28e62e..7c6ca9b05 100644--- a/packages/core/src/repowise/core/ingestion/graph/_resolvers.py+++ b/packages/core/src/repowise/core/ingestion/graph/_resolvers.py@@ -102,7 +102,7 @@ class ResolveMixin:if progress:progress.on_phase_start(phase, None)try:- cs_texts = collect_csharp_source_texts(self._parsed_files)+ cs_texts = collect_csharp_source_texts(self._parsed_files, self._source_map)type_to_file = build_csharp_type_to_file(self._parsed_files)added = resolve_csharp_member_reads(self._graph, cs_texts, type_to_file)log.info("member_read_edges", language="csharp", added=added)@@ -141,7 +141,7 @@ class ResolveMixin:progress.on_phase_start(phase, None)try:jvm_index = get_or_build_jvm_index(ctx)- texts = collect_jvm_source_texts(self._parsed_files)+ texts = collect_jvm_source_texts(self._parsed_files, self._source_map)added = resolve_jvm_same_package_refs(self._graph, jvm_index, texts)log.info("same_package_edges", added=added)except Exception as exc:@@ -178,7 +178,7 @@ class ResolveMixin:progress.on_phase_start(phase, None)try:index = get_or_build_index(ctx)- cs_texts = collect_csharp_source_texts(self._parsed_files)+ cs_texts = collect_csharp_source_texts(self._parsed_files, self._source_map)repo = getattr(index, "repo_path", None) if index is not None else Noneadded = resolve_csharp_same_namespace_refs(self._graph, index, cs_texts, repo@@ -410,7 +410,7 @@ class ResolveMixin:progress.on_phase_start(phase, None)try:swift_targets = get_or_build_swift_targets(ctx)- texts = collect_swift_source_texts(self._parsed_files)+ texts = collect_swift_source_texts(self._parsed_files, self._source_map)added = resolve_swift_same_module_refs(self._graph, swift_targets, texts)log.info("same_module_edges", language="swift", added=added)except Exception as exc:diff --git a/packages/core/src/repowise/core/ingestion/languages/csharp_member_reads.py b/packages/core/src/repowise/core/ingestion/languages/csharp_member_reads.pyindex 3744e6bde..e6e46661b 100644--- a/packages/core/src/repowise/core/ingestion/languages/csharp_member_reads.py+++ b/packages/core/src/repowise/core/ingestion/languages/csharp_member_reads.py@@ -38,9 +38,9 @@ behind a feature flag if it ever proves noisy.from __future__ import annotationsimport re-from pathlib import Pathfrom typing import TYPE_CHECKING, Any+from ..source_text import source_textfrom ..type_names import bare_type_nameif TYPE_CHECKING:@@ -180,16 +180,22 @@ def build_csharp_type_to_file(parsed_files: dict[str, Any]) -> dict[str, str]:return result-def collect_csharp_source_texts(parsed_files: dict[str, Any]) -> dict[str, str]:- """Read each parsed C# file's source from disk, keyed by repo path."""+def collect_csharp_source_texts(+ parsed_files: dict[str, Any], source_map: dict[str, bytes] | None = None+) -> dict[str, str]:+ """Text of each parsed C# file, keyed by repo path.++ ``utf-8-sig``, unlike the JVM and Swift collectors: C# files are routinely+ written with a BOM, and the scans below must not see it inside the first+ identifier of the file.+ """out: dict[str, str] = {}for path, parsed in parsed_files.items():if parsed.file_info.language != "csharp":continue- try:- out[path] = Path(parsed.file_info.abs_path).read_text(- encoding="utf-8-sig", errors="ignore"- )- except OSError:- continue+ text = source_text(+ path, parsed.file_info.abs_path, source_map, encoding="utf-8-sig"+ )+ if text is not None:+ out[path] = textreturn outdiff --git a/packages/core/src/repowise/core/ingestion/languages/jvm_same_package.py b/packages/core/src/repowise/core/ingestion/languages/jvm_same_package.pyindex f75ac1078..5e420e9f4 100644--- a/packages/core/src/repowise/core/ingestion/languages/jvm_same_package.py+++ b/packages/core/src/repowise/core/ingestion/languages/jvm_same_package.py@@ -157,6 +157,8 @@ def resolve_jvm_same_package_refs()-def collect_jvm_source_texts(parsed_files: dict[str, Any]) -> dict[str, str]:- """Read each parsed JVM file's source from disk, keyed by repo path."""- return collect_source_texts(parsed_files, _JVM_LANGUAGES)+def collect_jvm_source_texts(+ parsed_files: dict[str, Any], source_map: dict[str, bytes] | None = None+) -> dict[str, str]:+ """Text of each parsed JVM file, keyed by repo path."""+ return collect_source_texts(parsed_files, _JVM_LANGUAGES, source_map)diff --git a/packages/core/src/repowise/core/ingestion/languages/scope_scan.py b/packages/core/src/repowise/core/ingestion/languages/scope_scan.pyindex ff87f4dcf..085081e1a 100644--- a/packages/core/src/repowise/core/ingestion/languages/scope_scan.py+++ b/packages/core/src/repowise/core/ingestion/languages/scope_scan.py@@ -12,9 +12,10 @@ from __future__ import annotationsimport refrom collections.abc import Callable, Collection, Iterablefrom dataclasses import dataclass, field-from pathlib import Pathfrom typing import TYPE_CHECKING, Any+from repowise.core.ingestion.source_text import source_text+if TYPE_CHECKING:import networkx as nx@@ -112,17 +113,16 @@ def emit_scope_edges(def collect_source_texts(- parsed_files: dict[str, Any], languages: Collection[str]+ parsed_files: dict[str, Any],+ languages: Collection[str],+ source_map: dict[str, bytes] | None = None,) -> dict[str, str]:- """Read each parsed file of *languages* from disk, keyed by repo path."""+ """Text of each parsed file of *languages*, keyed by repo path."""out: dict[str, str] = {}for path, parsed in parsed_files.items():if parsed.file_info.language not in languages:continue- try:- out[path] = Path(parsed.file_info.abs_path).read_text(- encoding="utf-8", errors="ignore"- )- except OSError:- continue+ text = source_text(path, parsed.file_info.abs_path, source_map)+ if text is not None:+ out[path] = textreturn outdiff --git a/packages/core/src/repowise/core/ingestion/languages/swift_same_module.py b/packages/core/src/repowise/core/ingestion/languages/swift_same_module.pyindex 79398bea8..c8599f6bc 100644--- a/packages/core/src/repowise/core/ingestion/languages/swift_same_module.py+++ b/packages/core/src/repowise/core/ingestion/languages/swift_same_module.py@@ -144,6 +144,8 @@ def resolve_swift_same_module_refs()-def collect_swift_source_texts(parsed_files: dict[str, Any]) -> dict[str, str]:- """Read each parsed Swift file's source from disk, keyed by repo path."""- return collect_source_texts(parsed_files, ("swift",))+def collect_swift_source_texts(+ parsed_files: dict[str, Any], source_map: dict[str, bytes] | None = None+) -> dict[str, str]:+ """Text of each parsed Swift file, keyed by repo path."""+ return collect_source_texts(parsed_files, ("swift",), source_map)diff --git a/packages/core/src/repowise/core/ingestion/resolvers/context.py b/packages/core/src/repowise/core/ingestion/resolvers/context.pyindex b7c0c4d14..a9fa9761b 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/context.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/context.py@@ -60,6 +60,38 @@ class ResolverContext:default=None, init=False, repr=False, compare=False)+ # One pruned walk of the repo, replayed for every resolver glob that asks+ # for it. Built on first use and never from ``path_set``: these scans look+ # for files the traverser does not index — ``META-INF/services`` entries,+ # ``spring.factories``, ``*.sln`` — so a snapshot of the indexed set would+ # answer "absent" for exactly the files they exist to find.+ _walk_snapshot_cache: Any | None = field(+ default=None, init=False, repr=False, compare=False+ )++ @property+ def walk_snapshot(self) -> Any | None:+ """Shared :class:`~repowise.core.fs_walk.WalkSnapshot`, or None.++ Rooted at the *resolved* repo path, because every scan that asks for+ this resolves its own root first; rooting it anywhere else would leave+ the snapshot correct but unused — a query it cannot place falls back to+ a live walk, which is exactly the cost this removes.+ """+ if self._walk_snapshot_cache is None:+ if self.repo_path is None:+ return None+ from repowise.core.fs_walk import WalkSnapshot++ try:+ root = self.repo_path.resolve()+ except OSError:+ root = self.repo_path+ self._walk_snapshot_cache = WalkSnapshot(+ root, prune_nested_git=self.prune_nested_git+ )+ return self._walk_snapshot_cache+@propertydef sorted_paths(self) -> tuple[str, ...]:"""Deterministically ordered view of ``path_set`` (cached)."""diff --git a/packages/core/src/repowise/core/ingestion/resolvers/cpp_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/cpp_workspace.pyindex 127a7f544..14e297705 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/cpp_workspace.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/cpp_workspace.py@@ -49,6 +49,7 @@ from ..external_systems.cmake import (discover_cmake_reactor,parse_cmake_file_api_reply,)+from ..source_text import decode_sourceif TYPE_CHECKING:from .context import ResolverContext@@ -238,15 +239,19 @@ _PROJECT_EXPORT_MACRO_RE = re.compile(_EXPORT_LIKE_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]{2,}_(?:EXPORT|API|PUBLIC|DLL|VISIBLE)$")+def _export_macros_in(text: str) -> tuple[str, ...]:+ # Limit to the first ~8KB — export macros are nearly always near the top.+ return tuple({m.group(1) for m in _PROJECT_EXPORT_MACRO_RE.finditer(text[:8192])})++@lru_cache(maxsize=8192)def _scan_header_for_export_macros(abs_path: str) -> tuple[str, ...]:+ """:func:`_export_macros_in` for a header the source map did not have."""try:text = Path(abs_path).read_text(encoding="utf-8", errors="ignore")except OSError:return ()- # Limit to the first ~8KB — export macros are nearly always near the top.- head = text[:8192]- return tuple({m.group(1) for m in _PROJECT_EXPORT_MACRO_RE.finditer(head)})+ return _export_macros_in(text)def _classify_dir(root_dir: str) -> dict[str, bool]:@@ -362,6 +367,11 @@ def build_cpp_workspace_index(ctx: ResolverContext) -> CppWorkspaceIndex:repo_path = ctx.repo_path.resolve()path_set = set(ctx.path_set)+ # ``getattr``, not attribute access: the call resolver builds these+ # indexes from a minimal stand-in context that carries neither field+ # (``call_resolver._Ctx``). A miss means disk and a live walk, which+ # is what that path did before.+ source_map = getattr(ctx, "source_map", None)cmake_files = discover_cmake_reactor(repo_path)file_api_targets = parse_cmake_file_api_reply(repo_path)@@ -409,8 +419,11 @@ def build_cpp_workspace_index(ctx: ResolverContext) -> CppWorkspaceIndex:if len(header_candidates) > 200:breakfor rel in header_candidates:- abs_p = str((repo_path / rel).resolve())- macros.update(_scan_header_for_export_macros(abs_p))+ data = source_map.get(rel) if source_map is not None else None+ if data is not None:+ macros.update(_export_macros_in(decode_source(data)))+ else:+ macros.update(_scan_header_for_export_macros(str((repo_path / rel).resolve())))_scan_header_for_export_macros.cache_clear()# Filter out anything that doesn't look like an export marker name.diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dart.py b/packages/core/src/repowise/core/ingestion/resolvers/dart.pyindex c91694a33..0a92bdec8 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/dart.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/dart.py@@ -21,6 +21,7 @@ import posixpathimport refrom typing import TYPE_CHECKING+from ..source_text import source_textfrom .module_name_index import get_or_build_module_index, lookup_moduleif TYPE_CHECKING:@@ -42,9 +43,10 @@ def _pubspec_map(ctx: ResolverContext) -> dict[str, str]:continueif repo is None:continue- try:- text = (repo / path).read_text(encoding="utf-8", errors="replace")- except OSError:+ text = source_text(+ path, repo / path, getattr(ctx, "source_map", None), errors="replace"+ )+ if text is None:continuem = _PUBSPEC_NAME_RE.search(text)if m:diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.pyindex c401009d6..2bb5566d9 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py@@ -4,11 +4,12 @@ from __future__ import annotationsfrom dataclasses import dataclass, fieldfrom pathlib import Path-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyimport structlog-from repowise.core.fs_walk import iter_glob+from repowise.core.fs_walk import glob_via+from repowise.core.ingestion.source_text import source_textfrom .global_usings import collect_project_global_usingsfrom .msbuild import (@@ -156,7 +157,9 @@ class DotNetProjectIndex:return package_id in self.package_refs.get(csproj, set())-def _walk_repo_cs_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:+def _walk_repo_cs_files(+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None+) -> list[Path]:"""Single repo-wide rglob for ``*.cs`` files, dedup by resolved path.Lives at module scope (not nested inside ``build_index``) so it's@@ -165,7 +168,7 @@ def _walk_repo_cs_files(repo_path: Path, *, prune_nested_git: bool = True) -> li"""seen: set[Path] = set()out: list[Path] = []- for cs in iter_glob(repo_path, "*.cs", prune_nested_git=prune_nested_git):+ for cs in glob_via(snapshot, repo_path, "*.cs", prune_nested_git=prune_nested_git):if path_has_dotnet_scan_skip_dir(cs, repo_path):continuetry:@@ -214,7 +217,13 @@ def _bucket_files_by_project(return out-def build_index(repo_path: Path, *, prune_nested_git: bool = True) -> DotNetProjectIndex:+def build_index(+ repo_path: Path,+ *,+ prune_nested_git: bool = True,+ snapshot: Any | None = None,+ source_map: dict[str, bytes] | None = None,+) -> DotNetProjectIndex:"""Walk *repo_path* and construct a fully-populated DotNetProjectIndex.Performance note: a previous version of this function walked the@@ -227,12 +236,20 @@ def build_index(repo_path: Path, *, prune_nested_git: bool = True) -> DotNetProjONE master walk, reads each file ONCE, and dispatches the cachedtexts to both the namespace-map pass and per-project global-usingscollection. No data-quality loss — same regexes, same outputs.++ That master read is now itself avoidable: ingestion already read every+ indexed ``.cs`` file, so *source_map* answers most of it and only the+ files the traverser skipped reach the disk. *snapshot* does the same for+ the three walks — .csproj, .sln, .cs — which become dict replays of the+ context's single walk."""repo_path = repo_path.resolve()index = DotNetProjectIndex(repo_path=repo_path)# ---- 1. Parse every .csproj ----- for csproj_path in find_csproj_files(repo_path, prune_nested_git=prune_nested_git):+ for csproj_path in find_csproj_files(+ repo_path, prune_nested_git=prune_nested_git, snapshot=snapshot+ ):proj = parse_csproj(csproj_path)if proj is None:continue@@ -241,7 +258,9 @@ def build_index(repo_path: Path, *, prune_nested_git: bool = True) -> DotNetProjindex.package_refs[proj.path] = set(proj.package_references)# ---- 2. Walk .sln files (informational; surfaces orphaned .csprojs) ----- index.sln_paths = find_sln_files(repo_path, prune_nested_git=prune_nested_git)+ index.sln_paths = find_sln_files(+ repo_path, prune_nested_git=prune_nested_git, snapshot=snapshot+ )for sln in index.sln_paths:for entry in parse_sln(sln):if entry.csproj not in index.projects:@@ -255,13 +274,21 @@ def build_index(repo_path: Path, *, prune_nested_git: bool = True) -> DotNetProjindex.package_refs.setdefault(proj.path, set()).update(proj.package_references)# ---- 3. Single master walk: enumerate .cs files & read each once ----- all_cs_files = _walk_repo_cs_files(repo_path, prune_nested_git=prune_nested_git)+ all_cs_files = _walk_repo_cs_files(+ repo_path, prune_nested_git=prune_nested_git, snapshot=snapshot+ )cs_texts: dict[Path, str] = {}for f in all_cs_files:+ # ``_walk_repo_cs_files`` resolves every path and *repo_path* is+ # resolved too, so a file the traverser indexed keys the map exactly.+ # Files it skipped — and anything outside the root — miss and read.try:- cs_texts[f] = f.read_text(encoding="utf-8-sig", errors="replace")- except OSError:- continue+ rel: str | None = f.relative_to(repo_path).as_posix()+ except ValueError:+ rel = None+ text = source_text(rel, f, source_map, encoding="utf-8-sig", errors="replace")+ if text is not None:+ cs_texts[f] = text# Build the file → project map by longest-prefix match. The bucketer# walks each file's parent chain (deepest-first) against a dict of@@ -326,6 +353,11 @@ def get_or_build_index(ctx: ResolverContext) -> DotNetProjectIndex | None:cached = getattr(ctx, _INDEX_KEY, None)if cached is not None:return cached- index = build_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)+ index = build_index(+ ctx.repo_path,+ prune_nested_git=ctx.prune_nested_git,+ snapshot=getattr(ctx, "walk_snapshot", None),+ source_map=getattr(ctx, "source_map", None),+ )setattr(ctx, _INDEX_KEY, index)return indexdiff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.pyindex 58f91bfc2..30ebf7aa6 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py@@ -10,11 +10,12 @@ from __future__ import annotationsfrom dataclasses import dataclass, fieldfrom pathlib import Path+from typing import Anyfrom xml.etree import ElementTree as ETimport structlog-from repowise.core.fs_walk import iter_glob+from repowise.core.fs_walk import glob_vialog = structlog.get_logger(__name__)@@ -113,10 +114,14 @@ def parse_csproj(csproj_path: Path) -> MSBuildProject | None:return project-def find_csproj_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:+def find_csproj_files(+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None+) -> list[Path]:"""Return all .csproj files under *repo_path*, skipping bin/obj output."""out: list[Path] = []- for csproj in iter_glob(repo_path, "*.csproj", prune_nested_git=prune_nested_git):+ for csproj in glob_via(+ snapshot, repo_path, "*.csproj", prune_nested_git=prune_nested_git+ ):if path_has_dotnet_scan_skip_dir(csproj, repo_path):continueout.append(csproj)diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.pyindex 1821f062a..eb7f73ffc 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py@@ -13,10 +13,11 @@ from __future__ import annotationsimport refrom dataclasses import dataclassfrom pathlib import Path+from typing import Anyimport structlog-from repowise.core.fs_walk import iter_glob+from repowise.core.fs_walk import glob_viafrom .msbuild import path_has_dotnet_scan_skip_dir@@ -60,9 +61,11 @@ def parse_sln(sln_path: Path) -> list[SolutionEntry]:return out-def find_sln_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:+def find_sln_files(+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None+) -> list[Path]:out: list[Path] = []- for sln in iter_glob(repo_path, "*.sln", prune_nested_git=prune_nested_git):+ for sln in glob_via(snapshot, repo_path, "*.sln", prune_nested_git=prune_nested_git):if path_has_dotnet_scan_skip_dir(sln, repo_path):continueout.append(sln)diff --git a/packages/core/src/repowise/core/ingestion/resolvers/go_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/go_workspace.pyindex 4260ba623..da79ecee7 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/go_workspace.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/go_workspace.py@@ -23,6 +23,7 @@ from typing import TYPE_CHECKINGimport structlog+from ..source_text import source_textfrom .go import read_go_modulesif TYPE_CHECKING:@@ -133,10 +134,10 @@ def _scan_go_file(text: str) -> tuple[str, bool, bool, bool, bool]:def _read_text(ctx: ResolverContext, rel_path: str) -> str:if ctx.repo_path is None:return ""- try:- return (ctx.repo_path / rel_path).read_text(encoding="utf-8", errors="ignore")- except OSError:- return ""+ # ``getattr``: the call resolver builds this index from a stand-in+ # context with no source map (``call_resolver._Ctx``).+ source_map = getattr(ctx, "source_map", None)+ return source_text(rel_path, ctx.repo_path / rel_path, source_map) or ""def _import_path_for_dir(diff --git a/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py b/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.pyindex d574983fd..4539c3043 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py@@ -15,11 +15,12 @@ import refrom dataclasses import dataclass, fieldfrom functools import lru_cachefrom pathlib import Path-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyimport structlog-from repowise.core.fs_walk import iter_glob+from repowise.core.fs_walk import glob_via+from repowise.core.ingestion.source_text import decode_sourceif TYPE_CHECKING:from .context import ResolverContext@@ -251,20 +252,17 @@ def _detect_plugins(module_dir: Path) -> frozenset[str]:# Package extraction (cached per file)# ----------------------------------------------------------------------------@lru_cache(maxsize=8192)-def _extract_file_info(abs_path: str) -> tuple[str, tuple[str, ...], bool, bool]:+def _extract_text_info(+ abs_path: str, text: str | None+) -> tuple[str, tuple[str, ...], bool, bool]:"""Return (package_decl, top_level_types, is_module_info, is_package_info).- Reads the file once and caches the result. Cheap line scan, no AST.+ Cheap line scan, no AST. *text* of None means the file was unreadable."""- path = Path(abs_path)- name = path.name+ name = Path(abs_path).nameis_module_info = name == "module-info.java"is_package_info = name == "package-info.java"-- try:- text = path.read_text(encoding="utf-8", errors="ignore")- except OSError:+ if text is None:return "", (), is_module_info, is_package_infopackage = ""@@ -284,12 +282,26 @@ def _extract_file_info(abs_path: str) -> tuple[str, tuple[str, ...], bool, bool]return package, tuple(top_level), is_module_info, is_package_info+@lru_cache(maxsize=8192)+def _extract_file_info(abs_path: str) -> tuple[str, tuple[str, ...], bool, bool]:+ """:func:`_extract_text_info` for a file the source map did not have."""+ try:+ text: str | None = Path(abs_path).read_text(encoding="utf-8", errors="ignore")+ except OSError:+ text = None+ return _extract_text_info(abs_path, text)++# ---------------------------------------------------------------------------# Index building# ---------------------------------------------------------------------------def build_jvm_gradle_index(- repo_path: Path | None, *, prune_nested_git: bool = True+ repo_path: Path | None,+ *,+ prune_nested_git: bool = True,+ snapshot: Any | None = None,+ source_map: dict[str, bytes] | None = None,) -> JvmGradleIndex:"""Walk Gradle config + JVM sources to build the package-resolution index."""index = JvmGradleIndex()@@ -351,14 +363,23 @@ def build_jvm_gradle_index(continuerel_roots.append(rel)- for jvm_file in iter_glob(root_path, "*", prune_nested_git=prune_nested_git):+ for jvm_file in glob_via(+ snapshot, root_path, "*", prune_nested_git=prune_nested_git+ ):if jvm_file.suffix not in _JVM_EXTENSIONS or not jvm_file.is_file():continuetry:rel_file = jvm_file.relative_to(resolved_repo).as_posix()except ValueError:continue- package, _types, _is_mod, _is_pkg = _extract_file_info(str(jvm_file.resolve()))+ abs_file = str(jvm_file.resolve())+ data = source_map.get(rel_file) if source_map is not None else None+ if data is not None:+ package, _types, _is_mod, _is_pkg = _extract_text_info(+ abs_file, decode_source(data)+ )+ else:+ package, _types, _is_mod, _is_pkg = _extract_file_info(abs_file)if not package:continueindex.package_to_files.setdefault(package, []).append(rel_file)@@ -382,7 +403,12 @@ def get_or_build_jvm_gradle_index(ctx: ResolverContext) -> JvmGradleIndex:cached = getattr(ctx, "_jvm_gradle_index", None)if cached is not None:return cached- index = build_jvm_gradle_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)+ index = build_jvm_gradle_index(+ ctx.repo_path,+ prune_nested_git=ctx.prune_nested_git,+ snapshot=getattr(ctx, "walk_snapshot", None),+ source_map=getattr(ctx, "source_map", None),+ )ctx._jvm_gradle_index = index # type: ignore[attr-defined]return indexdiff --git a/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.pyindex 424162004..ae676f02d 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py@@ -18,11 +18,12 @@ import refrom dataclasses import dataclass, fieldfrom functools import lru_cachefrom pathlib import Path-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyimport structlog-from repowise.core.fs_walk import iter_glob+from repowise.core.fs_walk import glob_via+from repowise.core.ingestion.source_text import decode_sourceif TYPE_CHECKING:from .context import ResolverContext@@ -206,30 +207,28 @@ class JvmWorkspaceIndex:return bool(len(parts) == 1 and parts[0] in _JAVA_LANG_TYPES)-@lru_cache(maxsize=16384)-def _scan_jvm_file(abs_path: str) -> tuple[str, tuple[str, ...]]:- """Return (package_fqn, top_level_type_names) for a JVM source file.+def _scan_jvm_text(abs_path: str, text: str) -> tuple[str, tuple[str, ...]]:+ """Return (package_fqn, top_level_type_names) from already-read text.- Reads the file once and caches the result. Cheap line scan — no AST.+ Cheap line scan — no AST."""- try:- text = Path(abs_path).read_text(encoding="utf-8", errors="ignore")- except OSError:- return "", ()-- package = ""if abs_path.endswith(".scala"):package = _scala_package(text)else:pkg_match = _PACKAGE_RE.search(text)- if pkg_match:- package = pkg_match.group(1)+ package = pkg_match.group(1) if pkg_match else ""- types: list[str] = []- for m in _TOP_LEVEL_TYPE_RE.finditer(text):- types.append(m.group(1))+ return package, tuple(m.group(1) for m in _TOP_LEVEL_TYPE_RE.finditer(text))- return package, tuple(types)++@lru_cache(maxsize=16384)+def _scan_jvm_file(abs_path: str) -> tuple[str, tuple[str, ...]]:+ """:func:`_scan_jvm_text` for a file the source map did not have."""+ try:+ text = Path(abs_path).read_text(encoding="utf-8", errors="ignore")+ except OSError:+ return "", ()+ return _scan_jvm_text(abs_path, text)def _scala_package(text: str) -> str:@@ -260,7 +259,7 @@ _JPMS_PROVIDES_RE = re.compile(def _scan_jpms_provides(- repo_path: Path, *, prune_nested_git: bool = True+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None) -> dict[str, tuple[str, ...]]:"""Scan ``module-info.java`` files for ``provides X with Y, Z`` directives.@@ -271,7 +270,9 @@ def _scan_jpms_provides(the warmup fast."""out: dict[str, list[str]] = {}- for mi in iter_glob(repo_path, "module-info.java", prune_nested_git=prune_nested_git):+ for mi in glob_via(+ snapshot, repo_path, "module-info.java", prune_nested_git=prune_nested_git+ ):if not mi.is_file():continuetry:@@ -287,12 +288,12 @@ def _scan_jpms_provides(def _scan_meta_inf_services(- repo_path: Path, *, prune_nested_git: bool = True+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None) -> dict[str, tuple[str, ...]]:"""Scan META-INF/services/ directories for SPI declarations."""services: dict[str, list[str]] = {}- for services_dir in iter_glob(- repo_path, "META-INF/services", prune_nested_git=prune_nested_git+ for services_dir in glob_via(+ snapshot, repo_path, "META-INF/services", prune_nested_git=prune_nested_git):if not services_dir.is_dir():continue@@ -321,14 +322,14 @@ def _scan_meta_inf_services(def _scan_spring_autoconfig(- repo_path: Path, *, prune_nested_git: bool = True+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None) -> dict[str, tuple[str, ...]]:"""Scan spring.factories and Boot 3 AutoConfiguration.imports."""result: dict[str, list[str]] = {}# spring.factories (Boot 2 style)- for factories in iter_glob(- repo_path, "META-INF/spring.factories", prune_nested_git=prune_nested_git+ for factories in glob_via(+ snapshot, repo_path, "META-INF/spring.factories", prune_nested_git=prune_nested_git):if not factories.is_file():continue@@ -358,8 +359,8 @@ def _scan_spring_autoconfig(result.setdefault(rel, []).extend(current_values)# Boot 3 style: META-INF/spring/*.imports- for imports_file in iter_glob(- repo_path, "META-INF/spring/*.imports", prune_nested_git=prune_nested_git+ for imports_file in glob_via(+ snapshot, repo_path, "META-INF/spring/*.imports", prune_nested_git=prune_nested_git):if not imports_file.is_file():continue@@ -382,9 +383,11 @@ def _scan_spring_autoconfig(def build_jvm_workspace_index(ctx: ResolverContext) -> JvmWorkspaceIndex:"""Build the JVM workspace index from all ``.java`` and ``.kt`` files in the path set.- One walk over the path set; each file read at most once (via- ``_scan_jvm_file`` LRU cache). The index groups files by package- and builds FQN → file mappings.+ One pass over the path set, taking each file's text from the source map+ the pipeline already filled and falling back to disk per file. The three+ resource scans below share the context's single walk rather than each+ walking the tree again — on a large JVM repo those walks, not the file+ scan, were the bulk of this function."""index = JvmWorkspaceIndex()@@ -392,6 +395,12 @@ def build_jvm_workspace_index(ctx: ResolverContext) -> JvmWorkspaceIndex:return indexrepo_path = ctx.repo_path.resolve()+ # ``getattr``, not attribute access: the call resolver builds these+ # indexes from a minimal stand-in context that carries neither field+ # (``call_resolver._Ctx``). A miss means disk and a live walk, which+ # is what that path did before.+ source_map = getattr(ctx, "source_map", None)+ snapshot = getattr(ctx, "walk_snapshot", None)# Group files by packagepkg_files: dict[str, list[str]] = {}@@ -402,7 +411,13 @@ def build_jvm_workspace_index(ctx: ResolverContext) -> JvmWorkspaceIndex:continueabs_path = str((repo_path / path).resolve())- package, top_types = _scan_jvm_file(abs_path)+ # The map lookup is here, not inside the cached scan: an lru_cache+ # cannot take a dict, and the fallback still wants its cache.+ data = source_map.get(path) if source_map is not None else None+ if data is not None:+ package, top_types = _scan_jvm_text(abs_path, decode_source(data))+ else:+ package, top_types = _scan_jvm_file(abs_path)if not package:continue@@ -432,8 +447,12 @@ def build_jvm_workspace_index(ctx: ResolverContext) -> JvmWorkspaceIndex:# (cheap glob, O(matching files)). Both populate ``services``; later# phases treat any FQN listed there as reachable.try:- services = _scan_meta_inf_services(repo_path, prune_nested_git=ctx.prune_nested_git)- jpms = _scan_jpms_provides(repo_path, prune_nested_git=ctx.prune_nested_git)+ services = _scan_meta_inf_services(+ repo_path, prune_nested_git=ctx.prune_nested_git, snapshot=snapshot+ )+ jpms = _scan_jpms_provides(+ repo_path, prune_nested_git=ctx.prune_nested_git, snapshot=snapshot+ )merged: dict[str, list[str]] = {k: list(v) for k, v in services.items()}for iface, impls in jpms.items():merged.setdefault(iface, []).extend(impls)@@ -442,7 +461,7 @@ def build_jvm_workspace_index(ctx: ResolverContext) -> JvmWorkspaceIndex:passwith contextlib.suppress(Exception):index.autoconfig_imports = _scan_spring_autoconfig(- repo_path, prune_nested_git=ctx.prune_nested_git+ repo_path, prune_nested_git=ctx.prune_nested_git, snapshot=snapshot)_scan_jvm_file.cache_clear()diff --git a/packages/core/src/repowise/core/ingestion/source_text.py b/packages/core/src/repowise/core/ingestion/source_text.pynew file mode 100644index 000000000..93abe1634--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/source_text.py@@ -0,0 +1,59 @@+"""One reader for file text the ingestion pipeline has already read.++The pipeline reads every indexed file, hands the bytes to+``GraphBuilder.set_source_map``, and they reach the resolvers as+``ResolverContext.source_map``. The scans that run during ``build()`` used to+re-open the repo anyway. This is the single place that prefers those bytes.++Two things a caller must not get wrong, which is why they live here rather than+at each site:++* **The codec is the caller's, not a default.** JVM and Swift scans decode+ utf-8, C# decodes utf-8-sig — only C# strips a BOM — and a couple of the+ .NET readers pass ``errors="replace"`` where the rest pass ``"ignore"``.+ Decoding a caller's bytes with another caller's codec silently changes what+ its regexes match, and therefore which edges it emits.+* **``Path.read_text`` translates newlines and ``bytes.decode`` does not.**+ Text mode maps ``\\r\\n`` and lone ``\\r`` to ``\\n``; the raw bytes in the+ source map still carry the ``\\r``. Every CRLF file — which on Windows is+ most of a C# checkout — would decode to different text than the disk read it+ replaces. :func:`decode_source` normalises, so a map hit and a disk fallback+ produce the same string for the same file.++The fallback is not optional. ``pipeline/incremental.py`` fills the map only+when its caller asks for sources, so it can be partial or absent entirely.+"""++from __future__ import annotations++from pathlib import Path+++def decode_source(data: bytes, *, encoding: str = "utf-8", errors: str = "ignore") -> str:+ """Decode *data* the way ``Path.read_text(encoding, errors)`` would."""+ text = data.decode(encoding, errors=errors)+ if "\r" in text:+ text = text.replace("\r\n", "\n").replace("\r", "\n")+ return text+++def source_text(+ rel_path: str | None,+ abs_path: str | Path,+ source_map: dict[str, bytes] | None,+ *,+ encoding: str = "utf-8",+ errors: str = "ignore",+) -> str | None:+ """Text of one file, from *source_map* if it holds it, else from disk.++ Returns None when the file is neither in the map nor readable.+ """+ if source_map is not None and rel_path is not None:+ data = source_map.get(rel_path)+ if data is not None:+ return decode_source(data, encoding=encoding, errors=errors)+ try:+ return Path(abs_path).read_text(encoding=encoding, errors=errors)+ except OSError:+ return Nonediff --git a/tests/unit/ingestion/test_source_text.py b/tests/unit/ingestion/test_source_text.pynew file mode 100644index 000000000..c422ebd48--- /dev/null+++ b/tests/unit/ingestion/test_source_text.py@@ -0,0 +1,70 @@+"""A source-map hit and a disk read must produce the same string.++Every scan that moved onto the source map keeps a disk fallback, so the two+paths run against the same repo in the same build. If they decode differently+the scan sees different text for the same file depending on whether the+pipeline happened to fill the map — which shows up as edges appearing and+disappearing, not as an error.++The failure this pins is not hypothetical: ``Path.read_text`` opens in text+mode and translates newlines, ``bytes.decode`` does not, and CRLF is the+majority line ending in a Windows-authored C# or Swift checkout.+"""++from __future__ import annotations++import pytest++from repowise.core.ingestion.source_text import decode_source, source_text++CASES = [+ ("lf", b"class A {}\nclass B {}\n"),+ ("crlf", b"class A {}\r\nclass B {}\r\n"),+ ("cr", b"class A {}\rclass B {}\r"),+ ("mixed", b"a\r\nb\rc\n"),+ ("bom_lf", b"\xef\xbb\xbfclass A {}\n"),+ ("bom_crlf", b"\xef\xbb\xbfclass A {}\r\n"),+ ("non_ascii", "// café ☕\r\nclass A {}\r\n".encode()),+ ("invalid_utf8", b"class A {}\r\n\xff\xfe\r\n"),+ ("empty", b""),+]++[email protected]("label,data", CASES, ids=[c[0] for c in CASES])[email protected]("encoding", ["utf-8", "utf-8-sig"])[email protected]("errors", ["ignore", "replace"])+def test_map_hit_matches_disk_read(tmp_path, label, data, encoding, errors):+ f = tmp_path / "Sample.cs"+ f.write_bytes(data)++ from_map = source_text(+ "Sample.cs", f, {"Sample.cs": data}, encoding=encoding, errors=errors+ )+ from_disk = source_text("Sample.cs", f, None, encoding=encoding, errors=errors)++ assert from_map == from_disk+ assert from_map == f.read_text(encoding=encoding, errors=errors)+++def test_a_miss_falls_back_to_disk():+ """An empty or partial map must not read as an empty file."""+ import tempfile+ from pathlib import Path++ with tempfile.TemporaryDirectory() as d:+ f = Path(d) / "Other.cs"+ f.write_bytes(b"class Other {}\r\n")+ assert source_text("Other.cs", f, {}) == "class Other {}\n"+ assert source_text("Other.cs", f, {"Unrelated.cs": b"x"}) == "class Other {}\n"+++def test_unreadable_file_returns_none(tmp_path):+ assert source_text("gone.cs", tmp_path / "gone.cs", None) is None+ assert source_text("gone.cs", tmp_path / "gone.cs", {}) is None+++def test_only_utf_8_sig_strips_the_bom():+ """The codecs differ by caller, and only C# asks for the BOM to go."""+ data = b"\xef\xbb\xbfclass A {}\n"+ assert decode_source(data, encoding="utf-8").startswith("")+ assert decode_source(data, encoding="utf-8-sig").startswith("class")